baseline_CNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import cv2\n",
    "import math\n",
    "import random\n",
    "import numpy as np\n",
    "from PIL import Image\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.utils.data as Data\n",
    "from torch.autograd import Variable\n",
    "from torchvision import datasets, transforms, models\n",
    "from sklearn.metrics import classification_report\n",
    "from torch import optim\n",
    "import time\n",
    "from utils.Conv2d import Conv2d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# parameters setting\n",
    "input_shape = [3,96,96]\n",
    "num_classes = 7\n",
    "num_layers = 5\n",
    "cut_size = 96\n",
    "init_lr = 0.0003\n",
    "batch_size = 128\n",
    "use_fer=True\n",
    "device= torch.device(\"cuda\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load data\n",
    "data_dir = ''\n",
    "\n",
    "if use_fer:\n",
    "    full_data = torch.load(data_dir + 'pickles/fer2013.pt')\n",
    "    full_img, full_emo = full_data['images'][:,None,:,:], full_data['label']\n",
    "\n",
    "    train_img,train_y = full_img[:28709],full_emo[:28709]\n",
    "    test_img,test_y = full_img[28709:28709+3589],full_emo[28709:28709+3589]\n",
    "\n",
    "    train_img = torch.cat((train_img,train_img,train_img),1)\n",
    "    test_img = torch.cat((test_img,test_img,test_img),1)\n",
    "\n",
    "else:\n",
    "    train_data = torch.load(data_dir + 'pickles/AffectNet_train.pt')\n",
    "    train_img, train_y = train_data['images'], train_data['emotion']\n",
    "    \n",
    "    test_data = torch.load(data_dir + 'pickles/AffectNet_test.pt')\n",
    "    test_img, test_y = test_data['images'], test_data['emotion']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset, DataLoader\n",
    "#======= data loaders =======#    \n",
    "\n",
    "train_transform = transforms.Compose([\n",
    "    transforms.ToPILImage(),\n",
    "    transforms.Resize([int(cut_size*1.1),int(cut_size*1.1)]),\n",
    "    transforms.RandomCrop([cut_size,cut_size]),\n",
    "    transforms.RandomHorizontalFlip(),\n",
    "    transforms.ToTensor()\n",
    "])\n",
    "\n",
    "test_transform = transforms.Compose([\n",
    "    transforms.ToPILImage(),\n",
    "    transforms.Resize([int(cut_size*1.1),int(cut_size*1.1)]),\n",
    "    transforms.CenterCrop([cut_size,cut_size]),\n",
    "    transforms.ToTensor()\n",
    "])\n",
    "\n",
    "def default_loader(images):\n",
    "    img_tensor = train_transform(images)\n",
    "    return img_tensor\n",
    "\n",
    "def default_test_loader(images):\n",
    "    img_tensor = test_transform(images)\n",
    "    return img_tensor\n",
    "\n",
    "class Dataset(Dataset):\n",
    "    def __init__(self,imgs,ys,loader=default_loader):\n",
    "        self.images = imgs \n",
    "        self.label = ys\n",
    "        self.loader = loader\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "\n",
    "        fn = self.images[index]\n",
    "        img = self.loader(fn)\n",
    "        label = self.label[index]\n",
    "        return img,label\n",
    "\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.images)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CNN(nn.Module):\n",
    "\n",
    "    def __init__(self,lr=0.0001,num_classes=7):\n",
    "        super(CNN, self).__init__()\n",
    "        \n",
    "        self.classifier1 = nn.Sequential(\n",
    "            Conv2d(in_channels=3, out_channels=32, kernel_size=3, stride=1),\n",
    "            nn.ReLU(),\n",
    "            nn.BatchNorm2d(32),\n",
    "            # conv_1\n",
    "            Conv2d(in_channels=32, out_channels=64, kernel_size=2, stride=2),\n",
    "            nn.ReLU(),\n",
    "            nn.BatchNorm2d(64),\n",
    "            # conv_2\n",
    "            Conv2d(in_channels=64, out_channels=128, kernel_size=2, stride=2),\n",
    "            nn.ReLU(),\n",
    "            nn.BatchNorm2d(128),\n",
    "            # conv_3\n",
    "            Conv2d(in_channels=128, out_channels=256, kernel_size=2, stride=2),\n",
    "            nn.ReLU(),\n",
    "            nn.BatchNorm2d(256),\n",
    "            # conv_4\n",
    "            Conv2d(in_channels=256, out_channels=512, kernel_size=2, stride=2),\n",
    "            nn.ReLU(),\n",
    "            nn.BatchNorm2d(512),\n",
    "            nn.Conv2d(512,1024,3,1),\n",
    "            nn.ReLU(),\n",
    "            nn.BatchNorm2d(1024),\n",
    "            nn.Conv2d(1024,1024,3,1),\n",
    "            nn.ReLU(),\n",
    "            nn.BatchNorm2d(1024),\n",
    "        )\n",
    "        self.classifier2 = nn.Sequential(\n",
    "            nn.Linear(4096,1024),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(1024,7)\n",
    "        )\n",
    "        \n",
    "        self.loss_fn = nn.CrossEntropyLoss() #FocalLoss(4.0)\n",
    "        self.optimizer=torch.optim.Adam([p for p in self.parameters() if p.requires_grad] , lr=lr)\n",
    "        \n",
    "\n",
    "    def train_model(self, data_loader, num_epochs, lr):\n",
    "        self.train()\n",
    "        start=time.time()\n",
    "\n",
    "        for epoch in range(num_epochs):\n",
    "            running_loss=0\n",
    "            running_acc = 0.\n",
    "            total = 0\n",
    "            num_batches = 0\n",
    "            for count, (images, label) in enumerate(data_loader):\n",
    "                self.optimizer.zero_grad()\n",
    "\n",
    "                scores = 0.\n",
    "                \n",
    "                images=images.to(device)\n",
    "\n",
    "                x = self.classifier1(images).view(images.size(0),-1)\n",
    "                scores = self.classifier2(x)\n",
    "\n",
    "                label=label.to(device)\n",
    "                loss =  self.loss_fn(scores, label) \n",
    "                loss.backward()\n",
    "\n",
    "                self.optimizer.step()\n",
    "\n",
    "                running_loss += loss.detach().item()\n",
    "\n",
    "                acc = (scores.argmax(-1) == label).float().sum()\n",
    "                running_acc += acc.item()\n",
    "\n",
    "                total+=images.size(0)\n",
    "                num_batches += 1\n",
    "                \n",
    "            total_loss = running_loss/num_batches\n",
    "            total_acc = running_acc/total\n",
    "            elapsed = time.time()-start\n",
    "            print('\\t', 'epoch=',epoch, '\\t time = {:.0f}m {:.0f}s'.format(elapsed // 60, elapsed % 60),'\\t lr=', lr,'\\t loss = ', total_loss , '\\t Acc = {:.6f}'.format(total_acc*100),'%')\n",
    "\n",
    "    def test_model(self, data_loader):\n",
    "        self.eval()\n",
    "        start=time.time()\n",
    "        \n",
    "        running_loss = 0.\n",
    "        running_acc = 0\n",
    "        total = 0\n",
    "        num_batches = 0\n",
    "        \n",
    "        preds = []\n",
    "        truth = []\n",
    "        total_enc_im = []\n",
    "        total_dec_im = []\n",
    "        \n",
    "        for count, (images, label) in enumerate(data_loader):\n",
    "            self.optimizer.zero_grad()\n",
    "            scores = 0.\n",
    "            images = images.to(device)\n",
    "\n",
    "            x = self.classifier1(images).view(images.size(0),-1)\n",
    "            scores = self.classifier2(x).detach().cpu()\n",
    "\n",
    "            acc = (scores.argmax(-1) == label).float().sum()\n",
    "            running_acc += acc.item()\n",
    "            \n",
    "            total += images.size(0)\n",
    "            num_batches += 1\n",
    "\n",
    "            preds.extend(scores.argmax(-1).numpy())\n",
    "            truth.extend(label.numpy())\n",
    "            \n",
    "        total_acc = running_acc/total\n",
    "        elapsed = time.time()-start\n",
    "\n",
    "        print('testing','\\t time = {:.0f}m {:.0f}s'.format(elapsed // 60, elapsed % 60), '\\n Acc = {:.6f}'.format(total_acc*100),'%')\n",
    "        test_fscore = classification_report(truth, preds)\n",
    "        print(test_fscore)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AlexNet(nn.Module):\n",
    "\n",
    "    def __init__(self,lr=0.0001,num_classes=7):\n",
    "        super(AlexNet, self).__init__()\n",
    "        self.model = models.alexnet(pretrained=True)\n",
    "        self.model.classifier[1] = nn.Linear(1024, 4096)\n",
    "        self.model.classifier[-1] = nn.Linear(4096,7)\n",
    "        self.loss_fn = nn.CrossEntropyLoss() #FocalLoss(4.0)\n",
    "        self.optimizer=torch.optim.Adam([p for p in self.parameters() if p.requires_grad] , lr=lr)\n",
    "        \n",
    "\n",
    "    def train_model(self, data_loader, num_epochs, lr):\n",
    "        self.train()\n",
    "        start=time.time()\n",
    "\n",
    "        for epoch in range(num_epochs):\n",
    "            running_loss=0\n",
    "            running_acc = 0.\n",
    "            total = 0\n",
    "            num_batches = 0\n",
    "            for count, (images, label) in enumerate(data_loader):\n",
    "                self.optimizer.zero_grad()\n",
    "\n",
    "                scores = 0.\n",
    "                \n",
    "                images=images.to(device)\n",
    "\n",
    "                x = self.model.features(images).view(images.size(0),-1)\n",
    "                scores = self.model.classifier(x)\n",
    "\n",
    "                label=label.to(device)\n",
    "                loss =  self.loss_fn(scores, label) \n",
    "                loss.backward()\n",
    "\n",
    "                self.optimizer.step()\n",
    "\n",
    "                running_loss += loss.detach().item()\n",
    "\n",
    "                acc = (scores.argmax(-1) == label).float().sum()\n",
    "                running_acc += acc.item()\n",
    "\n",
    "                total+=images.size(0)\n",
    "                num_batches += 1\n",
    "                \n",
    "            total_loss = running_loss/num_batches\n",
    "            total_acc = running_acc/total\n",
    "            elapsed = time.time()-start\n",
    "            print('\\t', 'epoch=',epoch, '\\t time = {:.0f}m {:.0f}s'.format(elapsed // 60, elapsed % 60),'\\t lr=', lr,'\\t loss = ', total_loss , '\\t Acc = {:.6f}'.format(total_acc*100),'%')\n",
    "\n",
    "    def test_model(self, data_loader):\n",
    "        self.eval()\n",
    "        start=time.time()\n",
    "        \n",
    "        running_loss = 0.\n",
    "        running_acc = 0\n",
    "        total = 0\n",
    "        num_batches = 0\n",
    "        \n",
    "        preds = []\n",
    "        truth = []\n",
    "        \n",
    "        for count, (images, label) in enumerate(data_loader):\n",
    "            self.optimizer.zero_grad()\n",
    "            \n",
    "            images = images.to(device)\n",
    "\n",
    "            x = self.model.features(images).view(images.size(0),-1)\n",
    "            scores = self.model.classifier(x).detach().cpu()\n",
    "\n",
    "            acc = (scores.argmax(-1) == label).float().sum()\n",
    "            running_acc += acc.item()\n",
    "            \n",
    "            total += images.size(0)\n",
    "\n",
    "            preds.extend(scores.argmax(-1).numpy())\n",
    "            truth.extend(label.numpy())\n",
    "            \n",
    "        total_acc = running_acc/total\n",
    "        elapsed = time.time()-start\n",
    "\n",
    "        print('testing','\\t time = {:.0f}m {:.0f}s'.format(elapsed // 60, elapsed % 60), '\\n Acc = {:.6f}'.format(total_acc*100),'%')\n",
    "        test_fscore = classification_report(truth, preds)\n",
    "        print(test_fscore)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.Layers import LRN\n",
    "\n",
    "class JigsawPuzzle(nn.Module):\n",
    "\n",
    "    def __init__(self, lr=0.001,classes=1000):\n",
    "        super(JigsawPuzzle, self).__init__()\n",
    "\n",
    "        self.conv = nn.Sequential()\n",
    "        self.conv.add_module('conv1_s1',nn.Conv2d(3, 96, kernel_size=11, stride=2, padding=0))\n",
    "        self.conv.add_module('relu1_s1',nn.ReLU(inplace=True))\n",
    "        self.conv.add_module('pool1_s1',nn.MaxPool2d(kernel_size=3, stride=2))\n",
    "        self.conv.add_module('lrn1_s1',LRN(local_size=5, alpha=0.0001, beta=0.75))\n",
    "\n",
    "        self.conv.add_module('conv2_s1',nn.Conv2d(96, 256, kernel_size=5, padding=2, groups=2))\n",
    "        self.conv.add_module('relu2_s1',nn.ReLU(inplace=True))\n",
    "        self.conv.add_module('pool2_s1',nn.MaxPool2d(kernel_size=3, stride=2))\n",
    "        self.conv.add_module('lrn2_s1',LRN(local_size=5, alpha=0.0001, beta=0.75))\n",
    "\n",
    "        self.conv.add_module('conv3_s1',nn.Conv2d(256, 384, kernel_size=3, padding=1))\n",
    "        self.conv.add_module('relu3_s1',nn.ReLU(inplace=True))\n",
    "\n",
    "        self.conv.add_module('conv4_s1',nn.Conv2d(384, 384, kernel_size=3, padding=1, groups=2))\n",
    "        self.conv.add_module('relu4_s1',nn.ReLU(inplace=True))\n",
    "\n",
    "        self.conv.add_module('conv5_s1',nn.Conv2d(384, 256, kernel_size=3, padding=1, groups=2))\n",
    "        self.conv.add_module('relu5_s1',nn.ReLU(inplace=True))\n",
    "        self.conv.add_module('pool5_s1',nn.MaxPool2d(kernel_size=3, stride=2))\n",
    "\n",
    "        self.fc6 = nn.Sequential()\n",
    "        self.fc6.add_module('fc6_s1',nn.Linear(4096, 1024))\n",
    "        self.fc6.add_module('relu6_s1',nn.ReLU(inplace=True))\n",
    "        self.fc6.add_module('drop6_s1',nn.Dropout(p=0.5))\n",
    "\n",
    "#         self.fc7 = nn.Sequential()\n",
    "#         self.fc7.add_module('fc7',nn.Linear(9*1024,4096))\n",
    "#         self.fc7.add_module('relu7',nn.ReLU(inplace=True))\n",
    "#         self.fc7.add_module('drop7',nn.Dropout(p=0.5))\n",
    "\n",
    "        self.classifier = nn.Sequential()\n",
    "        self.classifier.add_module('fc7',nn.Linear(1024, classes))\n",
    "        \n",
    "#         self.apply(weights_init)\n",
    "        \n",
    "        \n",
    "        self.loss_fn = nn.CrossEntropyLoss()\n",
    "        self.optimizer=torch.optim.Adam([p for p in self.parameters() if p.requires_grad] , lr=lr)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        bs = x.size(0)\n",
    "        x = self.conv(x).view(bs,-1)\n",
    "        x = self.fc6(x)\n",
    "#         z = z.view([B,1,-1])\n",
    "\n",
    "#         x = self.fc7(x)\n",
    "        x = self.classifier(x)\n",
    "\n",
    "        return x\n",
    "    \n",
    "    def train_model(self, data_loader, num_epochs, lr=0.001, min_loss=5.):\n",
    "        self.train()\n",
    "        start=time.time()\n",
    "\n",
    "        for epoch in range(num_epochs):\n",
    "            running_loss=0\n",
    "            running_acc = 0.\n",
    "            total = 0\n",
    "            num_batches = 0\n",
    "            for count, (images, label) in enumerate(data_loader):\n",
    "                self.optimizer.zero_grad()\n",
    "                \n",
    "                images=images.to(device)\n",
    "                label=torch.squeeze(label).to(device)\n",
    "\n",
    "                scores = self.forward(images)\n",
    "\n",
    "                loss =  self.loss_fn(scores, label) \n",
    "                loss.backward()\n",
    "\n",
    "                self.optimizer.step()\n",
    "\n",
    "                running_loss += loss.detach().item()\n",
    "\n",
    "                acc = (scores.argmax(-1) == label).float().sum()\n",
    "                running_acc += acc.item()\n",
    "\n",
    "                total+=images.size(0)\n",
    "                num_batches += 1\n",
    "            total_loss = running_loss/num_batches\n",
    "            \n",
    "            total_acc = running_acc/total\n",
    "            elapsed = time.time()-start\n",
    "            print('\\t', 'epoch=',epoch, '\\t time = {:.0f}m {:.0f}s'.format(elapsed // 60, elapsed % 60),'\\t lr=', lr,'\\t loss = ', total_loss , '\\t Acc = {:.6f}'.format(total_acc*100),'%')\n",
    "        return total_loss\n",
    "    \n",
    "    def test_model(self, data_loader):\n",
    "        self.eval()\n",
    "        start=time.time()\n",
    "        \n",
    "        running_loss = 0.\n",
    "        running_acc = 0\n",
    "        total = 0\n",
    "        num_batches = 0\n",
    "        \n",
    "        preds = []\n",
    "        truth = []\n",
    "        \n",
    "        for count, (images, label) in enumerate(data_loader):\n",
    "            self.optimizer.zero_grad()\n",
    "            \n",
    "            images = images.to(device)\n",
    "\n",
    "            scores = self.forward(images).detach().cpu()\n",
    "\n",
    "            acc = (scores.argmax(-1) == label).float().sum()\n",
    "            running_acc += acc.item()\n",
    "            \n",
    "            total += images.size(0)\n",
    "\n",
    "            preds.extend(scores.argmax(-1).numpy())\n",
    "            truth.extend(label.numpy())\n",
    "            \n",
    "        total_acc = running_acc/total\n",
    "        elapsed = time.time()-start\n",
    "\n",
    "        print('testing','\\t time = {:.0f}m {:.0f}s'.format(elapsed // 60, elapsed % 60), '\\n Acc = {:.6f}'.format(total_acc*100),'%')\n",
    "        test_fscore = classification_report(truth, preds)\n",
    "        print(test_fscore)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_input  = Dataset(train_img,train_y)\n",
    "train_loader = DataLoader(train_input, batch_size=batch_size,shuffle=True)\n",
    "\n",
    "test_input  = Dataset(test_img,test_y,default_test_loader)\n",
    "test_loader = DataLoader(test_input, batch_size=batch_size,shuffle=False)\n",
    "\n",
    "net = CNN(lr = init_lr, num_classes=7)\n",
    "net.to(device)\n",
    "print(net)\n",
    "for i in range(20):\n",
    "    print('Iteration', i)\n",
    "    net.train_model(data_loader=train_loader, num_epochs=5,lr=init_lr)\n",
    "    preds=net.test_model(data_loader=test_loader)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}