08_Generative_Model/图像生成_GenerativeMSE.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 镜像与猫鼠游戏——反卷积生成网络与DCGAN\n",
    "\n",
    "本文件是集智学园开发的“火炬上的深度学习”系列课程第五节课：《镜像网络与猫鼠游戏》的配套文档。在本文档中，我们实现了一个反卷积神经网络生成器，它可以通过输入的信号，生成一张逼真的手写数字图像。为了让我们的生成图像能够足够逼真，我们尝试了三种不同的方法：\n",
    "\n",
    "1、在MINST数据集中，选出一个样本，输入数字标签，输出图像，并让输出的图像与样本图像尽可能相似，总误差最小化；\n",
    "2、同上，只不过并不直接比较输出和样本相似性，而是让一个已训练好的手写数字识别网络来判断这个伪造的图像是几；\n",
    "3、DCGAN，同时训练一个生成器一个判别器。每个时刻随机采样一个向量输入给生成器，它输出一张图像，同时读取一个数据样本，判别器判断样本图像\n",
    "和生成图像的真假。\n",
    "\n",
    "本文件是集智学园http://campus.swarma.org 出品的“火炬上的深度学习”第V课的配套源代码\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导入所需要的包，请保证torchvision已经在你的环境中安装好.\n",
    "# 在Windows需要单独安装torchvision包，在命令行运行pip install torchvision即可\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import torchvision.datasets as dsets\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.utils as vutil\n",
    "\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "import os\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义超参数\n",
    "image_size = 28 #图像尺寸大小\n",
    "input_dim = 100 #输入给生成器的向量维度，维度越大可以增加生成器输出样本的多样性\n",
    "num_channels = 1# 图像的通道数\n",
    "num_features = 64 #生成器中间的卷积核数量\n",
    "batch_size = 64 #批次大小\n",
    "\n",
    "# 如果系统中存在着GPU，我们将用GPU来完成张量的计算\n",
    "use_cuda = torch.cuda.is_available() #定义一个布尔型变量，标志当前的GPU是否可用\n",
    "\n",
    "# 如果当前GPU可用，则将优先在GPU上进行张量计算\n",
    "dtype = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor\n",
    "itype = torch.cuda.LongTensor if use_cuda else torch.LongTensor\n",
    "\n",
    "# 加载MINIST数据，如果没有下载过，就会在当前路径下新建/data子目录，并把文件存放其中\n",
    "# MNIST数据是属于torchvision包自带的数据，所以可以直接调用。\n",
    "# 在调用自己的数据的时候，我们可以用torchvision.datasets.ImageFolder或者torch.utils.data.TensorDataset来加载\n",
    "train_dataset = dsets.MNIST(root='./data',  #文件存放路径\n",
    "                            train=True,   #提取训练集\n",
    "                            transform=transforms.ToTensor(),  #将图像转化为Tensor，在加载数据的时候，就可以对图像做预处理\n",
    "                            download=True) #当找不到文件的时候，自动下载\n",
    "\n",
    "# 加载测试数据集\n",
    "test_dataset = dsets.MNIST(root='./data', \n",
    "                           train=False, \n",
    "                           transform=transforms.ToTensor())\n",
    "\n",
    "# 训练数据集的加载器，自动将数据分割成batch，顺序随机打乱\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n",
    "                                           batch_size=batch_size, \n",
    "                                           shuffle=True)\n",
    "\n",
    "'''我们希望将测试数据分成两部分，一部分作为校验数据，一部分作为测试数据。\n",
    "校验数据用于检测模型是否过拟合，并调整参数，测试数据检验整个模型的工作'''\n",
    "\n",
    "\n",
    "# 首先，我们定义下标数组indices，它相当于对所有test_dataset中数据的编码\n",
    "# 然后定义下标indices_val来表示校验集数据的那些下标，indices_test表示测试集的下标\n",
    "indices = range(len(test_dataset))\n",
    "indices_val = indices[:5000]\n",
    "indices_test = indices[5000:]\n",
    "\n",
    "# 根据这些下标，构造两个数据集的SubsetRandomSampler采样器，它会对下标进行采样\n",
    "sampler_val = torch.utils.data.sampler.SubsetRandomSampler(indices_val)\n",
    "sampler_test = torch.utils.data.sampler.SubsetRandomSampler(indices_test)\n",
    "\n",
    "# 根据两个采样器来定义加载器，注意将sampler_val和sampler_test分别赋值给了validation_loader和test_loader\n",
    "validation_loader = torch.utils.data.DataLoader(dataset =test_dataset,\n",
    "                                                batch_size = batch_size,\n",
    "                                                sampler = sampler_val\n",
    "                                               )\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n",
    "                                          batch_size=batch_size, \n",
    "                                          sampler = sampler_test\n",
    "                                         )\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 一、生成器预测图像模型\n",
    "\n",
    "在这个模型中，我们根据输入的手写数字生成一张图像，并让这个图像与数据中的样本图像尽可能一致"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#生成器模型定义\n",
    "\n",
    "class ModelG(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(ModelG,self).__init__()\n",
    "        self.model=nn.Sequential() #model为一个内嵌的序列化的神经网络模型\n",
    "        \n",
    "        # 利用add_module增加一个反卷积层，输入为input_dim维，输出为2*num_features维，窗口大小为5，padding是0\n",
    "        # 输入图像大小为1，输出图像大小为W'=(W-1)S-2P+K+P'=(1-1)*2-2*0+5+0=3, 5*5\n",
    "        self.model.add_module('deconv1',nn.ConvTranspose2d(input_dim, num_features*2, 5, 2, 0, bias=False))\n",
    "        # 增加一个batchnorm层\n",
    "        self.model.add_module('bnorm1',nn.BatchNorm2d(num_features*2))\n",
    "        # 增加非线性层\n",
    "        self.model.add_module('relu1',nn.ReLU(True))\n",
    "        # 增加第二层反卷积层，输入2*num_features维，输出num_features维，窗口5，padding=0\n",
    "        # 输入图像大小为5，输出图像大小为W'=(W-1)S-2P+K+P'=(5-1)*2-2*0+5+0=13, 13*13\n",
    "        self.model.add_module('deconv2',nn.ConvTranspose2d(num_features*2, num_features, 5, 2, 0, bias=False))\n",
    "        # 增加一个batchnorm层\n",
    "        self.model.add_module('bnorm2',nn.BatchNorm2d(num_features))\n",
    "        # 增加非线性层\n",
    "        self.model.add_module('relu2',nn.ReLU(True))\n",
    "\n",
    "        # 增加第二层反卷积层，输入2*num_features维，输出num_features维，窗口4，padding=0\n",
    "        # 输入图像大小为13，输出图像大小为W'=(W-1)S-2P+K+P'=(13-1)*2-2*0+4+0=28, 28*28\n",
    "        self.model.add_module('deconv3',nn.ConvTranspose2d(num_features, num_channels, 4, 2, 0,bias=False))\n",
    "        #self.model.add_module('tanh',nn.Tanh())\n",
    "        self.model.add_module('sigmoid',nn.Sigmoid())\n",
    "    def forward(self,input):\n",
    "        output = input\n",
    "        \n",
    "        #遍历网络的所有层，一层层输出信息\n",
    "        for name, module in self.model.named_children():\n",
    "            output = module(output)\n",
    "        #输出一张28*28的图像\n",
    "        return(output)\n",
    "\n",
    "\n",
    "def weight_init(m):\n",
    "    #模型参数初始化．\n",
    "    #默认的初始化参数卷积核的权重是均值大概为0，方差在10^{-2}. BatchNorm层的权重均值是大约0.5，方差在0.2左右\n",
    "    #使用如下初始化方式可以，可以让方差更小，使得收敛更快\n",
    "    class_name=m.__class__.__name__\n",
    "    if class_name.find('conv')!=-1:\n",
    "        m.weight.data.normal_(0,0.02)\n",
    "    if class_name.find('norm')!=-1:\n",
    "        m.weight.data.normal_(1.0,0.02)\n",
    "        \n",
    "def make_show(img):\n",
    "    # 将张量变成可以显示的图像\n",
    "    img = img.data.expand(batch_size, 3, image_size, image_size)\n",
    "    return img\n",
    "\n",
    "def imshow(inp, title=None):\n",
    "    # 在屏幕上绘制图像\n",
    "    \"\"\"Imshow for Tensor.\"\"\"\n",
    "    if inp.size()[0] > 1:\n",
    "        inp = inp.numpy().transpose((1, 2, 0))\n",
    "    else:\n",
    "        inp = inp[0].numpy()\n",
    "    mvalue = np.amin(inp)\n",
    "    maxvalue = np.amax(inp)\n",
    "    if maxvalue > mvalue:\n",
    "        inp = (inp - mvalue)/(maxvalue - mvalue)\n",
    "    plt.imshow(inp)\n",
    "    if title is not None:\n",
    "        plt.title(title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#训练模型\n",
    "\n",
    "print('Initialized!')\n",
    "\n",
    "#定义生成器模型\n",
    "net = ModelG()\n",
    "net = net.cuda() if use_cuda else net\n",
    "\n",
    "#目标函数采用最小均方误差\n",
    "criterion = nn.MSELoss()\n",
    "#定义优化器\n",
    "optimizer = optim.SGD(net.parameters(), lr=0.0001, momentum=0.9)\n",
    "\n",
    "# 随机选择生成0-9的数字，用于每个周期打印查看结果用\n",
    "samples = np.random.choice(10, batch_size) \n",
    "samples = torch.from_numpy(samples).type(dtype)\n",
    "\n",
    "#开始训练\n",
    "step = 0 #计数经历了多少时间步\n",
    "num_epochs = 100 #总的训练周期\n",
    "record = []\n",
    "for epoch in range(num_epochs):\n",
    "    train_loss = []\n",
    "    \n",
    "    # 加载数据批次\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        # 注意数据中的data转化为了要预测的target，数据中的target则转化成了输入给网络的标签\n",
    "        target, data = data.clone().detach().requires_grad_(True), target.clone().detach() #data为一批图像，target为一批标签\n",
    "        # 将数据加载到GPU中\n",
    "        if use_cuda:\n",
    "            target, data = target.cuda(), data.cuda()\n",
    "        #将输入的数字标签转化为生成器net能够接受的(batch_size, input_dim, 1, 1)维张量\n",
    "        data = data.type(dtype)\n",
    "        data = data.resize(data.size()[0], 1, 1, 1)\n",
    "        data = data.expand(data.size()[0], input_dim, 1, 1)\n",
    "\n",
    "        net.train() # 给网络模型做标记，标志说模型正在训练集上训练，\n",
    "                    #这种区分主要是为了打开关闭net的training标志\n",
    "        output = net(data) #神经网络完成一次前馈的计算过程，得到预测输出output\n",
    "        loss = criterion(output, target) #将output与标签target比较，计算误差\n",
    "        optimizer.zero_grad() #清空梯度\n",
    "        loss.backward() #反向传播\n",
    "        optimizer.step() #一步随机梯度下降算法\n",
    "        step += 1\n",
    "        # 记载损失函数值\n",
    "        if use_cuda:\n",
    "            loss = loss.cpu()\n",
    "        train_loss.append(loss.data.numpy())\n",
    "        \n",
    "        \n",
    "        if step % 100 == 0: #每间隔100个batch执行一次打印等操作    \n",
    "            net.eval() # 给网络模型做标记，标志说模型在校验集上运行\n",
    "            val_loss = [] #记录校验数据集准确率的容器\n",
    "            \n",
    "            '''开始在校验数据集上做循环，计算校验集上面的准确度'''\n",
    "            idx = 0\n",
    "            for (data, target) in validation_loader:\n",
    "                target, data = data.clone().detach().requires_grad_(True), target.clone().detach()\n",
    "                idx += 1\n",
    "                if use_cuda:\n",
    "                    target, data = target.cuda(), data.cuda()\n",
    "                data = data.type(dtype)\n",
    "                data = data.resize(data.size()[0], 1, 1, 1)\n",
    "                data = data.expand(data.size()[0], input_dim, 1, 1)\n",
    "                output = net(data) #完成一次前馈计算过程，得到目前训练得到的模型net在校验数据集上的表现\n",
    "                loss = criterion(output, target) #将output与标签target比较，计算误差\n",
    "                if use_cuda:\n",
    "                    loss = loss.cpu()\n",
    "                val_loss.append(loss.data.numpy())\n",
    "                #打印误差等数值，其中正确率为本训练周期Epoch开始后到目前撮的正确率的平均值\n",
    "            print('训练周期: {} [{}/{} ({:.0f}%)]\\t训练数据Loss: {:.6f}\\t校验数据Loss: {:.6f}'.format(\n",
    "                epoch, batch_idx * batch_size, len(train_loader.dataset),\n",
    "                100. * batch_idx / len(train_loader), np.mean(train_loss), np.mean(val_loss)))\n",
    "            record.append([np.mean(train_loss), np.mean(val_loss)])\n",
    "        \n",
    "    \n",
    "    # 产生一组图像保存到temp1文件夹下（需要事先建立好该文件夹），检测生成器当前的效果\n",
    "    # 改变输入数字的尺寸，适应于生成器网络\n",
    "    with torch.no_grad():\n",
    "        samples.resize_(batch_size,1,1,1)\n",
    "    samples = samples.data.expand(batch_size, input_dim, 1, 1)\n",
    "    samples = samples.cuda() if use_cuda else samples #加载到GPU\n",
    "    fake_u=net(samples) #用原始网络作为输入，得到伪造的图像数据\n",
    "    fake_u = fake_u.cpu() if use_cuda else fake_u\n",
    "    img = make_show(fake_u) #将张量转化成可绘制的图像\n",
    "    os.makedirs('temp1',exist_ok=True)\n",
    "    vutil.save_image(img,'temp1/fake%s.png'% (epoch)) #保存生成的图像"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize = (10, 7))\n",
    "plt.plot([i[0] for i in record], label='Training')\n",
    "plt.plot([i[1] for i in record], label='Validation')\n",
    "plt.xlabel('Batchs')\n",
    "plt.ylabel('Loss')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 绘制一批图像样本\n",
    "fake_u = fake_u.cpu() if use_cuda else fake_u\n",
    "samples = samples.cpu() if use_cuda else samples\n",
    "img = fake_u.data #将张量转化成可绘制的图像\n",
    "fig = plt.figure(figsize = (15, 15))\n",
    "\n",
    "for i in range(batch_size):\n",
    "    ax = plt.subplot(8,8,i+1)\n",
    "    ax.axis('off')\n",
    "    imshow(img[i].data, samples.data.numpy()[i][0,0,0].astype(int))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 二、生成器 － 识别器模型\n",
    "\n",
    "在这个模型中，我们不改变生成器，但是改变了网络的目标函数。我们加入了一个识别器，它通过固定值的方式迁移自一个手写体识别器\n",
    "然后让生成器生成图像，并让识别器进行识别，将识别的误差作为目标函数，调整生成器，从而能给出正确的分类标签"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义待迁移的网络框架，所有的神经网络模块包括：Conv2d、MaxPool2d，Linear等模块都不需要重新定义，会自动加载\n",
    "# 但是网络的forward功能没有办法自动实现，需要重写。\n",
    "# 一般的，加载网络只加载网络的属性，不加载方法\n",
    "depth = [4, 8]\n",
    "class ConvNet(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(ConvNet, self).__init__()\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.conv1(x))\n",
    "        x = self.pool(x)\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = self.pool(x)\n",
    "        # 将立体的Tensor全部转换成一维的Tensor。两次pooling操作，所以图像维度减少了1/4\n",
    "        x = x.view(-1, image_size // 4 * image_size // 4 * depth[1])\n",
    "        x = F.relu(self.fc1(x)) #全链接，激活函数\n",
    "        x = F.dropout(x, training=self.training) #以默认为0.5的概率对这一层进行dropout操作\n",
    "        x = self.fc2(x) #全链接，激活函数\n",
    "        x = F.log_softmax(x, dim = 1) #log_softmax可以理解为概率对数值\n",
    "        return x\n",
    "    def retrieve_features(self, x):\n",
    "        #该函数专门用于提取卷积神经网络的特征图的功能，返回feature_map1, feature_map2为前两层卷积层的特征图\n",
    "        feature_map1 = F.relu(self.conv1(x)) #完成第一层卷积\n",
    "        x = self.pool(feature_map1)  # 完成第一层pooling\n",
    "        feature_map2 = F.relu(self.conv2(x)) #第二层卷积，两层特征图都存储到了feature_map1, feature_map2中\n",
    "        return (feature_map1, feature_map2)\n",
    "def rightness(predictions, labels):\n",
    "    \"\"\"计算预测错误率的函数，其中predictions是模型给出的一组预测结果，batch_size行num_classes列的矩阵，labels是数据之中的正确答案\"\"\"\n",
    "    pred = torch.max(predictions.data, 1)[1] # 对于任意一行（一个样本）的输出值的第1个维度，求最大，得到每一行的最大元素的下标\n",
    "    rights = pred.eq(labels.data.view_as(pred)).sum() #将下标与labels中包含的类别进行比较，并累计得到比较正确的数量\n",
    "    return rights, len(labels) #返回正确的数量和这一次一共比较了多少元素\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "netR = torch.load('minst_conv_checkpoint') #读取硬盘上的minst_conv_checkpoint文件\n",
    "netR = netR.cuda() if use_cuda else netR #加载到GPU中\n",
    "for para in netR.parameters():\n",
    "    para.requires_grad = False #将识别器的权重设置为固定值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#开始训练\n",
    "\n",
    "print('Initialized!')\n",
    "\n",
    "netG = ModelG() #新建一个生成器\n",
    "netG = netG.cuda() if use_cuda else netG #加载到GPU上\n",
    "netG.apply(weight_init) #初始化参数\n",
    "\n",
    "criterion = nn.CrossEntropyLoss() #用交叉熵作为损失函数\n",
    "optimizer = optim.SGD(netG.parameters(), lr=0.0001, momentum=0.9) #定义优化器\n",
    "\n",
    "#随机选择batch_size个数字，用他们来生成数字图像\n",
    "samples = np.random.choice(10, batch_size)\n",
    "samples = torch.from_numpy(samples).type(dtype).requires_grad_(False)\n",
    "\n",
    "num_epochs = 100 #总训练周期\n",
    "statistics = [] #数据记载器\n",
    "for epoch in range(num_epochs):\n",
    "    train_loss = []\n",
    "    train_rights = []\n",
    "    \n",
    "    # 加载数据\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        # 注意图像和标签互换了\n",
    "        target, data = data.clone().detach().requires_grad_(True), target.clone().detach() #data为一批标签，target为一批图像\n",
    "        if use_cuda:\n",
    "            target, data = target.cuda(), data.cuda()\n",
    "        # 复制标签变量放到了label中\n",
    "        label = data.clone()\n",
    "        data = data.type(dtype)\n",
    "        # 改变张量形状以适用于生成器网络\n",
    "        data = data.resize(data.size()[0], 1, 1, 1)\n",
    "        data = data.expand(data.size()[0], input_dim, 1, 1)\n",
    "\n",
    "        netG.train() # 给网络模型做标记，标志说模型正在训练集上训练，\n",
    "        netR.train() #这种区分主要是为了打开关闭net的training标志，从而决定是否运行dropout\n",
    "        output1 = netG(data) #神经网络完成一次前馈的计算过程，得到预测输出output\n",
    "        output = netR(output1) #用识别器网络来做分类\n",
    "        loss = criterion(output, label) #将output与标签target比较，计算误差\n",
    "        optimizer.zero_grad() #清空梯度\n",
    "        loss.backward() #反向传播\n",
    "        optimizer.step() #一步随机梯度下降算法\n",
    "        step += 1\n",
    "        if use_cuda:\n",
    "            loss = loss.cpu()\n",
    "        train_loss.append(loss.data.numpy())\n",
    "        right = rightness(output, label) #计算准确率所需数值，返回数值为（正确样例数，总样本数）\n",
    "        train_rights.append(right) #将计算结果装到列表容器train_rights中\n",
    "        \n",
    "        if step % 100 == 0: #每间隔100个batch执行一次打印等操作\n",
    "            \n",
    "            netG.eval() # 给网络模型做标记，标志说模型正在校验集上运行，\n",
    "            netR.eval() #这种区分主要是为了打开关闭net的training标志，从而决定是否运行dropout\n",
    "            val_loss = [] #记录校验数据集准确率的容器\n",
    "            val_rights = []\n",
    "            \n",
    "            '''开始在校验数据集上做循环，计算校验集上面的准确度'''\n",
    "            for (data, target) in validation_loader:\n",
    "                # 注意target是图像，data是标签\n",
    "                target, data = data.clone().detach().requires_grad_(True), target.clone().detach()\n",
    "                if use_cuda:\n",
    "                    target, data = target.cuda(), data.cuda()\n",
    "                label = data.clone()\n",
    "                data = data.type(dtype)\n",
    "                #改变Tensor大小以适应生成网络\n",
    "                data = data.resize(data.size()[0], 1, 1, 1)\n",
    "                data = data.expand(data.size()[0], input_dim, 1, 1)\n",
    "                \n",
    "                output1 = netG(data) #神经网络完成一次前馈的计算过程，得到预测输出output\n",
    "                output = netR(output1) #利用识别器来识别\n",
    "                loss = criterion(output, label) #将output与标签target比较，计算误差\n",
    "                if use_cuda:\n",
    "                    loss = loss.cpu()\n",
    "                val_loss.append(loss.data.numpy())\n",
    "                right = rightness(output, label) #计算准确率所需数值，返回正确的数值为（正确样例数，总样本数）\n",
    "                val_rights.append(right)\n",
    "            # 分别计算在目前已经计算过的测试数据集，以及全部校验集上模型的表现：分类准确率\n",
    "            #train_r为一个二元组，分别记录目前已经经历过的所有训练集中分类正确的数量和该集合中总的样本数，\n",
    "            #train_r[0]/train_r[1]就是训练集的分类准确度，同样，val_r[0]/val_r[1]就是校验集上的分类准确度\n",
    "            train_r = (sum([tup[0] for tup in train_rights]), sum([tup[1] for tup in train_rights]))\n",
    "            #val_r为一个二元组，分别记录校验集中分类正确的数量和该集合中总的样本数\n",
    "            val_r = (sum([tup[0] for tup in val_rights]), sum([tup[1] for tup in val_rights]))\n",
    "            print(('训练周期: {} [{}/{} ({:.0f}%)]\\t训练数据Loss: {:.6f},正确率: {:.2f}%\\t校验数据Loss:' +\n",
    "                  '{:.6f},正确率:{:.2f}%').format(epoch, batch_idx * batch_size, len(train_loader.dataset),\n",
    "                100. * batch_idx / len(train_loader), np.mean(train_loss), \n",
    "                                               100. * train_r[0].cpu().numpy() / train_r[1], \n",
    "                                               np.mean(val_loss), \n",
    "                                               100. * val_r[0].cpu.numpy() / val_r[1]))\n",
    "            #记录中间的数据\n",
    "            statistics.append({'loss':np.mean(train_loss),'train': 100. * train_r[0] / train_r[1],\n",
    "                               'valid':100. * val_r[0] / val_r[1]})\n",
    "            \n",
    "    # 产生一组图像保存到temp1文件夹下（需要事先建立好该文件夹），检测生成器当前的效果\n",
    "    with torch.no_grad():\n",
    "        samples.resize_(batch_size,1,1,1)\n",
    "    samples = samples.data.expand(batch_size, input_dim, 1, 1)\n",
    "    samples = samples.cuda() if use_cuda else samples\n",
    "    fake_u=netG(samples)\n",
    "    fake_u = fake_u.cpu() if use_cuda else fake_u\n",
    "    img = make_show(fake_u)\n",
    "    os.makedirs('temp1',exist_ok=True)\n",
    "    vutil.save_image(img,'temp1/fake%s.png'% (epoch))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 训练曲线\n",
    "result1 = [100 - i['train'] for i in statistics]\n",
    "result2 = [100 - i['valid'] for i in statistics]\n",
    "plt.figure(figsize = (10, 7))\n",
    "plt.plot(result1, label = 'Training')\n",
    "plt.plot(result2, label = 'Validation')\n",
    "plt.xlabel('Step')\n",
    "plt.ylabel('Error Rate')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#绘制一批样本\n",
    "samples = torch.Tensor([0,1,2,3,4,5,6,7,8,9])\n",
    "samples = samples.type(dtype).clone()\n",
    "\n",
    "sample_size = 10\n",
    "with torch.no_grad():\n",
    "    samples.resize_(sample_size,1,1,1)\n",
    "samples = samples.data.expand(sample_size, input_dim, 1, 1)\n",
    "samples = samples.cuda() if use_cuda else samples\n",
    "fake_u = netG(samples)\n",
    "fake_u = fake_u.cpu() if use_cuda else fake_u\n",
    "samples = samples.cpu() if use_cuda else samples\n",
    "img = fake_u #.expand(sample_size, 3, image_size, image_size) #将张量转化成可绘制的图像\n",
    "\n",
    "plt.figure(figsize=(15,6))\n",
    "for i in range(sample_size):\n",
    "    ax = plt.subplot(2,5,i+1)\n",
    "    ax.axis('off')\n",
    "    imshow(img[i].data, samples.data.numpy()[i][0,0,0].astype(int))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = 9\n",
    "ax = plt.gca()\n",
    "ax.axis('off')\n",
    "imshow(fake_u[idx].data, 9)\n",
    "print(samples[idx].item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#提取第一层卷积层的卷积核\n",
    "plt.figure(figsize = (10, 7))\n",
    "for i in range(4):\n",
    "    plt.subplot(1,4,i + 1)\n",
    "    weight = netR.conv1.weight.cpu() if use_cuda else netR.conv1.weight\n",
    "    plt.imshow(weight.data.numpy()[i,0,...])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 绘制特征图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#调用net的retrieve_features方法可以抽取出喂入当前数据后吐出来的所有特征图（第一个卷积和第二个卷积层）\n",
    "\n",
    "#首先定义读入的图片\n",
    "\n",
    "#它是从test_dataset中提取第idx个批次的第0个图，其次unsqueeze的作用是在最前面添加一维，\n",
    "#目的是为了让这个input_x的tensor是四维的，这样才能输入给net。补充的那一维表示batch。\n",
    "input_x = fake_u[idx]\n",
    "input_x = input_x.unsqueeze(0)\n",
    "input_x = input_x.cuda() if use_cuda else input_x\n",
    "output = netR(input_x)\n",
    "_, prediction = torch.max(output, 1)\n",
    "print(prediction)\n",
    "feature_maps = netR.retrieve_features(input_x) #feature_maps是有两个元素的列表，分别表示第一层和第二层卷积的所有特征图\n",
    "feature_maps = (feature_maps[0].cpu(), feature_maps[1].cpu()) if use_cuda else feature_maps\n",
    "plt.figure(figsize = (10, 7))\n",
    "\n",
    "#有四个特征图，循环把它们打印出来\n",
    "for i in range(4):\n",
    "    plt.subplot(1,4,i + 1)\n",
    "    plt.axis('off')\n",
    "    plt.imshow(feature_maps[0][0, i,...].data.numpy())\n",
    "#第二层有8个特征图，循环把它们打印出来\n",
    "plt.figure(figsize = (10, 7))\n",
    "for i in range(8):\n",
    "    plt.subplot(2,4,i + 1)\n",
    "    plt.axis('off')\n",
    "    plt.imshow(feature_maps[1][0, i,...].data.numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch = next(iter(test_loader))\n",
    "indx = torch.nonzero(batch[1] == 9)\n",
    "data = batch[0][indx[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "img = data.expand(1, 1, image_size, image_size)\n",
    "print(img.size())\n",
    "plt.axis('off')\n",
    "imshow(img[0], 9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_x = data.clone().detach().requires_grad_(True)\n",
    "input_x = input_x.cuda() if use_cuda else input_x\n",
    "output = netR(input_x)\n",
    "_, prediction = torch.max(output, 1)\n",
    "print(prediction)\n",
    "feature_maps = netR.retrieve_features(input_x) #feature_maps是有两个元素的列表，分别表示第一层和第二层卷积的所有特征图\n",
    "feature_maps = (feature_maps[0].cpu(), feature_maps[1].cpu()) if use_cuda else feature_maps\n",
    "plt.figure(figsize = (10, 7))\n",
    "\n",
    "#有四个特征图，循环把它们打印出来\n",
    "for i in range(4):\n",
    "    plt.subplot(1,4,i + 1)\n",
    "    plt.axis('off')\n",
    "    plt.imshow(feature_maps[0][0, i,...].data.numpy())\n",
    "\n",
    "plt.figure(figsize = (10, 7))\n",
    "for i in range(8):\n",
    "    plt.subplot(2,4,i + 1)\n",
    "    plt.axis('off')\n",
    "    plt.imshow(feature_maps[1][0, i,...].data.numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 三、DCGAN\n",
    "\n",
    "同样的生成器，根据输入的一个随机噪声向量生成手写数字的图像，然后再同时训练一个辨别器，它的任务就是负责辨别一张输入的图像是来源于生成器\n",
    "造假还是来源于原始的数据文件。我们将两个网络一起训练。值得注意的是，辨别器和生成器的目标函数是反的。\n",
    "\n",
    "具体训练步骤：\n",
    "\n",
    "1、读取一个batch的原始数据，将图像喂给辨别器，辨别器应该输出为真，计算误差：D_x，\n",
    "\n",
    "2、用随机噪声输入生成器，生成器创造一个batch的假图片，将这些图片输入给辨别器，辨别器应该输出为假，计算误差D_x2\n",
    "\n",
    "3、将两个误差和起来，反向传播训练辨别器\n",
    "\n",
    "4、通过生成图像计算误差，对生成器进行反向传播更新梯度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构造判别器\n",
    "class ModelD(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(ModelD,self).__init__()\n",
    "        self.model=nn.Sequential() #序列化模块构造的神经网络\n",
    "        self.model.add_module('conv1',nn.Conv2d(num_channels, num_features, 5, 2, 0, bias=False)) #卷积层\n",
    "        self.model.add_module('relu1',nn.ReLU()) #激活函数使用了ReLu\n",
    "        #self.model.add_module('relu1',nn.LeakyReLU(0.2, inplace = True)) #激活函数使用了leakyReLu，可以防止dead ReLu的问题\n",
    "        \n",
    "        #第二层卷积\n",
    "        self.model.add_module('conv2',nn.Conv2d(num_features, num_features * 2, 5, 2, 0, bias=False))\n",
    "        self.model.add_module('bnorm2',nn.BatchNorm2d(num_features * 2)) \n",
    "        self.model.add_module('linear1', nn.Linear(num_features * 2 * 4 * 4,   #全链接网络层\n",
    "                                                   num_features))\n",
    "        self.model.add_module('linear2', nn.Linear(num_features, 1)) #全链接网络层\n",
    "        self.model.add_module('sigmoid',nn.Sigmoid())\n",
    "    def forward(self,input):\n",
    "        output = input\n",
    "        # 对网络中的所有神经模块进行循环，并挑选出特定的模块linear1，将feature map展平\n",
    "        for name, module in self.model.named_children():\n",
    "            if name == 'linear1':\n",
    "                output = output.view(-1, num_features * 2 * 4 * 4)\n",
    "            output = module(output)\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构建一个生成器模型，并加载到GPU上\n",
    "netG = ModelG().cuda() if use_cuda else ModelG()\n",
    "# 初始化网络的权重\n",
    "netG.apply(weight_init)\n",
    "print(netG)\n",
    "\n",
    "# 构建一个辨别器网络，并加载的GPU上\n",
    "netD=ModelD().cuda() if use_cuda else ModelD()\n",
    "# 初始化权重\n",
    "netD.apply(weight_init)\n",
    "\n",
    "# 要优化两个网络，所以需要有两个优化器\n",
    "# 使用Adam优化器，可以自动调节收敛速度\n",
    "#optimizerD=optim.SGD(netD.parameters(),lr=0.0002)\n",
    "#optimizerG=optim.SGD(netG.parameters(),lr=0.0002)\n",
    "optimizerD=optim.Adam(netD.parameters(),lr=0.0002,betas=(0.5,0.999))\n",
    "optimizerG=optim.Adam(netG.parameters(),lr=0.0002,betas=(0.5,0.999))\n",
    "\n",
    "\n",
    "# 模型的输入输出\n",
    "# 生成一个随机噪声输入给生成器\n",
    "noise=torch.tensor((batch_size, input_dim, 1, 1), dtype = torch.float)\n",
    "#固定噪声是用于评估生成器结果的，它在训练过程中始终不变\n",
    "fixed_noise=torch.FloatTensor(batch_size, input_dim, 1, 1).normal_(0,1).requires_grad_(True)\n",
    "if use_cuda:\n",
    "    noise = noise.cuda()\n",
    "    fixed_noise = fixed_noise.cuda()\n",
    "\n",
    "\n",
    "#BCE损失函数\n",
    "criterion=nn.BCELoss()\n",
    "error_G = None #总误差\n",
    "num_epochs = 100 #训练周期\n",
    "results = []\n",
    "for epoch in range(num_epochs):\n",
    "    \n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        \n",
    "        #训练辨别器网络\n",
    "        #清空梯度\n",
    "        optimizerD.zero_grad()\n",
    "        #1、输入真实图片\n",
    "        data, target = data.clone().detach().requires_grad_(True), target.clone().detach()\n",
    "        # 用于鉴别赝品还是真品的标签\n",
    "        label = torch.ones(data.size()[0])  #正确的标签是1（真实）\n",
    "        label = label.cuda() if use_cuda else label\n",
    "        \n",
    "        if use_cuda:\n",
    "            data, target, label = data.cuda(), target.cuda(), label.cuda()\n",
    "        netD.train()\n",
    "        output=netD(data) #放到辨别网络里辨别\n",
    "\n",
    "        \n",
    "        #计算损失函数\n",
    "        label.data.fill_(1)\n",
    "        error_real=criterion(output, label)\n",
    "        error_real.backward() #辨别器的反向误差传播\n",
    "        D_x=output.data.mean()\n",
    "        \n",
    "        #2、用噪声生成一张假图片\n",
    "        #噪声是一个input_dim维度的向量\n",
    "        with torch.no_grad():\n",
    "            noise.resize_(data.size()[0],input_dim,1,1).normal_(0,1)\n",
    "        #喂给生成器生成图像\n",
    "        fake_pic=netG(noise).detach() #这里的detach是为了让生成器不参与梯度更新\n",
    "        output2=netD(fake_pic) #用辨别器识别假图像\n",
    "        label.data.fill_(0) #正确的标签应该是0（伪造）\n",
    "        error_fake=criterion(output2,label) #计算损失函数\n",
    "        error_fake.backward() #反向传播误差\n",
    "        error_D=error_real + error_fake #计算真实图像和机器生成图像的总误差\n",
    "        optimizerD.step() #开始优化\n",
    "        # 单独训练生成器网络\n",
    "        if error_G is None or np.random.rand() < 0.5:\n",
    "            optimizerG.zero_grad() #清空生成器梯度\n",
    "\n",
    "            '''注意生成器的目标函数是与辨别器的相反的，故而当辨别器无法辨别的时候为正确'''\n",
    "            label.data.fill_(1) #分类标签全部标为1，即真实图像\n",
    "            noise.data.normal_(0,1) #重新随机生成一个噪声向量\n",
    "            netG.train()\n",
    "            fake_pic=netG(noise) #生成器生成一张伪造图像\n",
    "            output=netD(fake_pic) #辨别器进行分辨\n",
    "            error_G=criterion(output,label) #辨别器的损失函数\n",
    "            error_G.backward() #反向传播\n",
    "            optimizerG.step() #优化网络\n",
    "        if use_cuda:\n",
    "            error_D = error_D.cpu()\n",
    "            error_G = error_G.cpu()\n",
    "        # 记录数据\n",
    "        results.append([float(error_D.data.numpy()), float(error_G.data.numpy())])\n",
    "        \n",
    "        # 打印分类器损失等指标\n",
    "        if batch_idx % 100 == 0:\n",
    "            print ('第{}周期，第{}/{}撮, 分类器Loss:{:.2f}, 生成器Loss:{:.2f}'.format(\n",
    "                epoch,batch_idx,len(train_loader),\n",
    "                error_D.data.item(), \n",
    "                error_G.data.item()))\n",
    "    #生成一些随机图片，但因输出到文件\n",
    "    netG.eval()\n",
    "    fake_u=netG(fixed_noise)\n",
    "    fake_u = fake_u.cpu() if use_cuda else fake_u\n",
    "    img = make_show(fake_u)\n",
    "\n",
    "    #挑选一些真实数据中的图像图像保存\n",
    "    data, _ = next(iter(train_loader))\n",
    "    os.makedirs('temp',exist_ok=True)\n",
    "    os.makedirs('net',exist_ok=True)\n",
    "    vutil.save_image(img,'temp/fake%s.png'% (epoch))\n",
    "    # 保存网络状态到硬盘文件\n",
    "    torch.save(netG.state_dict(), '%s/netG_epoch_%d.pth' % ('net', epoch))\n",
    "    torch.save(netD.state_dict(), '%s/netD_epoch_%d.pth' % ('net', epoch))\n",
    "    if epoch == 0:\n",
    "        img = make_show(data.clone().detach().requires_grad_(True))\n",
    "        vutil.save_image(img,'temp/real%s.png' % (epoch))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 预测曲线\n",
    "plt.figure(figsize = (10, 7))\n",
    "plt.plot([i[1] for i in results], '.', label = 'Generator', alpha = 0.5)\n",
    "plt.plot([i[0] for i in results], '.', label = 'Discreminator', alpha = 0.5)\n",
    "plt.xlabel('Step')\n",
    "plt.ylabel('Loss')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 绘制一些样本\n",
    "noise = torch.FloatTensor(batch_size, input_dim, 1, 1)\n",
    "noise.data.normal_(0,1)\n",
    "noise = noise.cuda() if use_cuda else noise\n",
    "sample_size = batch_size\n",
    "netG.eval()\n",
    "fake_u = netG(noise)\n",
    "fake_u = fake_u.cpu() if use_cuda else fake_u\n",
    "noise = noise.cpu() if use_cuda else samples\n",
    "img = fake_u #.expand(sample_size, 3, image_size, image_size) #将张量转化成可绘制的图像\n",
    "#print(img.size())\n",
    "plt.figure(figsize=(15,15))\n",
    "for i in range(batch_size):\n",
    "    plt.subplot(8,8,i+1)\n",
    "    plt.axis('off')\n",
    "    imshow(img[i].data, None)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}