t81_558_class_13_02_checkpoint.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"https://colab.research.google.com/github/jeffheaton/t81_558_deep_learning/blob/master/t81_558_class_13_02_checkpoint.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# T81-558: Applications of Deep Neural Networks\n",
    "**Module 13: Advanced/Other Topics**\n",
    "* Instructor: [Jeff Heaton](https://sites.wustl.edu/jeffheaton/), McKelvey School of Engineering, [Washington University in St. Louis](https://engineering.wustl.edu/Programs/Pages/default.aspx)\n",
    "* For more information visit the [class website](https://sites.wustl.edu/jeffheaton/t81-558/)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Module 13 Video Material\n",
    "\n",
    "* Part 13.1: Flask and Deep Learning Web Services [[Video]](https://www.youtube.com/watch?v=H73m9XvKHug&list=PLjy4p-07OYzulelvJ5KVaT2pDlxivl_BN) [[Notebook]](t81_558_class_13_01_flask.ipynb)\n",
    "* **Part 13.2: Interrupting and Continuing Training**  [[Video]](https://www.youtube.com/watch?v=8ygCyvRZ074&list=PLjy4p-07OYzulelvJ5KVaT2pDlxivl_BN) [[Notebook]](t81_558_class_13_02_cloud.ipynb)\n",
    "* Part 13.3: Using a Keras Deep Neural Network with a Web Application  [[Video]](https://www.youtube.com/watch?v=OBbw0e-UroI&list=PLjy4p-07OYzulelvJ5KVaT2pDlxivl_BN) [[Notebook]](t81_558_class_13_03_web.ipynb)\n",
    "* Part 13.4: When to Retrain Your Neural Network [[Video]](https://www.youtube.com/watch?v=K2Tjdx_1v9g&list=PLjy4p-07OYzulelvJ5KVaT2pDlxivl_BN) [[Notebook]](t81_558_class_13_04_retrain.ipynb)\n",
    "* Part 13.5: AI at the Edge: Using Keras on a Mobile Device  [[Video]]() [[Notebook]](t81_558_class_13_05_edge.ipynb)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Google CoLab Instructions\n",
    "The following code ensures that Google CoLab is running the correct version of TensorFlow."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    from google.colab import drive\n",
    "    drive.mount('/content/drive', force_remount=True)\n",
    "    COLAB = True\n",
    "    print(\"Note: using Google CoLab\")\n",
    "    %tensorflow_version 2.x\n",
    "except:\n",
    "    print(\"Note: not using Google CoLab\")\n",
    "    COLAB = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part 13.2: Interrupting and Continuing Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import sys\n",
    "import time\n",
    "from typing import Any, List, Tuple, Union\n",
    "import tensorflow as tf\n",
    "import tensorflow.keras\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras import regularizers\n",
    "from tensorflow.keras.datasets import mnist\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Dropout, Flatten\n",
    "from tensorflow.keras.layers import Conv2D, MaxPooling2D\n",
    "from tensorflow.keras import backend as K\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_output_dir(outdir, run_desc):\n",
    "    prev_run_dirs = []\n",
    "    if os.path.isdir(outdir):\n",
    "        prev_run_dirs = [x for x in os.listdir(outdir) if os.path.isdir(os.path.join(outdir, x))]\n",
    "    prev_run_ids = [re.match(r'^\\d+', x) for x in prev_run_dirs]\n",
    "    prev_run_ids = [int(x.group()) for x in prev_run_ids if x is not None]\n",
    "    cur_run_id = max(prev_run_ids, default=-1) + 1\n",
    "    run_dir = os.path.join(outdir, f'{cur_run_id:05d}-{run_desc}')\n",
    "    assert not os.path.exists(run_dir)\n",
    "    os.makedirs(run_dir)\n",
    "    return run_dir\n",
    "\n",
    "# From StyleGAN2\n",
    "class Logger(object):\n",
    "    \"\"\"Redirect stderr to stdout, optionally print stdout to a file, and optionally force flushing on both stdout and the file.\"\"\"\n",
    "\n",
    "    def __init__(self, file_name: str = None, file_mode: str = \"w\", should_flush: bool = True):\n",
    "        self.file = None\n",
    "\n",
    "        if file_name is not None:\n",
    "            self.file = open(file_name, file_mode)\n",
    "\n",
    "        self.should_flush = should_flush\n",
    "        self.stdout = sys.stdout\n",
    "        self.stderr = sys.stderr\n",
    "\n",
    "        sys.stdout = self\n",
    "        sys.stderr = self\n",
    "\n",
    "    def __enter__(self) -> \"Logger\":\n",
    "        return self\n",
    "\n",
    "    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:\n",
    "        self.close()\n",
    "\n",
    "    def write(self, text: str) -> None:\n",
    "        \"\"\"Write text to stdout (and a file) and optionally flush.\"\"\"\n",
    "        if len(text) == 0: # workaround for a bug in VSCode debugger: sys.stdout.write(''); sys.stdout.flush() => crash\n",
    "            return\n",
    "\n",
    "        if self.file is not None:\n",
    "            self.file.write(text)\n",
    "\n",
    "        self.stdout.write(text)\n",
    "\n",
    "        if self.should_flush:\n",
    "            self.flush()\n",
    "\n",
    "    def flush(self) -> None:\n",
    "        \"\"\"Flush written text to both stdout and a file, if open.\"\"\"\n",
    "        if self.file is not None:\n",
    "            self.file.flush()\n",
    "\n",
    "        self.stdout.flush()\n",
    "\n",
    "    def close(self) -> None:\n",
    "        \"\"\"Flush, close possible files, and remove stdout/stderr mirroring.\"\"\"\n",
    "        self.flush()\n",
    "\n",
    "        # if using multiple loggers, prevent closing in wrong order\n",
    "        if sys.stdout is self:\n",
    "            sys.stdout = self.stdout\n",
    "        if sys.stderr is self:\n",
    "            sys.stderr = self.stderr\n",
    "\n",
    "        if self.file is not None:\n",
    "            self.file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results saved to: ./data/00000-test-train\n"
     ]
    }
   ],
   "source": [
    "outdir = \"./data/\"\n",
    "run_desc = \"test-train\"\n",
    "batch_size = 128\n",
    "num_classes = 10\n",
    "epochs = 12\n",
    "\n",
    "run_dir = generate_output_dir(outdir, run_desc)\n",
    "print(f\"Results saved to: {run_dir}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def obtain_data():\n",
    "    (x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "    print(\"Shape of x_train: {}\".format(x_train.shape))\n",
    "    print(\"Shape of y_train: {}\".format(y_train.shape))\n",
    "    print()\n",
    "    print(\"Shape of x_test: {}\".format(x_test.shape))\n",
    "    print(\"Shape of y_test: {}\".format(y_test.shape))\n",
    "\n",
    "    # input image dimensions\n",
    "    img_rows, img_cols = 28, 28\n",
    "    if K.image_data_format() == 'channels_first':\n",
    "        x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)\n",
    "        x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)\n",
    "        input_shape = (1, img_rows, img_cols)\n",
    "    else:\n",
    "        x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)\n",
    "        x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)\n",
    "        input_shape = (img_rows, img_cols, 1)\n",
    "    x_train = x_train.astype('float32')\n",
    "    x_test = x_test.astype('float32')\n",
    "    x_train /= 255\n",
    "    x_test /= 255\n",
    "    print('x_train shape:', x_train.shape)\n",
    "    print(\"Training samples: {}\".format(x_train.shape[0]))\n",
    "    print(\"Test samples: {}\".format(x_test.shape[0]))\n",
    "    # convert class vectors to binary class matrices\n",
    "    y_train = tensorflow.keras.utils.to_categorical(y_train, num_classes)\n",
    "    y_test = tensorflow.keras.utils.to_categorical(y_test, num_classes)\n",
    "    \n",
    "    return input_shape, x_train, y_train, x_test, y_test\n",
    "\n",
    "def build_model(input_shape, num_classes):\n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(32, kernel_size=(3, 3),\n",
    "                     activation='relu',\n",
    "                     input_shape=input_shape))\n",
    "    model.add(Conv2D(64, (3, 3), activation='relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(0.25))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(128, activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(num_classes, activation='softmax'))\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam',\n",
    "                  metrics=['accuracy'])\n",
    "    return model\n",
    "\n",
    "def load_model(path):\n",
    "    model = load_model(path)\n",
    "    epoch = int(path.split('.')[1])\n",
    "    return epoch, model\n",
    "\n",
    "def train_model(model):\n",
    "    start_time = time.time()\n",
    "\n",
    "    ckpt_filename = os.path.join(run_dir, 'weights.{epoch:02d}-{val_loss:.2f}.hdf5')\n",
    "    ckpt_callback = ModelCheckpoint(ckpt_filename, monitor='val_loss',verbose=1)\n",
    "    callbacks = [ckpt_callback]\n",
    "\n",
    "    model.fit(x_train, y_train,\n",
    "              batch_size=batch_size,\n",
    "              epochs=epochs,\n",
    "              verbose=2, callbacks=callbacks,\n",
    "              validation_data=(x_test, y_test))\n",
    "    score = model.evaluate(x_test, y_test, verbose=0, callbacks=callbacks)\n",
    "    print('Test loss: {}'.format(score[0]))\n",
    "    print('Test accuracy: {}'.format(score[1]))\n",
    "\n",
    "    elapsed_time = time.time() - start_time\n",
    "    print(\"Elapsed time: {}\".format(hms_string(elapsed_time)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'Logger' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-5c0b2affc417>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mLogger\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'log.txt'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0minput_shape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mobtain_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbuild_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_shape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_classes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mtrain_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'Logger' is not defined"
     ]
    }
   ],
   "source": [
    "with Logger(os.path.join(run_dir, 'log.txt')):\n",
    "    input_shape, x_train, y_train, x_test, y_test = obtain_data()\n",
    "    model = build_model(input_shape, num_classes)\n",
    "    train_model(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3.8 (tensorflow)",
   "language": "python",
   "name": "tensorflow"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}