pytorch_ipynb/cnn/cnn-resnet18-mnist.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "convnet-vgg16.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.1"
    },
    "toc": {
      "nav_menu": {},
      "number_sections": true,
      "sideBar": true,
      "skip_h1_title": false,
      "title_cell": "Table of Contents",
      "title_sidebar": "Contents",
      "toc_cell": true,
      "toc_position": {
        "height": "calc(100% - 180px)",
        "left": "10px",
        "top": "150px",
        "width": "371px"
      },
      "toc_section_display": true,
      "toc_window_display": true
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "fb116836ee8e43d98c01517ce3c6bf55": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f963909691ed44c19cd1d961024a37e8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3c7a7d130e7743f3b4a53450af99e594",
              "IPY_MODEL_0f0bf3896136445b8c6336db8a20bcd0"
            ]
          }
        },
        "f963909691ed44c19cd1d961024a37e8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3c7a7d130e7743f3b4a53450af99e594": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_97fed581c2f64a6d8ef94804208145a6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_abe84a0ecf454f8d897a3a7a37f8d304"
          }
        },
        "0f0bf3896136445b8c6336db8a20bcd0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f7808f0de2204b4eb891e1bb6ee08603",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 9920512/? [00:20&lt;00:00, 801180.12it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_60ebb39490f34bd1a45291f01630f3dd"
          }
        },
        "97fed581c2f64a6d8ef94804208145a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "abe84a0ecf454f8d897a3a7a37f8d304": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f7808f0de2204b4eb891e1bb6ee08603": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "60ebb39490f34bd1a45291f01630f3dd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a5a911fa9d0344a8984e0b8e68a676ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_745ec1c6e99044c9b08a6174f581a18a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_feddc949df7d4eebbe5fe54a49719c18",
              "IPY_MODEL_20b8e1596cbc4c40838a89874fae73b1"
            ]
          }
        },
        "745ec1c6e99044c9b08a6174f581a18a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "feddc949df7d4eebbe5fe54a49719c18": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_15cdc2881ee54842afdf28d90937e39a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_02fbf38bcfe24b4c8b32dc023b64635b"
          }
        },
        "20b8e1596cbc4c40838a89874fae73b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4a9d624827224f2bb4e44be436015958",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 32768/? [00:01&lt;00:00, 24537.96it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1dce1df751d04bf682a876d974c96058"
          }
        },
        "15cdc2881ee54842afdf28d90937e39a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "02fbf38bcfe24b4c8b32dc023b64635b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4a9d624827224f2bb4e44be436015958": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1dce1df751d04bf682a876d974c96058": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "62223304a6df42b89e331149045e2d7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3b416fc0b0654516988e57eaa17f21cb",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_dc2375cdc89c4a2fba19e59c12a1b21c",
              "IPY_MODEL_ff91fbb5a5c243078d60ea6dc056cfbf"
            ]
          }
        },
        "3b416fc0b0654516988e57eaa17f21cb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dc2375cdc89c4a2fba19e59c12a1b21c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_68466c143d4c4b39a773bbbcab7397a7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_09e83eaa6fe9472e95c18fbeeb477510"
          }
        },
        "ff91fbb5a5c243078d60ea6dc056cfbf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8b96b8c4605e4d18aa9a82d730a96cba",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1654784/? [00:01&lt;00:00, 1575195.81it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5732e24bc65949c393ff41d6c427a1b6"
          }
        },
        "68466c143d4c4b39a773bbbcab7397a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "09e83eaa6fe9472e95c18fbeeb477510": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8b96b8c4605e4d18aa9a82d730a96cba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5732e24bc65949c393ff41d6c427a1b6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "68b3b1f8bf634390b637b7befe0ec617": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_31ab65c12b794b04849b1556d92ddf5c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2704662bdf67422cbdc443435b4ba985",
              "IPY_MODEL_703d98bfbe6e401e968d54bc9c69a5ae"
            ]
          }
        },
        "31ab65c12b794b04849b1556d92ddf5c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2704662bdf67422cbdc443435b4ba985": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_eeb77e10d04746a4b6bda725749aee1b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f8162e16f3054bde9f44347c0f04c8f0"
          }
        },
        "703d98bfbe6e401e968d54bc9c69a5ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2064aaf67ada46878e6c953b66827971",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 8192/? [00:00&lt;00:00, 24363.63it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0de5bacb4e96469bbefb4295abf3cd66"
          }
        },
        "eeb77e10d04746a4b6bda725749aee1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f8162e16f3054bde9f44347c0f04c8f0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2064aaf67ada46878e6c953b66827971": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0de5bacb4e96469bbefb4295abf3cd66": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "UEBilEjLj5wY"
      },
      "source": [
        "Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
        "- Author: Sebastian Raschka\n",
        "- GitHub Repository: https://github.com/rasbt/deeplearning-models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1Aaa78eDlbVo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install -q IPython\n",
        "!pip install -q ipykernel\n",
        "!pip install -q watermark\n",
        "!pip install -q matplotlib\n",
        "!pip install -q sklearn\n",
        "!pip install -q pandas\n",
        "!pip install -q pydot\n",
        "!pip install -q hiddenlayer\n",
        "!pip install -q graphviz"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "GOzuY8Yvj5wb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "outputId": "58f393a0-7082-48cb-9d3d-a37745b901c0"
      },
      "source": [
        "%load_ext watermark\n",
        "%watermark -a 'Sebastian Raschka' -v -p torch"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sebastian Raschka \n",
            "\n",
            "CPython 3.6.9\n",
            "IPython 5.5.0\n",
            "\n",
            "torch 1.5.1+cu101\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "rH4XmErYj5wm"
      },
      "source": [
        "# Model Zoo -- ResNet-18 MNIST Digits Classifier with Data Parallelism"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BBsPbOpUlXyu",
        "colab_type": "text"
      },
      "source": [
        "### Network Architecture"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s9YYYnW8lXyu",
        "colab_type": "text"
      },
      "source": [
        "The network in this notebook is an implementation of the ResNet-18 [1] architecture on the MNIST digits dataset (http://yann.lecun.com/exdb/mnist/) to train a handwritten digit classifier.  \n",
        "\n",
        "\n",
        "References\n",
        "    \n",
        "- [1] He, K., Zhang, X., Ren, S., & Sun, J. (2016). Deep residual learning for image recognition. In Proceedings of the IEEE conference on computer vision and pattern recognition (pp. 770-778). ([CVPR Link](https://www.cv-foundation.org/openaccess/content_cvpr_2016/html/He_Deep_Residual_Learning_CVPR_2016_paper.html))\n",
        "\n",
        "- [2] http://yann.lecun.com/exdb/mnist/"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X2Qut7r-lXyv",
        "colab_type": "text"
      },
      "source": [
        "The following figure illustrates residual blocks with skip connections such that the input passed via the shortcut matches the dimensions of the main path's output, which allows the network to learn identity functions.\n",
        "\n",
        "![](https://github.com/DeepSE/deeplearning-models/blob/master/pytorch_ipynb/images/resnets/resnet-ex-1-1.png?raw=1)\n",
        "\n",
        "\n",
        "The ResNet-18 architecture actually uses residual blocks with skip connections such that the input passed via the shortcut matches is resized to dimensions of the main path's output. Such a residual block is illustrated below:\n",
        "\n",
        "![](https://github.com/DeepSE/deeplearning-models/blob/master/pytorch_ipynb/images/resnets/resnet-ex-1-2.png?raw=1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5bUUK_aOlXyv",
        "colab_type": "text"
      },
      "source": [
        "For a more detailed explanation see the other notebook, [resnet-ex-1.ipynb](resnet-ex-1.ipynb)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "MkoGLH_Tj5wn"
      },
      "source": [
        "## Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ORj09gnrj5wp",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import time\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import DataLoader\n",
        "\n",
        "from torchvision import datasets\n",
        "from torchvision import transforms\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "from PIL import Image\n",
        "\n",
        "\n",
        "if torch.cuda.is_available():\n",
        "    torch.backends.cudnn.deterministic = True"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "I6hghKPxj5w0"
      },
      "source": [
        "## Model Settings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "NnT0sZIwj5wu",
        "colab": {}
      },
      "source": [
        "##########################\n",
        "### SETTINGS\n",
        "##########################\n",
        "\n",
        "# Hyperparameters\n",
        "RANDOM_SEED = 1\n",
        "LEARNING_RATE = 0.001\n",
        "BATCH_SIZE = 128\n",
        "NUM_EPOCHS = 10\n",
        "\n",
        "# Architecture\n",
        "NUM_FEATURES = 28*28\n",
        "NUM_CLASSES = 10\n",
        "\n",
        "# Other\n",
        "DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "GRAYSCALE = True"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QlumT-MjlXy2",
        "colab_type": "text"
      },
      "source": [
        "### MNIST Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_zAvMhZmlXy2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 441,
          "referenced_widgets": [
            "fb116836ee8e43d98c01517ce3c6bf55",
            "f963909691ed44c19cd1d961024a37e8",
            "3c7a7d130e7743f3b4a53450af99e594",
            "0f0bf3896136445b8c6336db8a20bcd0",
            "97fed581c2f64a6d8ef94804208145a6",
            "abe84a0ecf454f8d897a3a7a37f8d304",
            "f7808f0de2204b4eb891e1bb6ee08603",
            "60ebb39490f34bd1a45291f01630f3dd",
            "a5a911fa9d0344a8984e0b8e68a676ef",
            "745ec1c6e99044c9b08a6174f581a18a",
            "feddc949df7d4eebbe5fe54a49719c18",
            "20b8e1596cbc4c40838a89874fae73b1",
            "15cdc2881ee54842afdf28d90937e39a",
            "02fbf38bcfe24b4c8b32dc023b64635b",
            "4a9d624827224f2bb4e44be436015958",
            "1dce1df751d04bf682a876d974c96058",
            "62223304a6df42b89e331149045e2d7d",
            "3b416fc0b0654516988e57eaa17f21cb",
            "dc2375cdc89c4a2fba19e59c12a1b21c",
            "ff91fbb5a5c243078d60ea6dc056cfbf",
            "68466c143d4c4b39a773bbbcab7397a7",
            "09e83eaa6fe9472e95c18fbeeb477510",
            "8b96b8c4605e4d18aa9a82d730a96cba",
            "5732e24bc65949c393ff41d6c427a1b6",
            "68b3b1f8bf634390b637b7befe0ec617",
            "31ab65c12b794b04849b1556d92ddf5c",
            "2704662bdf67422cbdc443435b4ba985",
            "703d98bfbe6e401e968d54bc9c69a5ae",
            "eeb77e10d04746a4b6bda725749aee1b",
            "f8162e16f3054bde9f44347c0f04c8f0",
            "2064aaf67ada46878e6c953b66827971",
            "0de5bacb4e96469bbefb4295abf3cd66"
          ]
        },
        "outputId": "6b882db4-c08d-4bcf-8400-6d89ed0e418a"
      },
      "source": [
        "##########################\n",
        "### MNIST DATASET\n",
        "##########################\n",
        "\n",
        "# Note transforms.ToTensor() scales input images\n",
        "# to 0-1 range\n",
        "train_dataset = datasets.MNIST(root='data', \n",
        "                               train=True, \n",
        "                               transform=transforms.ToTensor(),\n",
        "                               download=True)\n",
        "\n",
        "test_dataset = datasets.MNIST(root='data', \n",
        "                              train=False, \n",
        "                              transform=transforms.ToTensor())\n",
        "\n",
        "\n",
        "train_loader = DataLoader(dataset=train_dataset, \n",
        "                          batch_size=BATCH_SIZE, \n",
        "                          shuffle=True)\n",
        "\n",
        "test_loader = DataLoader(dataset=test_dataset, \n",
        "                         batch_size=BATCH_SIZE, \n",
        "                         shuffle=False)\n",
        "\n",
        "# Checking the dataset\n",
        "for images, labels in train_loader:  \n",
        "    print('Image batch dimensions:', images.shape)\n",
        "    print('Image label dimensions:', labels.shape)\n",
        "    break"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to data/MNIST/raw/train-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fb116836ee8e43d98c01517ce3c6bf55",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting data/MNIST/raw/train-images-idx3-ubyte.gz to data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to data/MNIST/raw/train-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a5a911fa9d0344a8984e0b8e68a676ef",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting data/MNIST/raw/train-labels-idx1-ubyte.gz to data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to data/MNIST/raw/t10k-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "62223304a6df42b89e331149045e2d7d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting data/MNIST/raw/t10k-images-idx3-ubyte.gz to data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to data/MNIST/raw/t10k-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "68b3b1f8bf634390b637b7befe0ec617",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting data/MNIST/raw/t10k-labels-idx1-ubyte.gz to data/MNIST/raw\n",
            "Processing...\n",
            "Done!\n",
            "\n",
            "\n",
            "\n",
            "Image batch dimensions: torch.Size([128, 1, 28, 28])\n",
            "Image label dimensions: torch.Size([128])\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/pytorch/torch/csrc/utils/tensor_numpy.cpp:141: UserWarning: The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "awWDVI9BlXy5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "4aa65033-7ccc-4e0a-c9f2-3ec416aec5ac"
      },
      "source": [
        "device = torch.device(DEVICE)\n",
        "torch.manual_seed(0)\n",
        "\n",
        "for epoch in range(2):\n",
        "\n",
        "    for batch_idx, (x, y) in enumerate(train_loader):\n",
        "        \n",
        "        print('Epoch:', epoch+1, end='')\n",
        "        print(' | Batch index:', batch_idx, end='')\n",
        "        print(' | Batch size:', y.size()[0])\n",
        "        \n",
        "        x = x.to(device)\n",
        "        y = y.to(device)\n",
        "        break"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 1 | Batch index: 0 | Batch size: 128\n",
            "Epoch: 2 | Batch index: 0 | Batch size: 128\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "InUqxoJslXy8",
        "colab_type": "text"
      },
      "source": [
        "The following code cell that implements the ResNet-34 architecture is a derivative of the code provided at https://pytorch.org/docs/0.4.0/_modules/torchvision/models/resnet.html."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6L5jLjHPlXy8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "##########################\n",
        "### MODEL\n",
        "##########################\n",
        "\n",
        "\n",
        "def conv3x3(in_planes, out_planes, stride=1):\n",
        "    \"\"\"3x3 convolution with padding\"\"\"\n",
        "    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,\n",
        "                     padding=1, bias=False)\n",
        "\n",
        "\n",
        "class BasicBlock(nn.Module):\n",
        "    expansion = 1\n",
        "\n",
        "    def __init__(self, inplanes, planes, stride=1, downsample=None):\n",
        "        super(BasicBlock, self).__init__()\n",
        "        self.conv1 = conv3x3(inplanes, planes, stride)\n",
        "        self.bn1 = nn.BatchNorm2d(planes)\n",
        "        self.relu = nn.ReLU(inplace=True)\n",
        "        self.conv2 = conv3x3(planes, planes)\n",
        "        self.bn2 = nn.BatchNorm2d(planes)\n",
        "        self.downsample = downsample\n",
        "        self.stride = stride\n",
        "\n",
        "    def forward(self, x):\n",
        "        residual = x\n",
        "\n",
        "        out = self.conv1(x)\n",
        "        out = self.bn1(out)\n",
        "        out = self.relu(out)\n",
        "\n",
        "        out = self.conv2(out)\n",
        "        out = self.bn2(out)\n",
        "\n",
        "        if self.downsample is not None:\n",
        "            residual = self.downsample(x)\n",
        "\n",
        "        out += residual\n",
        "        out = self.relu(out)\n",
        "\n",
        "        return out\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "class ResNet(nn.Module):\n",
        "\n",
        "    def __init__(self, block, layers, num_classes, grayscale):\n",
        "        self.inplanes = 64\n",
        "        if grayscale:\n",
        "            in_dim = 1\n",
        "        else:\n",
        "            in_dim = 3\n",
        "        super(ResNet, self).__init__()\n",
        "        self.conv1 = nn.Conv2d(in_dim, 64, kernel_size=7, stride=2, padding=3,\n",
        "                               bias=False)\n",
        "        self.bn1 = nn.BatchNorm2d(64)\n",
        "        self.relu = nn.ReLU(inplace=True)\n",
        "        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n",
        "        self.layer1 = self._make_layer(block, 64, layers[0])\n",
        "        self.layer2 = self._make_layer(block, 128, layers[1], stride=2)\n",
        "        self.layer3 = self._make_layer(block, 256, layers[2], stride=2)\n",
        "        self.layer4 = self._make_layer(block, 512, layers[3], stride=2)\n",
        "        self.avgpool = nn.AvgPool2d(7, stride=1)\n",
        "        self.fc = nn.Linear(512 * block.expansion, num_classes)\n",
        "\n",
        "        for m in self.modules():\n",
        "            if isinstance(m, nn.Conv2d):\n",
        "                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n",
        "                m.weight.data.normal_(0, (2. / n)**.5)\n",
        "            elif isinstance(m, nn.BatchNorm2d):\n",
        "                m.weight.data.fill_(1)\n",
        "                m.bias.data.zero_()\n",
        "\n",
        "    def _make_layer(self, block, planes, blocks, stride=1):\n",
        "        downsample = None\n",
        "        if stride != 1 or self.inplanes != planes * block.expansion:\n",
        "            downsample = nn.Sequential(\n",
        "                nn.Conv2d(self.inplanes, planes * block.expansion,\n",
        "                          kernel_size=1, stride=stride, bias=False),\n",
        "                nn.BatchNorm2d(planes * block.expansion),\n",
        "            )\n",
        "\n",
        "        layers = []\n",
        "        layers.append(block(self.inplanes, planes, stride, downsample))\n",
        "        self.inplanes = planes * block.expansion\n",
        "        for i in range(1, blocks):\n",
        "            layers.append(block(self.inplanes, planes))\n",
        "\n",
        "        return nn.Sequential(*layers)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.conv1(x)\n",
        "        x = self.bn1(x)\n",
        "        x = self.relu(x)\n",
        "        x = self.maxpool(x)\n",
        "\n",
        "        x = self.layer1(x)\n",
        "        x = self.layer2(x)\n",
        "        x = self.layer3(x)\n",
        "        x = self.layer4(x)\n",
        "        # because MNIST is already 1x1 here:\n",
        "        # disable avg pooling\n",
        "        #x = self.avgpool(x)\n",
        "        \n",
        "        x = x.view(x.size(0), -1)\n",
        "        logits = self.fc(x)\n",
        "        probas = F.softmax(logits, dim=1)\n",
        "        return logits, probas\n",
        "\n",
        "\n",
        "\n",
        "def resnet18(num_classes):\n",
        "    \"\"\"Constructs a ResNet-18 model.\"\"\"\n",
        "    model = ResNet(block=BasicBlock, \n",
        "                   layers=[2, 2, 2, 2],\n",
        "                   num_classes=NUM_CLASSES,\n",
        "                   grayscale=GRAYSCALE)\n",
        "    return model\n"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "_lza9t_uj5w1",
        "colab": {}
      },
      "source": [
        "torch.manual_seed(RANDOM_SEED)\n",
        "\n",
        "model = resnet18(NUM_CLASSES)\n",
        "model.to(DEVICE)\n",
        "\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)  "
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LQw3SPYsli54",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 332
        },
        "outputId": "d42c2886-a835-42a3-b599-a935038b0a2d"
      },
      "source": [
        "import hiddenlayer as hl\n",
        "hl.build_graph(model, torch.zeros([64, 1, 28, 28]).to(DEVICE))"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<hiddenlayer.graph.Graph at 0x7fbf44cf9b00>"
            ],
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n -->\n<!-- Title: %3 Pages: 1 -->\n<svg width=\"6806pt\" height=\"218pt\"\n viewBox=\"0.00 0.00 6806.00 218.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(72 182)\">\n<title>%3</title>\n<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-72,36 -72,-182 6734,-182 6734,36 -72,36\"/>\n<!-- /outputs/130 -->\n<g id=\"node1\" class=\"node\">\n<title>/outputs/130</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"304,-65 234,-65 234,-29 304,-29 304,-65\"/>\n<text text-anchor=\"start\" x=\"242\" y=\"-44\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">MaxPool3x3</text>\n</g>\n<!-- /outputs/144 -->\n<g id=\"node2\" class=\"node\">\n<title>/outputs/144</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"824,-70 770,-70 770,-34 824,-34 824,-70\"/>\n<text text-anchor=\"start\" x=\"788\" y=\"-49\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Add</text>\n</g>\n<!-- /outputs/130&#45;&gt;/outputs/144 -->\n<g id=\"edge1\" class=\"edge\">\n<title>/outputs/130&#45;&gt;/outputs/144</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M304.2456,-53.8346C310.1521,-54.7354 316.2339,-55.5139 322,-56 512.4356,-72.0541 561.0766,-64.4688 752,-56 754.5368,-55.8875 757.1516,-55.7394 759.7842,-55.5667\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"760.1914,-59.046 769.8984,-54.8006 759.6626,-52.0659 760.1914,-59.046\"/>\n<text text-anchor=\"middle\" x=\"570\" y=\"-67\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x64x7x7</text>\n</g>\n<!-- 5313932832081751649 -->\n<g id=\"node29\" class=\"node\">\n<title>5313932832081751649</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"528,-42 388,-42 388,-6 528,-6 528,-42\"/>\n<text text-anchor=\"start\" x=\"396\" y=\"-21\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; BatchNorm &gt; Relu</text>\n</g>\n<!-- /outputs/130&#45;&gt;5313932832081751649 -->\n<g id=\"edge26\" class=\"edge\">\n<title>/outputs/130&#45;&gt;5313932832081751649</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M304.1065,-42.7278C324.8399,-40.2047 351.9879,-36.9009 377.7005,-33.7719\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"378.3419,-37.2198 387.8458,-32.5373 377.4962,-30.271 378.3419,-37.2198\"/>\n<text text-anchor=\"middle\" x=\"346\" y=\"-44\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x64x7x7</text>\n</g>\n<!-- /outputs/145 -->\n<g id=\"node3\" class=\"node\">\n<title>/outputs/145</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"962,-70 908,-70 908,-34 962,-34 962,-70\"/>\n<text text-anchor=\"start\" x=\"925\" y=\"-49\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Relu</text>\n</g>\n<!-- /outputs/144&#45;&gt;/outputs/145 -->\n<g id=\"edge2\" class=\"edge\">\n<title>/outputs/144&#45;&gt;/outputs/145</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M824.2644,-52C845.2445,-52 874.4706,-52 897.6299,-52\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"897.7923,-55.5001 907.7923,-52 897.7922,-48.5001 897.7923,-55.5001\"/>\n<text text-anchor=\"middle\" x=\"866\" y=\"-55\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x64x7x7</text>\n</g>\n<!-- /outputs/159 -->\n<g id=\"node4\" class=\"node\">\n<title>/outputs/159</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1482,-60 1428,-60 1428,-24 1482,-24 1482,-60\"/>\n<text text-anchor=\"start\" x=\"1446\" y=\"-39\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Add</text>\n</g>\n<!-- /outputs/145&#45;&gt;/outputs/159 -->\n<g id=\"edge3\" class=\"edge\">\n<title>/outputs/145&#45;&gt;/outputs/159</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M962.1064,-53.2285C1034.7922,-56.2201 1240.0493,-62.6794 1410,-49 1412.5515,-48.7946 1415.1773,-48.5293 1417.8179,-48.2223\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1418.4996,-51.6625 1427.9493,-46.8717 1417.5745,-44.7239 1418.4996,-51.6625\"/>\n<text text-anchor=\"middle\" x=\"1228\" y=\"-60\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x64x7x7</text>\n</g>\n<!-- 6668030213502541238 -->\n<g id=\"node30\" class=\"node\">\n<title>6668030213502541238</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1186,-37 1046,-37 1046,-1 1186,-1 1186,-37\"/>\n<text text-anchor=\"start\" x=\"1054\" y=\"-16\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; BatchNorm &gt; Relu</text>\n</g>\n<!-- /outputs/145&#45;&gt;6668030213502541238 -->\n<g id=\"edge27\" class=\"edge\">\n<title>/outputs/145&#45;&gt;6668030213502541238</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M962.0147,-44.2921C967.9139,-42.7504 974.1443,-41.2291 980,-40 997.9525,-36.2317 1017.3702,-32.8347 1035.7592,-29.9253\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1036.5081,-33.3511 1045.8536,-28.3599 1035.4353,-26.4338 1036.5081,-33.3511\"/>\n<text text-anchor=\"middle\" x=\"1004\" y=\"-43\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x64x7x7</text>\n</g>\n<!-- /outputs/160 -->\n<g id=\"node5\" class=\"node\">\n<title>/outputs/160</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1620,-60 1566,-60 1566,-24 1620,-24 1620,-60\"/>\n<text text-anchor=\"start\" x=\"1583\" y=\"-39\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Relu</text>\n</g>\n<!-- /outputs/159&#45;&gt;/outputs/160 -->\n<g id=\"edge4\" class=\"edge\">\n<title>/outputs/159&#45;&gt;/outputs/160</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1482.2644,-42C1503.2445,-42 1532.4706,-42 1555.6299,-42\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1555.7923,-45.5001 1565.7923,-42 1555.7922,-38.5001 1555.7923,-45.5001\"/>\n<text text-anchor=\"middle\" x=\"1524\" y=\"-45\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x64x7x7</text>\n</g>\n<!-- 3921912183911047002 -->\n<g id=\"node31\" class=\"node\">\n<title>3921912183911047002</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1844,-89 1704,-89 1704,-53 1844,-53 1844,-89\"/>\n<text text-anchor=\"start\" x=\"1712\" y=\"-68\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; BatchNorm &gt; Relu</text>\n</g>\n<!-- /outputs/160&#45;&gt;3921912183911047002 -->\n<g id=\"edge28\" class=\"edge\">\n<title>/outputs/160&#45;&gt;3921912183911047002</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1620.0975,-46.3416C1639.6857,-49.48 1667.2377,-53.8944 1693.6266,-58.1225\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1693.2366,-61.6046 1703.6644,-59.7308 1694.3441,-54.6927 1693.2366,-61.6046\"/>\n<text text-anchor=\"middle\" x=\"1662\" y=\"-59\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x64x7x7</text>\n</g>\n<!-- 259402024918033348 -->\n<g id=\"node40\" class=\"node\">\n<title>259402024918033348</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1973,-43 1862,-43 1862,-7 1973,-7 1973,-43\"/>\n<text text-anchor=\"start\" x=\"1870.5\" y=\"-22\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv1x1 &gt; BatchNorm</text>\n</g>\n<!-- /outputs/160&#45;&gt;259402024918033348 -->\n<g id=\"edge40\" class=\"edge\">\n<title>/outputs/160&#45;&gt;259402024918033348</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1620.0382,-36.5438C1625.936,-35.5357 1632.1599,-34.61 1638,-34 1710.8457,-26.3919 1794.8983,-24.6252 1851.6522,-24.4574\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1851.9276,-27.9571 1861.9228,-24.4436 1851.9181,-20.9571 1851.9276,-27.9571\"/>\n<text text-anchor=\"middle\" x=\"1662\" y=\"-37\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x64x7x7</text>\n</g>\n<!-- /outputs/180 -->\n<g id=\"node6\" class=\"node\">\n<title>/outputs/180</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"2208,-59 2154,-59 2154,-23 2208,-23 2208,-59\"/>\n<text text-anchor=\"start\" x=\"2172\" y=\"-38\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Add</text>\n</g>\n<!-- /outputs/181 -->\n<g id=\"node7\" class=\"node\">\n<title>/outputs/181</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"2351,-59 2297,-59 2297,-23 2351,-23 2351,-59\"/>\n<text text-anchor=\"start\" x=\"2314\" y=\"-38\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Relu</text>\n</g>\n<!-- /outputs/180&#45;&gt;/outputs/181 -->\n<g id=\"edge5\" class=\"edge\">\n<title>/outputs/180&#45;&gt;/outputs/181</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M2208.2338,-41C2230.4983,-41 2262.2039,-41 2286.8066,-41\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2286.8676,-44.5001 2296.8676,-41 2286.8676,-37.5001 2286.8676,-44.5001\"/>\n<text text-anchor=\"middle\" x=\"2252.5\" y=\"-44\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x128x4x4</text>\n</g>\n<!-- /outputs/195 -->\n<g id=\"node8\" class=\"node\">\n<title>/outputs/195</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"2886,-64 2832,-64 2832,-28 2886,-28 2886,-64\"/>\n<text text-anchor=\"start\" x=\"2850\" y=\"-43\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Add</text>\n</g>\n<!-- /outputs/181&#45;&gt;/outputs/195 -->\n<g id=\"edge6\" class=\"edge\">\n<title>/outputs/181&#45;&gt;/outputs/195</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M2351.4765,-47.3887C2357.2345,-48.4755 2363.2915,-49.4347 2369,-50 2565.8153,-69.4886 2616.4165,-58.7642 2814,-50 2816.5368,-49.8875 2819.1516,-49.7394 2821.7842,-49.5667\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2822.1914,-53.046 2831.8984,-48.8006 2821.6626,-46.0659 2822.1914,-53.046\"/>\n<text text-anchor=\"middle\" x=\"2624.5\" y=\"-63\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x128x4x4</text>\n</g>\n<!-- 4716718483235440639 -->\n<g id=\"node32\" class=\"node\">\n<title>4716718483235440639</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"2580,-36 2440,-36 2440,0 2580,0 2580,-36\"/>\n<text text-anchor=\"start\" x=\"2448\" y=\"-15\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; BatchNorm &gt; Relu</text>\n</g>\n<!-- /outputs/181&#45;&gt;4716718483235440639 -->\n<g id=\"edge29\" class=\"edge\">\n<title>/outputs/181&#45;&gt;4716718483235440639</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M2351.046,-37.6556C2371.7748,-35.0924 2401.5101,-31.4154 2429.667,-27.9337\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2430.4547,-31.363 2439.9495,-26.6622 2429.5956,-24.4159 2430.4547,-31.363\"/>\n<text text-anchor=\"middle\" x=\"2395.5\" y=\"-38\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x128x4x4</text>\n</g>\n<!-- /outputs/196 -->\n<g id=\"node9\" class=\"node\">\n<title>/outputs/196</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"3029,-64 2975,-64 2975,-28 3029,-28 3029,-64\"/>\n<text text-anchor=\"start\" x=\"2992\" y=\"-43\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Relu</text>\n</g>\n<!-- /outputs/195&#45;&gt;/outputs/196 -->\n<g id=\"edge7\" class=\"edge\">\n<title>/outputs/195&#45;&gt;/outputs/196</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M2886.2338,-46C2908.4983,-46 2940.2039,-46 2964.8066,-46\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2964.8676,-49.5001 2974.8676,-46 2964.8676,-42.5001 2964.8676,-49.5001\"/>\n<text text-anchor=\"middle\" x=\"2930.5\" y=\"-49\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x128x4x4</text>\n</g>\n<!-- 5822059805001512140 -->\n<g id=\"node33\" class=\"node\">\n<title>5822059805001512140</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"3258,-93 3118,-93 3118,-57 3258,-57 3258,-93\"/>\n<text text-anchor=\"start\" x=\"3126\" y=\"-72\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; BatchNorm &gt; Relu</text>\n</g>\n<!-- /outputs/196&#45;&gt;5822059805001512140 -->\n<g id=\"edge30\" class=\"edge\">\n<title>/outputs/196&#45;&gt;5822059805001512140</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M3029.046,-50.2168C3049.8657,-53.4629 3079.7712,-58.1256 3108.0374,-62.5327\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"3107.5297,-65.9958 3117.9495,-64.0782 3108.6081,-59.0793 3107.5297,-65.9958\"/>\n<text text-anchor=\"middle\" x=\"3073.5\" y=\"-63\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x128x4x4</text>\n</g>\n<!-- 3085655361959055150 -->\n<g id=\"node43\" class=\"node\">\n<title>3085655361959055150</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"3387,-47 3276,-47 3276,-11 3387,-11 3387,-47\"/>\n<text text-anchor=\"start\" x=\"3284.5\" y=\"-26\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv1x1 &gt; BatchNorm</text>\n</g>\n<!-- /outputs/196&#45;&gt;3085655361959055150 -->\n<g id=\"edge46\" class=\"edge\">\n<title>/outputs/196&#45;&gt;3085655361959055150</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M3029.0379,-40.5413C3034.9357,-39.5334 3041.1597,-38.6083 3047,-38 3121.6512,-30.2246 3207.8638,-28.5074 3265.6432,-28.3967\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"3265.7357,-31.8968 3275.7345,-28.3932 3265.7332,-24.8968 3265.7357,-31.8968\"/>\n<text text-anchor=\"middle\" x=\"3073.5\" y=\"-41\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x128x4x4</text>\n</g>\n<!-- /outputs/216 -->\n<g id=\"node10\" class=\"node\">\n<title>/outputs/216</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"3622,-63 3568,-63 3568,-27 3622,-27 3622,-63\"/>\n<text text-anchor=\"start\" x=\"3586\" y=\"-42\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Add</text>\n</g>\n<!-- /outputs/217 -->\n<g id=\"node11\" class=\"node\">\n<title>/outputs/217</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"3765,-63 3711,-63 3711,-27 3765,-27 3765,-63\"/>\n<text text-anchor=\"start\" x=\"3728\" y=\"-42\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Relu</text>\n</g>\n<!-- /outputs/216&#45;&gt;/outputs/217 -->\n<g id=\"edge8\" class=\"edge\">\n<title>/outputs/216&#45;&gt;/outputs/217</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M3622.2338,-45C3644.4983,-45 3676.2039,-45 3700.8066,-45\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"3700.8676,-48.5001 3710.8676,-45 3700.8676,-41.5001 3700.8676,-48.5001\"/>\n<text text-anchor=\"middle\" x=\"3666.5\" y=\"-48\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x256x2x2</text>\n</g>\n<!-- /outputs/231 -->\n<g id=\"node12\" class=\"node\">\n<title>/outputs/231</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"4300,-63 4246,-63 4246,-27 4300,-27 4300,-63\"/>\n<text text-anchor=\"start\" x=\"4264\" y=\"-42\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Add</text>\n</g>\n<!-- /outputs/217&#45;&gt;/outputs/231 -->\n<g id=\"edge9\" class=\"edge\">\n<title>/outputs/217&#45;&gt;/outputs/231</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M3765.4765,-51.3887C3771.2345,-52.4755 3777.2915,-53.4347 3783,-54 3806.4409,-56.3211 3812.4445,-54.0601 3836,-54 4010.2239,-53.5556 4054.3287,-65.8728 4228,-52 4230.5516,-51.7962 4233.1775,-51.532 4235.8181,-51.2259\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"4236.4991,-54.6662 4245.9498,-49.877 4235.5753,-47.7274 4236.4991,-54.6662\"/>\n<text text-anchor=\"middle\" x=\"4038.5\" y=\"-61\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x256x2x2</text>\n</g>\n<!-- 6485309018765742331 -->\n<g id=\"node34\" class=\"node\">\n<title>6485309018765742331</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"3994,-40 3854,-40 3854,-4 3994,-4 3994,-40\"/>\n<text text-anchor=\"start\" x=\"3862\" y=\"-19\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; BatchNorm &gt; Relu</text>\n</g>\n<!-- /outputs/217&#45;&gt;6485309018765742331 -->\n<g id=\"edge31\" class=\"edge\">\n<title>/outputs/217&#45;&gt;6485309018765742331</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M3765.046,-41.6556C3785.7748,-39.0924 3815.5101,-35.4154 3843.667,-31.9337\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"3844.4547,-35.363 3853.9495,-30.6622 3843.5956,-28.4159 3844.4547,-35.363\"/>\n<text text-anchor=\"middle\" x=\"3809.5\" y=\"-42\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x256x2x2</text>\n</g>\n<!-- /outputs/232 -->\n<g id=\"node13\" class=\"node\">\n<title>/outputs/232</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"4443,-63 4389,-63 4389,-27 4443,-27 4443,-63\"/>\n<text text-anchor=\"start\" x=\"4406\" y=\"-42\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Relu</text>\n</g>\n<!-- /outputs/231&#45;&gt;/outputs/232 -->\n<g id=\"edge10\" class=\"edge\">\n<title>/outputs/231&#45;&gt;/outputs/232</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M4300.2338,-45C4322.4983,-45 4354.2039,-45 4378.8066,-45\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"4378.8676,-48.5001 4388.8676,-45 4378.8676,-41.5001 4378.8676,-48.5001\"/>\n<text text-anchor=\"middle\" x=\"4344.5\" y=\"-48\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x256x2x2</text>\n</g>\n<!-- 2051786642957083811 -->\n<g id=\"node35\" class=\"node\">\n<title>2051786642957083811</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"4672,-92 4532,-92 4532,-56 4672,-56 4672,-92\"/>\n<text text-anchor=\"start\" x=\"4540\" y=\"-71\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; BatchNorm &gt; Relu</text>\n</g>\n<!-- /outputs/232&#45;&gt;2051786642957083811 -->\n<g id=\"edge32\" class=\"edge\">\n<title>/outputs/232&#45;&gt;2051786642957083811</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M4443.046,-49.2168C4463.8657,-52.4629 4493.7712,-57.1256 4522.0374,-61.5327\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"4521.5297,-64.9958 4531.9495,-63.0782 4522.6081,-58.0793 4521.5297,-64.9958\"/>\n<text text-anchor=\"middle\" x=\"4487.5\" y=\"-62\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x256x2x2</text>\n</g>\n<!-- 10149767426592778049 -->\n<g id=\"node46\" class=\"node\">\n<title>10149767426592778049</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"4801,-46 4690,-46 4690,-10 4801,-10 4801,-46\"/>\n<text text-anchor=\"start\" x=\"4698.5\" y=\"-25\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv1x1 &gt; BatchNorm</text>\n</g>\n<!-- /outputs/232&#45;&gt;10149767426592778049 -->\n<g id=\"edge52\" class=\"edge\">\n<title>/outputs/232&#45;&gt;10149767426592778049</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M4443.0379,-39.5413C4448.9357,-38.5334 4455.1597,-37.6083 4461,-37 4535.6512,-29.2246 4621.8638,-27.5074 4679.6432,-27.3967\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"4679.7357,-30.8968 4689.7345,-27.3932 4679.7332,-23.8968 4679.7357,-30.8968\"/>\n<text text-anchor=\"middle\" x=\"4487.5\" y=\"-40\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x256x2x2</text>\n</g>\n<!-- /outputs/252 -->\n<g id=\"node14\" class=\"node\">\n<title>/outputs/252</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"5036,-62 4982,-62 4982,-26 5036,-26 5036,-62\"/>\n<text text-anchor=\"start\" x=\"5000\" y=\"-41\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Add</text>\n</g>\n<!-- /outputs/253 -->\n<g id=\"node15\" class=\"node\">\n<title>/outputs/253</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"5179,-62 5125,-62 5125,-26 5179,-26 5179,-62\"/>\n<text text-anchor=\"start\" x=\"5142\" y=\"-41\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Relu</text>\n</g>\n<!-- /outputs/252&#45;&gt;/outputs/253 -->\n<g id=\"edge11\" class=\"edge\">\n<title>/outputs/252&#45;&gt;/outputs/253</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M5036.2338,-44C5058.4983,-44 5090.2039,-44 5114.8066,-44\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"5114.8676,-47.5001 5124.8676,-44 5114.8676,-40.5001 5114.8676,-47.5001\"/>\n<text text-anchor=\"middle\" x=\"5080.5\" y=\"-47\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x512x1x1</text>\n</g>\n<!-- /outputs/267 -->\n<g id=\"node16\" class=\"node\">\n<title>/outputs/267</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"5714,-62 5660,-62 5660,-26 5714,-26 5714,-62\"/>\n<text text-anchor=\"start\" x=\"5678\" y=\"-41\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Add</text>\n</g>\n<!-- /outputs/253&#45;&gt;/outputs/267 -->\n<g id=\"edge12\" class=\"edge\">\n<title>/outputs/253&#45;&gt;/outputs/267</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M5179.4765,-50.3887C5185.2345,-51.4755 5191.2915,-52.4347 5197,-53 5220.4409,-55.3211 5226.4445,-53.0601 5250,-53 5424.2239,-52.5556 5468.3287,-64.8728 5642,-51 5644.5516,-50.7962 5647.1775,-50.532 5649.8181,-50.2259\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"5650.4991,-53.6662 5659.9498,-48.877 5649.5753,-46.7274 5650.4991,-53.6662\"/>\n<text text-anchor=\"middle\" x=\"5452.5\" y=\"-60\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x512x1x1</text>\n</g>\n<!-- 15346338019729001917 -->\n<g id=\"node36\" class=\"node\">\n<title>15346338019729001917</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"5408,-39 5268,-39 5268,-3 5408,-3 5408,-39\"/>\n<text text-anchor=\"start\" x=\"5276\" y=\"-18\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; BatchNorm &gt; Relu</text>\n</g>\n<!-- /outputs/253&#45;&gt;15346338019729001917 -->\n<g id=\"edge33\" class=\"edge\">\n<title>/outputs/253&#45;&gt;15346338019729001917</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M5179.046,-40.6556C5199.7748,-38.0924 5229.5101,-34.4154 5257.667,-30.9337\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"5258.4547,-34.363 5267.9495,-29.6622 5257.5956,-27.4159 5258.4547,-34.363\"/>\n<text text-anchor=\"middle\" x=\"5223.5\" y=\"-41\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x512x1x1</text>\n</g>\n<!-- /outputs/268 -->\n<g id=\"node17\" class=\"node\">\n<title>/outputs/268</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"5857,-62 5803,-62 5803,-26 5857,-26 5857,-62\"/>\n<text text-anchor=\"start\" x=\"5820\" y=\"-41\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Relu</text>\n</g>\n<!-- /outputs/267&#45;&gt;/outputs/268 -->\n<g id=\"edge13\" class=\"edge\">\n<title>/outputs/267&#45;&gt;/outputs/268</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M5714.2338,-44C5736.4983,-44 5768.2039,-44 5792.8066,-44\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"5792.8676,-47.5001 5802.8676,-44 5792.8676,-40.5001 5792.8676,-47.5001\"/>\n<text text-anchor=\"middle\" x=\"5758.5\" y=\"-47\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x512x1x1</text>\n</g>\n<!-- /outputs/269 -->\n<g id=\"node18\" class=\"node\">\n<title>/outputs/269</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"6000,-92 5946,-92 5946,-56 6000,-56 6000,-92\"/>\n<text text-anchor=\"start\" x=\"5961\" y=\"-71\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Shape</text>\n</g>\n<!-- /outputs/268&#45;&gt;/outputs/269 -->\n<g id=\"edge14\" class=\"edge\">\n<title>/outputs/268&#45;&gt;/outputs/269</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M5857.2338,-49.7134C5879.4983,-54.3843 5911.2039,-61.0358 5935.8066,-66.1972\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"5935.362,-69.68 5945.8676,-68.3079 5936.7993,-62.8292 5935.362,-69.68\"/>\n<text text-anchor=\"middle\" x=\"5901.5\" y=\"-67\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x512x1x1</text>\n</g>\n<!-- /outputs/276 -->\n<g id=\"node25\" class=\"node\">\n<title>/outputs/276</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"6423,-58 6369,-58 6369,-22 6423,-22 6423,-58\"/>\n<text text-anchor=\"start\" x=\"6379\" y=\"-37\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Reshape</text>\n</g>\n<!-- /outputs/268&#45;&gt;/outputs/276 -->\n<g id=\"edge15\" class=\"edge\">\n<title>/outputs/268&#45;&gt;/outputs/276</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M5857.0473,-39.1561C5885.7283,-34.4468 5932.4054,-28 5973,-28 5973,-28 5973,-28 6305,-28 6322.6472,-28 6342.0554,-30.2456 6358.3988,-32.7939\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"6358.2092,-36.3101 6368.6468,-34.4917 6359.3534,-29.4043 6358.2092,-36.3101\"/>\n<text text-anchor=\"middle\" x=\"6135.5\" y=\"-31\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x512x1x1</text>\n</g>\n<!-- /outputs/271 -->\n<g id=\"node20\" class=\"node\">\n<title>/outputs/271</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"6091,-92 6037,-92 6037,-56 6091,-56 6091,-92\"/>\n<text text-anchor=\"start\" x=\"6050\" y=\"-71\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Gather</text>\n</g>\n<!-- /outputs/269&#45;&gt;/outputs/271 -->\n<g id=\"edge16\" class=\"edge\">\n<title>/outputs/269&#45;&gt;/outputs/271</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M6000.303,-74C6008.5813,-74 6017.8217,-74 6026.6405,-74\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"6026.7163,-77.5001 6036.7163,-74 6026.7162,-70.5001 6026.7163,-77.5001\"/>\n</g>\n<!-- /outputs/270 -->\n<g id=\"node19\" class=\"node\">\n<title>/outputs/270</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"6000,-146 5946,-146 5946,-110 6000,-110 6000,-146\"/>\n<text text-anchor=\"start\" x=\"5955\" y=\"-125\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Constant</text>\n</g>\n<!-- /outputs/270&#45;&gt;/outputs/271 -->\n<g id=\"edge17\" class=\"edge\">\n<title>/outputs/270&#45;&gt;/outputs/271</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M6000.303,-111.7982C6009.0312,-106.6189 6018.829,-100.8048 6028.0744,-95.3185\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"6029.9025,-98.3035 6036.7163,-90.1903 6026.3303,-92.2836 6029.9025,-98.3035\"/>\n</g>\n<!-- /outputs/273 -->\n<g id=\"node22\" class=\"node\">\n<title>/outputs/273</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"6241,-92 6180,-92 6180,-56 6241,-56 6241,-92\"/>\n<text text-anchor=\"start\" x=\"6188.5\" y=\"-71\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Unsqueeze</text>\n</g>\n<!-- /outputs/271&#45;&gt;/outputs/273 -->\n<g id=\"edge18\" class=\"edge\">\n<title>/outputs/271&#45;&gt;/outputs/273</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M6091.2122,-74C6113.223,-74 6144.5605,-74 6169.5927,-74\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"6169.8892,-77.5001 6179.8891,-74 6169.8891,-70.5001 6169.8892,-77.5001\"/>\n</g>\n<!-- /outputs/272 -->\n<g id=\"node21\" class=\"node\">\n<title>/outputs/272</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"6091,-146 6037,-146 6037,-110 6091,-110 6091,-146\"/>\n<text text-anchor=\"start\" x=\"6046\" y=\"-125\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Constant</text>\n</g>\n<!-- /outputs/274 -->\n<g id=\"node23\" class=\"node\">\n<title>/outputs/274</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"6241,-146 6180,-146 6180,-110 6241,-110 6241,-146\"/>\n<text text-anchor=\"start\" x=\"6188.5\" y=\"-125\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Unsqueeze</text>\n</g>\n<!-- /outputs/272&#45;&gt;/outputs/274 -->\n<g id=\"edge19\" class=\"edge\">\n<title>/outputs/272&#45;&gt;/outputs/274</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M6091.2122,-128C6113.223,-128 6144.5605,-128 6169.5927,-128\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"6169.8892,-131.5001 6179.8891,-128 6169.8891,-124.5001 6169.8892,-131.5001\"/>\n</g>\n<!-- /outputs/275 -->\n<g id=\"node24\" class=\"node\">\n<title>/outputs/275</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"6332,-92 6278,-92 6278,-56 6332,-56 6332,-92\"/>\n<text text-anchor=\"start\" x=\"6290\" y=\"-71\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Concat</text>\n</g>\n<!-- /outputs/273&#45;&gt;/outputs/275 -->\n<g id=\"edge20\" class=\"edge\">\n<title>/outputs/273&#45;&gt;/outputs/275</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M6241.1814,-74C6249.7033,-74 6259.0485,-74 6267.8957,-74\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"6267.9735,-77.5001 6277.9735,-74 6267.9734,-70.5001 6267.9735,-77.5001\"/>\n</g>\n<!-- /outputs/274&#45;&gt;/outputs/275 -->\n<g id=\"edge21\" class=\"edge\">\n<title>/outputs/274&#45;&gt;/outputs/275</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M6241.1814,-110.4677C6250.0738,-105.3864 6259.8627,-99.7927 6269.0467,-94.5447\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"6271.0275,-97.444 6277.9735,-89.4437 6267.5545,-91.3663 6271.0275,-97.444\"/>\n</g>\n<!-- /outputs/275&#45;&gt;/outputs/276 -->\n<g id=\"edge22\" class=\"edge\">\n<title>/outputs/275&#45;&gt;/outputs/276</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M6332.303,-63.7989C6340.7612,-60.6387 6350.224,-57.1031 6359.215,-53.7438\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"6360.5738,-56.9726 6368.7163,-50.1939 6358.1237,-50.4153 6360.5738,-56.9726\"/>\n</g>\n<!-- /outputs/277 -->\n<g id=\"node26\" class=\"node\">\n<title>/outputs/277</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"6545,-58 6491,-58 6491,-22 6545,-22 6545,-58\"/>\n<text text-anchor=\"start\" x=\"6505\" y=\"-37\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Linear</text>\n</g>\n<!-- /outputs/276&#45;&gt;/outputs/277 -->\n<g id=\"edge23\" class=\"edge\">\n<title>/outputs/276&#45;&gt;/outputs/277</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M6423.0758,-40C6440.0553,-40 6462.1767,-40 6480.7924,-40\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"6481,-43.5001 6490.9999,-40 6480.9999,-36.5001 6481,-43.5001\"/>\n<text text-anchor=\"middle\" x=\"6457\" y=\"-43\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x512</text>\n</g>\n<!-- /outputs/278 -->\n<g id=\"node27\" class=\"node\">\n<title>/outputs/278</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"6662,-58 6608,-58 6608,-22 6662,-22 6662,-58\"/>\n<text text-anchor=\"start\" x=\"6618\" y=\"-37\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Softmax</text>\n</g>\n<!-- /outputs/277&#45;&gt;/outputs/278 -->\n<g id=\"edge24\" class=\"edge\">\n<title>/outputs/277&#45;&gt;/outputs/278</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M6545.1362,-40C6560.6773,-40 6580.4112,-40 6597.4253,-40\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"6597.6057,-43.5001 6607.6056,-40 6597.6056,-36.5001 6597.6057,-43.5001\"/>\n<text text-anchor=\"middle\" x=\"6576.5\" y=\"-43\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x10</text>\n</g>\n<!-- 8515703964465469436 -->\n<g id=\"node28\" class=\"node\">\n<title>8515703964465469436</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"140,-65 0,-65 0,-29 140,-29 140,-65\"/>\n<text text-anchor=\"start\" x=\"8\" y=\"-44\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv7x7 &gt; BatchNorm &gt; Relu</text>\n</g>\n<!-- 8515703964465469436&#45;&gt;/outputs/130 -->\n<g id=\"edge25\" class=\"edge\">\n<title>8515703964465469436&#45;&gt;/outputs/130</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M140.16,-47C167.8694,-47 198.9129,-47 223.584,-47\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"223.7389,-50.5001 233.7388,-47 223.7388,-43.5001 223.7389,-50.5001\"/>\n<text text-anchor=\"middle\" x=\"187\" y=\"-50\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x64x14x14</text>\n</g>\n<!-- 16810485990822705389 -->\n<g id=\"node37\" class=\"node\">\n<title>16810485990822705389</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"723,-42 612,-42 612,-6 723,-6 723,-42\"/>\n<text text-anchor=\"start\" x=\"620.5\" y=\"-21\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; BatchNorm</text>\n</g>\n<!-- 5313932832081751649&#45;&gt;16810485990822705389 -->\n<g id=\"edge34\" class=\"edge\">\n<title>5313932832081751649&#45;&gt;16810485990822705389</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M528.0517,-24C551.7802,-24 578.1867,-24 601.5825,-24\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"601.7528,-27.5001 611.7528,-24 601.7528,-20.5001 601.7528,-27.5001\"/>\n<text text-anchor=\"middle\" x=\"570\" y=\"-27\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x64x7x7</text>\n</g>\n<!-- 10545596119689989224 -->\n<g id=\"node38\" class=\"node\">\n<title>10545596119689989224</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1381,-37 1270,-37 1270,-1 1381,-1 1381,-37\"/>\n<text text-anchor=\"start\" x=\"1278.5\" y=\"-16\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; BatchNorm</text>\n</g>\n<!-- 6668030213502541238&#45;&gt;10545596119689989224 -->\n<g id=\"edge36\" class=\"edge\">\n<title>6668030213502541238&#45;&gt;10545596119689989224</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1186.0517,-19C1209.7802,-19 1236.1867,-19 1259.5825,-19\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1259.7528,-22.5001 1269.7528,-19 1259.7528,-15.5001 1259.7528,-22.5001\"/>\n<text text-anchor=\"middle\" x=\"1228\" y=\"-22\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x64x7x7</text>\n</g>\n<!-- 2705904738287483694 -->\n<g id=\"node39\" class=\"node\">\n<title>2705904738287483694</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"2102,-90 1991,-90 1991,-54 2102,-54 2102,-90\"/>\n<text text-anchor=\"start\" x=\"1999.5\" y=\"-69\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; BatchNorm</text>\n</g>\n<!-- 3921912183911047002&#45;&gt;2705904738287483694 -->\n<g id=\"edge38\" class=\"edge\">\n<title>3921912183911047002&#45;&gt;2705904738287483694</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1844.1755,-71.2575C1886.1889,-71.4117 1939.2618,-71.6065 1980.5086,-71.7578\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1980.7135,-75.2585 1990.7263,-71.7953 1980.7392,-68.2586 1980.7135,-75.2585\"/>\n<text text-anchor=\"middle\" x=\"1917.5\" y=\"-74\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x128x4x4</text>\n</g>\n<!-- 11611387990805817384 -->\n<g id=\"node41\" class=\"node\">\n<title>11611387990805817384</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"2780,-36 2669,-36 2669,0 2780,0 2780,-36\"/>\n<text text-anchor=\"start\" x=\"2677.5\" y=\"-15\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; BatchNorm</text>\n</g>\n<!-- 4716718483235440639&#45;&gt;11611387990805817384 -->\n<g id=\"edge42\" class=\"edge\">\n<title>4716718483235440639&#45;&gt;11611387990805817384</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M2580.2353,-18C2605.4712,-18 2633.8234,-18 2658.671,-18\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2658.7245,-21.5001 2668.7244,-18 2658.7244,-14.5001 2658.7245,-21.5001\"/>\n<text text-anchor=\"middle\" x=\"2624.5\" y=\"-21\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x128x4x4</text>\n</g>\n<!-- 6104276403587783776 -->\n<g id=\"node42\" class=\"node\">\n<title>6104276403587783776</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"3516,-94 3405,-94 3405,-58 3516,-58 3516,-94\"/>\n<text text-anchor=\"start\" x=\"3413.5\" y=\"-73\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; BatchNorm</text>\n</g>\n<!-- 5822059805001512140&#45;&gt;6104276403587783776 -->\n<g id=\"edge44\" class=\"edge\">\n<title>5822059805001512140&#45;&gt;6104276403587783776</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M3258.1755,-75.2575C3300.1889,-75.4117 3353.2618,-75.6065 3394.5086,-75.7578\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"3394.7135,-79.2585 3404.7263,-75.7953 3394.7392,-72.2586 3394.7135,-79.2585\"/>\n<text text-anchor=\"middle\" x=\"3331.5\" y=\"-78\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x256x2x2</text>\n</g>\n<!-- 6502780788632146745 -->\n<g id=\"node44\" class=\"node\">\n<title>6502780788632146745</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"4194,-40 4083,-40 4083,-4 4194,-4 4194,-40\"/>\n<text text-anchor=\"start\" x=\"4091.5\" y=\"-19\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; BatchNorm</text>\n</g>\n<!-- 6485309018765742331&#45;&gt;6502780788632146745 -->\n<g id=\"edge48\" class=\"edge\">\n<title>6485309018765742331&#45;&gt;6502780788632146745</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M3994.2353,-22C4019.4712,-22 4047.8234,-22 4072.671,-22\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"4072.7245,-25.5001 4082.7244,-22 4072.7244,-18.5001 4072.7245,-25.5001\"/>\n<text text-anchor=\"middle\" x=\"4038.5\" y=\"-25\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x256x2x2</text>\n</g>\n<!-- 3791255491530683333 -->\n<g id=\"node45\" class=\"node\">\n<title>3791255491530683333</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"4930,-93 4819,-93 4819,-57 4930,-57 4930,-93\"/>\n<text text-anchor=\"start\" x=\"4827.5\" y=\"-72\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; BatchNorm</text>\n</g>\n<!-- 2051786642957083811&#45;&gt;3791255491530683333 -->\n<g id=\"edge50\" class=\"edge\">\n<title>2051786642957083811&#45;&gt;3791255491530683333</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M4672.1755,-74.2575C4714.1889,-74.4117 4767.2618,-74.6065 4808.5086,-74.7578\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"4808.7135,-78.2585 4818.7263,-74.7953 4808.7392,-71.2586 4808.7135,-78.2585\"/>\n<text text-anchor=\"middle\" x=\"4745.5\" y=\"-77\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x512x1x1</text>\n</g>\n<!-- 8551728508530118685 -->\n<g id=\"node47\" class=\"node\">\n<title>8551728508530118685</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"5608,-39 5497,-39 5497,-3 5608,-3 5608,-39\"/>\n<text text-anchor=\"start\" x=\"5505.5\" y=\"-18\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; BatchNorm</text>\n</g>\n<!-- 15346338019729001917&#45;&gt;8551728508530118685 -->\n<g id=\"edge54\" class=\"edge\">\n<title>15346338019729001917&#45;&gt;8551728508530118685</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M5408.2353,-21C5433.4712,-21 5461.8234,-21 5486.671,-21\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"5486.7245,-24.5001 5496.7244,-21 5486.7244,-17.5001 5486.7245,-24.5001\"/>\n<text text-anchor=\"middle\" x=\"5452.5\" y=\"-24\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x512x1x1</text>\n</g>\n<!-- 16810485990822705389&#45;&gt;/outputs/144 -->\n<g id=\"edge35\" class=\"edge\">\n<title>16810485990822705389&#45;&gt;/outputs/144</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M723.1712,-36.037C735.5578,-38.7152 748.3966,-41.4912 759.8258,-43.9623\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"759.3584,-47.4421 769.8723,-46.1345 760.8378,-40.6002 759.3584,-47.4421\"/>\n<text text-anchor=\"middle\" x=\"746.5\" y=\"-46\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64</text>\n</g>\n<!-- 10545596119689989224&#45;&gt;/outputs/159 -->\n<g id=\"edge37\" class=\"edge\">\n<title>10545596119689989224&#45;&gt;/outputs/159</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1381.1712,-28.8876C1393.5578,-31.0875 1406.3966,-33.3677 1417.8258,-35.3976\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1417.4143,-38.8792 1427.8723,-37.1819 1418.6384,-31.9871 1417.4143,-38.8792\"/>\n<text text-anchor=\"middle\" x=\"1404.5\" y=\"-37\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64</text>\n</g>\n<!-- 2705904738287483694&#45;&gt;/outputs/180 -->\n<g id=\"edge39\" class=\"edge\">\n<title>2705904738287483694&#45;&gt;/outputs/180</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M2102.3716,-59.1225C2116.3815,-55.8935 2131.07,-52.508 2143.9088,-49.5489\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2144.802,-52.9349 2153.7604,-47.2783 2143.2298,-46.1137 2144.802,-52.9349\"/>\n<text text-anchor=\"middle\" x=\"2128\" y=\"-57\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">128</text>\n</g>\n<!-- 259402024918033348&#45;&gt;/outputs/180 -->\n<g id=\"edge41\" class=\"edge\">\n<title>259402024918033348&#45;&gt;/outputs/180</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1973.2424,-27.1435C2009.8623,-28.6935 2058.8081,-31.0341 2102,-34 2115.6573,-34.9378 2130.6219,-36.203 2143.8186,-37.3998\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2143.6846,-40.9023 2153.9643,-38.3374 2144.3288,-33.932 2143.6846,-40.9023\"/>\n<text text-anchor=\"middle\" x=\"2046.5\" y=\"-37\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">128</text>\n</g>\n<!-- 11611387990805817384&#45;&gt;/outputs/195 -->\n<g id=\"edge43\" class=\"edge\">\n<title>11611387990805817384&#45;&gt;/outputs/195</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M2780.3716,-29.6313C2794.3815,-32.5478 2809.07,-35.6056 2821.9088,-38.2784\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2821.2569,-41.7177 2831.7604,-40.3293 2822.6837,-34.8646 2821.2569,-41.7177\"/>\n<text text-anchor=\"middle\" x=\"2806\" y=\"-40\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">128</text>\n</g>\n<!-- 6104276403587783776&#45;&gt;/outputs/216 -->\n<g id=\"edge45\" class=\"edge\">\n<title>6104276403587783776&#45;&gt;/outputs/216</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M3516.3716,-63.1225C3530.3815,-59.8935 3545.07,-56.508 3557.9088,-53.5489\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"3558.802,-56.9349 3567.7604,-51.2783 3557.2298,-50.1137 3558.802,-56.9349\"/>\n<text text-anchor=\"middle\" x=\"3542\" y=\"-61\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">256</text>\n</g>\n<!-- 3085655361959055150&#45;&gt;/outputs/216 -->\n<g id=\"edge47\" class=\"edge\">\n<title>3085655361959055150&#45;&gt;/outputs/216</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M3387.2424,-31.1435C3423.8623,-32.6935 3472.8081,-35.0341 3516,-38 3529.6573,-38.9378 3544.6219,-40.203 3557.8186,-41.3998\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"3557.6846,-44.9023 3567.9643,-42.3374 3558.3288,-37.932 3557.6846,-44.9023\"/>\n<text text-anchor=\"middle\" x=\"3460.5\" y=\"-41\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">256</text>\n</g>\n<!-- 6502780788632146745&#45;&gt;/outputs/231 -->\n<g id=\"edge49\" class=\"edge\">\n<title>6502780788632146745&#45;&gt;/outputs/231</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M4194.3716,-31.5543C4208.2428,-33.9263 4222.7792,-36.412 4235.5269,-38.592\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"4235.3135,-42.1062 4245.7604,-40.3419 4236.4935,-35.2064 4235.3135,-42.1062\"/>\n<text text-anchor=\"middle\" x=\"4220\" y=\"-40\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">256</text>\n</g>\n<!-- 3791255491530683333&#45;&gt;/outputs/252 -->\n<g id=\"edge51\" class=\"edge\">\n<title>3791255491530683333&#45;&gt;/outputs/252</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M4930.3716,-62.1225C4944.3815,-58.8935 4959.07,-55.508 4971.9088,-52.5489\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"4972.802,-55.9349 4981.7604,-50.2783 4971.2298,-49.1137 4972.802,-55.9349\"/>\n<text text-anchor=\"middle\" x=\"4956\" y=\"-60\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">512</text>\n</g>\n<!-- 10149767426592778049&#45;&gt;/outputs/252 -->\n<g id=\"edge53\" class=\"edge\">\n<title>10149767426592778049&#45;&gt;/outputs/252</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M4801.2424,-30.1435C4837.8623,-31.6935 4886.8081,-34.0341 4930,-37 4943.6573,-37.9378 4958.6219,-39.203 4971.8186,-40.3998\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"4971.6846,-43.9023 4981.9643,-41.3374 4972.3288,-36.932 4971.6846,-43.9023\"/>\n<text text-anchor=\"middle\" x=\"4874.5\" y=\"-40\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">512</text>\n</g>\n<!-- 8551728508530118685&#45;&gt;/outputs/267 -->\n<g id=\"edge55\" class=\"edge\">\n<title>8551728508530118685&#45;&gt;/outputs/267</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M5608.3716,-30.5543C5622.2428,-32.9263 5636.7792,-35.412 5649.5269,-37.592\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"5649.3135,-41.1062 5659.7604,-39.3419 5650.4935,-34.2064 5649.3135,-41.1062\"/>\n<text text-anchor=\"middle\" x=\"5634\" y=\"-39\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">512</text>\n</g>\n</g>\n</svg>\n"
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "RAodboScj5w6"
      },
      "source": [
        "## Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Dzh3ROmRj5w7",
        "scrolled": false,
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "cf0ee808-a38e-456c-db31-1e33f1d65870"
      },
      "source": [
        "def compute_accuracy(model, data_loader, device):\n",
        "    correct_pred, num_examples = 0, 0\n",
        "    for i, (features, targets) in enumerate(data_loader):\n",
        "            \n",
        "        features = features.to(device)\n",
        "        targets = targets.to(device)\n",
        "\n",
        "        logits, probas = model(features)\n",
        "        _, predicted_labels = torch.max(probas, 1)\n",
        "        num_examples += targets.size(0)\n",
        "        correct_pred += (predicted_labels == targets).sum()\n",
        "    return correct_pred.float()/num_examples * 100\n",
        "    \n",
        "\n",
        "start_time = time.time()\n",
        "for epoch in range(NUM_EPOCHS):\n",
        "    \n",
        "    model.train()\n",
        "    for batch_idx, (features, targets) in enumerate(train_loader):\n",
        "        \n",
        "        features = features.to(DEVICE)\n",
        "        targets = targets.to(DEVICE)\n",
        "            \n",
        "        ### FORWARD AND BACK PROP\n",
        "        logits, probas = model(features)\n",
        "        cost = F.cross_entropy(logits, targets)\n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        cost.backward()\n",
        "        \n",
        "        ### UPDATE MODEL PARAMETERS\n",
        "        optimizer.step()\n",
        "        \n",
        "        ### LOGGING\n",
        "        if not batch_idx % 50:\n",
        "            print ('Epoch: %03d/%03d | Batch %04d/%04d | Cost: %.4f' \n",
        "                   %(epoch+1, NUM_EPOCHS, batch_idx, \n",
        "                     len(train_loader), cost))\n",
        "\n",
        "        \n",
        "\n",
        "    model.eval()\n",
        "    with torch.set_grad_enabled(False): # save memory during inference\n",
        "        print('Epoch: %03d/%03d | Train: %.3f%%' % (\n",
        "              epoch+1, NUM_EPOCHS, \n",
        "              compute_accuracy(model, train_loader, device=DEVICE)))\n",
        "        \n",
        "    print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))\n",
        "    \n",
        "print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 001/010 | Batch 0000/0469 | Cost: 2.7034\n",
            "Epoch: 001/010 | Batch 0050/0469 | Cost: 0.1411\n",
            "Epoch: 001/010 | Batch 0100/0469 | Cost: 0.0602\n",
            "Epoch: 001/010 | Batch 0150/0469 | Cost: 0.1213\n",
            "Epoch: 001/010 | Batch 0200/0469 | Cost: 0.1145\n",
            "Epoch: 001/010 | Batch 0250/0469 | Cost: 0.0601\n",
            "Epoch: 001/010 | Batch 0300/0469 | Cost: 0.0696\n",
            "Epoch: 001/010 | Batch 0350/0469 | Cost: 0.0928\n",
            "Epoch: 001/010 | Batch 0400/0469 | Cost: 0.0605\n",
            "Epoch: 001/010 | Batch 0450/0469 | Cost: 0.0588\n",
            "Epoch: 001/010 | Train: 98.287%\n",
            "Time elapsed: 0.33 min\n",
            "Epoch: 002/010 | Batch 0000/0469 | Cost: 0.0425\n",
            "Epoch: 002/010 | Batch 0050/0469 | Cost: 0.0653\n",
            "Epoch: 002/010 | Batch 0100/0469 | Cost: 0.0695\n",
            "Epoch: 002/010 | Batch 0150/0469 | Cost: 0.0538\n",
            "Epoch: 002/010 | Batch 0200/0469 | Cost: 0.0359\n",
            "Epoch: 002/010 | Batch 0250/0469 | Cost: 0.0151\n",
            "Epoch: 002/010 | Batch 0300/0469 | Cost: 0.0172\n",
            "Epoch: 002/010 | Batch 0350/0469 | Cost: 0.0826\n",
            "Epoch: 002/010 | Batch 0400/0469 | Cost: 0.0578\n",
            "Epoch: 002/010 | Batch 0450/0469 | Cost: 0.0780\n",
            "Epoch: 002/010 | Train: 98.197%\n",
            "Time elapsed: 0.67 min\n",
            "Epoch: 003/010 | Batch 0000/0469 | Cost: 0.0483\n",
            "Epoch: 003/010 | Batch 0050/0469 | Cost: 0.0242\n",
            "Epoch: 003/010 | Batch 0100/0469 | Cost: 0.0385\n",
            "Epoch: 003/010 | Batch 0150/0469 | Cost: 0.0299\n",
            "Epoch: 003/010 | Batch 0200/0469 | Cost: 0.0335\n",
            "Epoch: 003/010 | Batch 0250/0469 | Cost: 0.0178\n",
            "Epoch: 003/010 | Batch 0300/0469 | Cost: 0.0806\n",
            "Epoch: 003/010 | Batch 0350/0469 | Cost: 0.0280\n",
            "Epoch: 003/010 | Batch 0400/0469 | Cost: 0.0077\n",
            "Epoch: 003/010 | Batch 0450/0469 | Cost: 0.0182\n",
            "Epoch: 003/010 | Train: 99.267%\n",
            "Time elapsed: 1.00 min\n",
            "Epoch: 004/010 | Batch 0000/0469 | Cost: 0.0628\n",
            "Epoch: 004/010 | Batch 0050/0469 | Cost: 0.1660\n",
            "Epoch: 004/010 | Batch 0100/0469 | Cost: 0.0272\n",
            "Epoch: 004/010 | Batch 0150/0469 | Cost: 0.0437\n",
            "Epoch: 004/010 | Batch 0200/0469 | Cost: 0.0735\n",
            "Epoch: 004/010 | Batch 0250/0469 | Cost: 0.0016\n",
            "Epoch: 004/010 | Batch 0300/0469 | Cost: 0.0171\n",
            "Epoch: 004/010 | Batch 0350/0469 | Cost: 0.0059\n",
            "Epoch: 004/010 | Batch 0400/0469 | Cost: 0.0034\n",
            "Epoch: 004/010 | Batch 0450/0469 | Cost: 0.0183\n",
            "Epoch: 004/010 | Train: 99.128%\n",
            "Time elapsed: 1.34 min\n",
            "Epoch: 005/010 | Batch 0000/0469 | Cost: 0.0403\n",
            "Epoch: 005/010 | Batch 0050/0469 | Cost: 0.0007\n",
            "Epoch: 005/010 | Batch 0100/0469 | Cost: 0.0700\n",
            "Epoch: 005/010 | Batch 0150/0469 | Cost: 0.0177\n",
            "Epoch: 005/010 | Batch 0200/0469 | Cost: 0.0287\n",
            "Epoch: 005/010 | Batch 0250/0469 | Cost: 0.0333\n",
            "Epoch: 005/010 | Batch 0300/0469 | Cost: 0.0140\n",
            "Epoch: 005/010 | Batch 0350/0469 | Cost: 0.0064\n",
            "Epoch: 005/010 | Batch 0400/0469 | Cost: 0.0183\n",
            "Epoch: 005/010 | Batch 0450/0469 | Cost: 0.0082\n",
            "Epoch: 005/010 | Train: 99.297%\n",
            "Time elapsed: 1.67 min\n",
            "Epoch: 006/010 | Batch 0000/0469 | Cost: 0.0205\n",
            "Epoch: 006/010 | Batch 0050/0469 | Cost: 0.0288\n",
            "Epoch: 006/010 | Batch 0100/0469 | Cost: 0.0201\n",
            "Epoch: 006/010 | Batch 0150/0469 | Cost: 0.0207\n",
            "Epoch: 006/010 | Batch 0200/0469 | Cost: 0.0165\n",
            "Epoch: 006/010 | Batch 0250/0469 | Cost: 0.0121\n",
            "Epoch: 006/010 | Batch 0300/0469 | Cost: 0.0282\n",
            "Epoch: 006/010 | Batch 0350/0469 | Cost: 0.0275\n",
            "Epoch: 006/010 | Batch 0400/0469 | Cost: 0.0580\n",
            "Epoch: 006/010 | Batch 0450/0469 | Cost: 0.0228\n",
            "Epoch: 006/010 | Train: 99.477%\n",
            "Time elapsed: 2.00 min\n",
            "Epoch: 007/010 | Batch 0000/0469 | Cost: 0.0424\n",
            "Epoch: 007/010 | Batch 0050/0469 | Cost: 0.0039\n",
            "Epoch: 007/010 | Batch 0100/0469 | Cost: 0.0060\n",
            "Epoch: 007/010 | Batch 0150/0469 | Cost: 0.0513\n",
            "Epoch: 007/010 | Batch 0200/0469 | Cost: 0.0129\n",
            "Epoch: 007/010 | Batch 0250/0469 | Cost: 0.0022\n",
            "Epoch: 007/010 | Batch 0300/0469 | Cost: 0.0937\n",
            "Epoch: 007/010 | Batch 0350/0469 | Cost: 0.0312\n",
            "Epoch: 007/010 | Batch 0400/0469 | Cost: 0.0576\n",
            "Epoch: 007/010 | Batch 0450/0469 | Cost: 0.0014\n",
            "Epoch: 007/010 | Train: 99.385%\n",
            "Time elapsed: 2.34 min\n",
            "Epoch: 008/010 | Batch 0000/0469 | Cost: 0.0699\n",
            "Epoch: 008/010 | Batch 0050/0469 | Cost: 0.0016\n",
            "Epoch: 008/010 | Batch 0100/0469 | Cost: 0.0098\n",
            "Epoch: 008/010 | Batch 0150/0469 | Cost: 0.0253\n",
            "Epoch: 008/010 | Batch 0200/0469 | Cost: 0.0016\n",
            "Epoch: 008/010 | Batch 0250/0469 | Cost: 0.0031\n",
            "Epoch: 008/010 | Batch 0300/0469 | Cost: 0.0104\n",
            "Epoch: 008/010 | Batch 0350/0469 | Cost: 0.0025\n",
            "Epoch: 008/010 | Batch 0400/0469 | Cost: 0.0202\n",
            "Epoch: 008/010 | Batch 0450/0469 | Cost: 0.0529\n",
            "Epoch: 008/010 | Train: 99.223%\n",
            "Time elapsed: 2.67 min\n",
            "Epoch: 009/010 | Batch 0000/0469 | Cost: 0.0265\n",
            "Epoch: 009/010 | Batch 0050/0469 | Cost: 0.0371\n",
            "Epoch: 009/010 | Batch 0100/0469 | Cost: 0.0252\n",
            "Epoch: 009/010 | Batch 0150/0469 | Cost: 0.0121\n",
            "Epoch: 009/010 | Batch 0200/0469 | Cost: 0.0021\n",
            "Epoch: 009/010 | Batch 0250/0469 | Cost: 0.0038\n",
            "Epoch: 009/010 | Batch 0300/0469 | Cost: 0.0095\n",
            "Epoch: 009/010 | Batch 0350/0469 | Cost: 0.0014\n",
            "Epoch: 009/010 | Batch 0400/0469 | Cost: 0.0696\n",
            "Epoch: 009/010 | Batch 0450/0469 | Cost: 0.0027\n",
            "Epoch: 009/010 | Train: 98.938%\n",
            "Time elapsed: 3.01 min\n",
            "Epoch: 010/010 | Batch 0000/0469 | Cost: 0.0008\n",
            "Epoch: 010/010 | Batch 0050/0469 | Cost: 0.0147\n",
            "Epoch: 010/010 | Batch 0100/0469 | Cost: 0.0098\n",
            "Epoch: 010/010 | Batch 0150/0469 | Cost: 0.0094\n",
            "Epoch: 010/010 | Batch 0200/0469 | Cost: 0.0014\n",
            "Epoch: 010/010 | Batch 0250/0469 | Cost: 0.0181\n",
            "Epoch: 010/010 | Batch 0300/0469 | Cost: 0.0154\n",
            "Epoch: 010/010 | Batch 0350/0469 | Cost: 0.0031\n",
            "Epoch: 010/010 | Batch 0400/0469 | Cost: 0.0049\n",
            "Epoch: 010/010 | Batch 0450/0469 | Cost: 0.0034\n",
            "Epoch: 010/010 | Train: 99.837%\n",
            "Time elapsed: 3.34 min\n",
            "Total Training Time: 3.34 min\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "paaeEQHQj5xC"
      },
      "source": [
        "## Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "gzQMWKq5j5xE",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "64501a70-6b33-4760-d464-d8042d12e2c5"
      },
      "source": [
        "with torch.set_grad_enabled(False): # save memory during inference\n",
        "    print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader, device=DEVICE)))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test accuracy: 99.49%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6KyigbIjlXzG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "outputId": "ff993d03-75cb-4409-aaa7-125f287a3dee"
      },
      "source": [
        "for batch_idx, (features, targets) in enumerate(test_loader):\n",
        "\n",
        "    features = features\n",
        "    targets = targets\n",
        "    break\n",
        "    \n",
        "    \n",
        "nhwc_img = np.transpose(features[0], axes=(1, 2, 0))\n",
        "nhw_img = np.squeeze(nhwc_img.numpy(), axis=2)\n",
        "plt.imshow(nhw_img, cmap='Greys');"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAANMUlEQVR4nO3db4hd9Z3H8c9nY6PBFs2YIQ5pdGIRjC5uUoYYbCguZYN/HsQ8UBqlZFGaPlBpsQ/8sw8aBTEs29Y8WArpJibVrqXQxkSQ2myomIIGR5lqorijcSQJ+XNDwFgRqsl3H8xJd4xzz4z3nPsn+b5fMNx7z/eec74c8sm59/zuvT9HhACc+/6h2w0A6AzCDiRB2IEkCDuQBGEHkjivkzubM2dODA4OdnKXQCpjY2M6duyYJ6tVCrvtGyWtlzRD0n9FxLqy5w8ODmp4eLjKLgGUGBoaalpr+WW87RmS/lPSTZKulrTK9tWtbg9Ae1V5z75E0rsRsS8i/ibpN5JW1NMWgLpVCfs8SfsnPD5QLPsc22tsD9sebjQaFXYHoIq2X42PiA0RMRQRQ/39/e3eHYAmqoT9oKT5Ex5/vVgGoAdVCfurkq60vcD2TEnflbS9nrYA1K3lobeI+Mz2vZJe0PjQ26aI2FtbZwBqVWmcPSKel/R8Tb0AaCM+LgskQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IotKUzbbHJH0k6aSkzyJiqI6mANSvUtgL/xwRx2rYDoA24mU8kETVsIekP9p+zfaayZ5ge43tYdvDjUaj4u4AtKpq2JdFxDcl3STpHtvfPvMJEbEhIoYiYqi/v7/i7gC0qlLYI+JgcXtU0lZJS+poCkD9Wg677Qttf+30fUnLJe2pqzEA9apyNX6upK22T2/nvyPiD7V0BaB2LYc9IvZJ+qcaewHQRgy9AUkQdiAJwg4kQdiBJAg7kEQdX4RJ4ZVXXmlaW79+fem68+bNK63PmjWrtL569erSel9fX0s15MKZHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSYJx9msrGukdHR9u678cee6y0ftFFFzWtLV26tO52zhqDg4NNaw899FDpupdddlnN3XQfZ3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJx9ml69tlnm9ZGRkZK173mmmtK63v37i2t7969u7S+bdu2prUXXnihdN0FCxaU1t9///3SehXnnVf+z29gYKC0vn///pb3XTYGL0kPPPBAy9vuVZzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtmnaeHChS3VpuPaa68tra9ataq0vm7duqa1sbGx0nWnGmfft29fab2KmTNnltanGmefqvdGo9G0dtVVV5Wuey6a8sxue5Pto7b3TFjWZ3uH7dHidnZ72wRQ1XRexm+WdOMZyx6UtDMirpS0s3gMoIdNGfaIeEnS8TMWr5C0pbi/RdKtNfcFoGatXqCbGxGHivuHJc1t9kTba2wP2x4uew8FoL0qX42PiJAUJfUNETEUEUP9/f1VdwegRa2G/YjtAUkqbo/W1xKAdmg17Nslnf5t5dWSmn/HEkBPmHKc3fYzkm6QNMf2AUk/kbRO0m9t3y3pA0m3t7NJlLvgggua1qqOJ1f9DEEVU32P/9ixY6X16667rmlt+fLlLfV0Npsy7BHR7BMd36m5FwBtxMdlgSQIO5AEYQeSIOxAEoQdSIKvuKJrPv7449L6ypUrS+unTp0qrT/xxBNNa7NmzSpd91zEmR1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkmCcHV2zefPm0vrhw4dL65dccklp/fLLL/+yLZ3TOLMDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKMs6Ot3nvvvaa1+++/v9K2X3755dL6pZdeWmn75xrO7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBOPsaKvnnnuuae3TTz8tXfe2224rrV9xxRUt9ZTVlGd225tsH7W9Z8KytbYP2h4p/m5ub5sAqprOy/jNkm6cZPnPI2JR8fd8vW0BqNuUYY+IlyQd70AvANqoygW6e22/UbzMn93sSbbX2B62PdxoNCrsDkAVrYb9F5K+IWmRpEOSftrsiRGxISKGImKov7+/xd0BqKqlsEfEkYg4GRGnJP1S0pJ62wJQt5bCbntgwsOVkvY0ey6A3jDlOLvtZyTdIGmO7QOSfiLpBtuLJIWkMUk/aGOP6GFTjZVv3bq1ae38888vXffxxx8vrc+YMaO0js+bMuwRsWqSxRvb0AuANuLjskAShB1IgrADSRB2IAnCDiTBV1xRycaN5QMzu3btalq74447StflK6z14swOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kwzo5SIyMjpfX77ruvtH7xxRc3rT366KMt9YTWcGYHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQYZ0/uk08+Ka2vWjXZjwv/v5MnT5bW77zzzqY1vq/eWZzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtnPcadOnSqt33LLLaX1d955p7S+cOHC0vojjzxSWkfnTHlmtz3f9p9sv2V7r+0fFsv7bO+wPVrczm5/uwBaNZ2X8Z9J+nFEXC1pqaR7bF8t6UFJOyPiSkk7i8cAetSUYY+IQxHxenH/I0lvS5onaYWkLcXTtki6tV1NAqjuS12gsz0oabGk3ZLmRsShonRY0twm66yxPWx7uNFoVGgVQBXTDrvtr0r6naQfRcSJibWICEkx2XoRsSEihiJiqL+/v1KzAFo3rbDb/orGg/7riPh9sfiI7YGiPiDpaHtaBFCHKYfebFvSRklvR8TPJpS2S1otaV1xu60tHaKS48ePl9ZffPHFStt/6qmnSut9fX2Vto/6TGec/VuSvifpTdunf0T8YY2H/Le275b0gaTb29MigDpMGfaI+LMkNyl/p952ALQLH5cFkiDsQBKEHUiCsANJEHYgCb7ieg748MMPm9aWLl1aadtPP/10aX3x4sWVto/O4cwOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kwzn4OePLJJ5vW9u3bV2nby5YtK62P/9wBzgac2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcbZzwKjo6Ol9bVr13amEZzVOLMDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBLTmZ99vqRfSZorKSRtiIj1ttdK+r6kRvHUhyPi+XY1mtmuXbtK6ydOnGh52wsXLiytz5o1q+Vto7dM50M1n0n6cUS8bvtrkl6zvaOo/Twi/qN97QGoy3TmZz8k6VBx/yPbb0ua1+7GANTrS71ntz0oabGk3cWie22/YXuT7dlN1llje9j2cKPRmOwpADpg2mG3/VVJv5P0o4g4IekXkr4haZHGz/w/nWy9iNgQEUMRMdTf319DywBaMa2w2/6KxoP+64j4vSRFxJGIOBkRpyT9UtKS9rUJoKopw+7xnw/dKOntiPjZhOUDE562UtKe+tsDUJfpXI3/lqTvSXrT9kix7GFJq2wv0vhw3JikH7SlQ1Ry/fXXl9Z37NhRWmfo7dwxnavxf5Y02Y+DM6YOnEX4BB2QBGEHkiDsQBKEHUiCsANJEHYgCX5K+ixw1113VaoDEmd2IA3CDiRB2IEkCDuQBGEHkiDsQBKEHUjCEdG5ndkNSR9MWDRH0rGONfDl9GpvvdqXRG+tqrO3yyNi0t9/62jYv7BzezgihrrWQIle7a1X+5LorVWd6o2X8UAShB1Iotth39Dl/Zfp1d56tS+J3lrVkd66+p4dQOd0+8wOoEMIO5BEV8Ju+0bb79h+1/aD3eihGdtjtt+0PWJ7uMu9bLJ91PaeCcv6bO+wPVrcTjrHXpd6W2v7YHHsRmzf3KXe5tv+k+23bO+1/cNieVePXUlfHTluHX/PbnuGpP+V9C+SDkh6VdKqiHiro400YXtM0lBEdP0DGLa/Lemvkn4VEf9YLPt3SccjYl3xH+XsiHigR3pbK+mv3Z7Gu5itaGDiNOOSbpX0r+risSvp63Z14Lh148y+RNK7EbEvIv4m6TeSVnShj54XES9JOn7G4hWSthT3t2j8H0vHNemtJ0TEoYh4vbj/kaTT04x39diV9NUR3Qj7PEn7Jzw+oN6a7z0k/dH2a7bXdLuZScyNiEPF/cOS5nazmUlMOY13J50xzXjPHLtWpj+vigt0X7QsIr4p6SZJ9xQvV3tSjL8H66Wx02lN490pk0wz/nfdPHatTn9eVTfCflDS/AmPv14s6wkRcbC4PSppq3pvKuojp2fQLW6Pdrmfv+ulabwnm2ZcPXDsujn9eTfC/qqkK20vsD1T0nclbe9CH19g+8LiwolsXyhpuXpvKurtklYX91dL2tbFXj6nV6bxbjbNuLp87Lo+/XlEdPxP0s0avyL/nqR/60YPTfq6QtJfir+93e5N0jMaf1n3qcavbdwt6RJJOyWNSvofSX091NtTkt6U9IbGgzXQpd6Wafwl+huSRoq/m7t97Er66shx4+OyQBJcoAOSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJP4PW2vnUJwzgQIAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6Ph4wKTDlXzJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "e02cba66-2f8c-483c-8cab-69a890ef4aef"
      },
      "source": [
        "model.eval()\n",
        "logits, probas = model(features.to(device)[0, None])\n",
        "print('Probability 7 %.2f%%' % (probas[0][7]*100))"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Probability 7 100.00%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lAEICKXYlXzL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        },
        "outputId": "7ff48dce-42f4-43f5-d609-e0ceb14ad1bc"
      },
      "source": [
        "%watermark -iv"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "pandas    1.0.5\n",
            "PIL.Image 7.0.0\n",
            "torch     1.5.1+cu101\n",
            "numpy     1.18.5\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}