ex4_neuralNetwork(BP)_20190204.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 神经网络解决多分类问题\n",
    "手写数字识别，这次需要我们自己训练权重"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy.io as sio\n",
    "from scipy.optimize import minimize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "#数据集\n",
    "path = '/home/alex/test/NG_exercise/ex4_neuralNetwork(BP)/ex4data1.mat'\n",
    "data = sio.loadmat(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[10],\n",
       "       [10],\n",
       "       [10],\n",
       "       ...,\n",
       "       [ 9],\n",
       "       [ 9],\n",
       "       [ 9]], dtype=uint8)"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_x = data['X'] #存放图片数据\n",
    "raw_y = data['y'] #存放期望结果（数字）\n",
    "raw_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  8.56059680e-06,  1.94035948e-06,\n",
       "        -7.37438725e-04, -8.13403799e-03, -1.86104473e-02,\n",
       "        -1.87412865e-02, -1.87572508e-02, -1.90963542e-02,\n",
       "        -1.64039011e-02, -3.78191381e-03,  3.30347316e-04,\n",
       "         1.27655229e-05,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  1.16421569e-04,\n",
       "         1.20052179e-04, -1.40444581e-02, -2.84542484e-02,\n",
       "         8.03826593e-02,  2.66540339e-01,  2.73853746e-01,\n",
       "         2.78729541e-01,  2.74293607e-01,  2.24676403e-01,\n",
       "         2.77562977e-02, -7.06315478e-03,  2.34715414e-04,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         1.28335523e-17, -3.26286765e-04, -1.38651604e-02,\n",
       "         8.15651552e-02,  3.82800381e-01,  8.57849775e-01,\n",
       "         1.00109761e+00,  9.69710638e-01,  9.30928598e-01,\n",
       "         1.00383757e+00,  9.64157356e-01,  4.49256553e-01,\n",
       "        -5.60408259e-03, -3.78319036e-03,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         5.10620915e-06,  4.36410675e-04, -3.95509940e-03,\n",
       "        -2.68537241e-02,  1.00755014e-01,  6.42031710e-01,\n",
       "         1.03136838e+00,  8.50968614e-01,  5.43122379e-01,\n",
       "         3.42599738e-01,  2.68918777e-01,  6.68374643e-01,\n",
       "         1.01256958e+00,  9.03795598e-01,  1.04481574e-01,\n",
       "        -1.66424973e-02,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  2.59875260e-05,\n",
       "        -3.10606987e-03,  7.52456076e-03,  1.77539831e-01,\n",
       "         7.92890120e-01,  9.65626503e-01,  4.63166079e-01,\n",
       "         6.91720680e-02, -3.64100526e-03, -4.12180405e-02,\n",
       "        -5.01900656e-02,  1.56102907e-01,  9.01762651e-01,\n",
       "         1.04748346e+00,  1.51055252e-01, -2.16044665e-02,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         5.87012352e-05, -6.40931373e-04, -3.23305249e-02,\n",
       "         2.78203465e-01,  9.36720163e-01,  1.04320956e+00,\n",
       "         5.98003217e-01, -3.59409041e-03, -2.16751770e-02,\n",
       "        -4.81021923e-03,  6.16566793e-05, -1.23773318e-02,\n",
       "         1.55477482e-01,  9.14867477e-01,  9.20401348e-01,\n",
       "         1.09173902e-01, -1.71058007e-02,  0.00000000e+00,\n",
       "         0.00000000e+00,  1.56250000e-04, -4.27724104e-04,\n",
       "        -2.51466503e-02,  1.30532561e-01,  7.81664862e-01,\n",
       "         1.02836583e+00,  7.57137601e-01,  2.84667194e-01,\n",
       "         4.86865128e-03, -3.18688725e-03,  0.00000000e+00,\n",
       "         8.36492601e-04, -3.70751123e-02,  4.52644165e-01,\n",
       "         1.03180133e+00,  5.39028101e-01, -2.43742611e-03,\n",
       "        -4.80290033e-03,  0.00000000e+00,  0.00000000e+00,\n",
       "        -7.03635621e-04, -1.27262443e-02,  1.61706648e-01,\n",
       "         7.79865383e-01,  1.03676705e+00,  8.04490400e-01,\n",
       "         1.60586724e-01, -1.38173339e-02,  2.14879493e-03,\n",
       "        -2.12622549e-04,  2.04248366e-04, -6.85907627e-03,\n",
       "         4.31712963e-04,  7.20680947e-01,  8.48136063e-01,\n",
       "         1.51383408e-01, -2.28404366e-02,  1.98971950e-04,\n",
       "         0.00000000e+00,  0.00000000e+00, -9.40410539e-03,\n",
       "         3.74520505e-02,  6.94389110e-01,  1.02844844e+00,\n",
       "         1.01648066e+00,  8.80488426e-01,  3.92123945e-01,\n",
       "        -1.74122413e-02, -1.20098039e-04,  5.55215142e-05,\n",
       "        -2.23907271e-03, -2.76068376e-02,  3.68645493e-01,\n",
       "         9.36411169e-01,  4.59006723e-01, -4.24701797e-02,\n",
       "         1.17356610e-03,  1.88929739e-05,  0.00000000e+00,\n",
       "         0.00000000e+00, -1.93511951e-02,  1.29999794e-01,\n",
       "         9.79821705e-01,  9.41862388e-01,  7.75147704e-01,\n",
       "         8.73632241e-01,  2.12778350e-01, -1.72353349e-02,\n",
       "         0.00000000e+00,  1.09937426e-03, -2.61793751e-02,\n",
       "         1.22872879e-01,  8.30812662e-01,  7.26501773e-01,\n",
       "         5.24441863e-02, -6.18971913e-03,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "        -9.36563862e-03,  3.68349741e-02,  6.99079299e-01,\n",
       "         1.00293583e+00,  6.05704402e-01,  3.27299224e-01,\n",
       "        -3.22099249e-02, -4.83053002e-02, -4.34069138e-02,\n",
       "        -5.75151144e-02,  9.55674190e-02,  7.26512627e-01,\n",
       "         6.95366966e-01,  1.47114481e-01, -1.20048679e-02,\n",
       "        -3.02798203e-04,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00, -6.76572712e-04,\n",
       "        -6.51415556e-03,  1.17339359e-01,  4.21948410e-01,\n",
       "         9.93210937e-01,  8.82013974e-01,  7.45758734e-01,\n",
       "         7.23874268e-01,  7.23341725e-01,  7.20020340e-01,\n",
       "         8.45324959e-01,  8.31859739e-01,  6.88831870e-02,\n",
       "        -2.77765012e-02,  3.59136710e-04,  7.14869281e-05,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  1.53186275e-04,  3.17353553e-04,\n",
       "        -2.29167177e-02, -4.14402914e-03,  3.87038450e-01,\n",
       "         5.04583435e-01,  7.74885876e-01,  9.90037446e-01,\n",
       "         1.00769478e+00,  1.00851440e+00,  7.37905042e-01,\n",
       "         2.15455291e-01, -2.69624864e-02,  1.32506127e-03,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  2.36366422e-04,\n",
       "        -2.26031454e-03, -2.51994485e-02, -3.73889910e-02,\n",
       "         6.62121228e-02,  2.91134498e-01,  3.23055726e-01,\n",
       "         3.06260315e-01,  8.76070942e-02, -2.50581917e-02,\n",
       "         2.37438725e-04,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         6.20939216e-18,  6.72618320e-04, -1.13151411e-02,\n",
       "        -3.54641066e-02, -3.88214912e-02, -3.71077412e-02,\n",
       "        -1.33524928e-02,  9.90964718e-04,  4.89176960e-05,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\n",
       "         0.00000000e+00]])"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_x[0:1,:]#打印第一行看下图片数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5000, 400), (5000, 1))"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_x.shape,raw_y.shape  #20*20的图片"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 0., 0., ..., 0., 0., 0.],\n",
       "       [1., 0., 0., ..., 0., 0., 0.],\n",
       "       [1., 0., 0., ..., 0., 0., 0.],\n",
       "       ...,\n",
       "       [1., 0., 0., ..., 0., 0., 0.],\n",
       "       [1., 0., 0., ..., 0., 0., 0.],\n",
       "       [1., 0., 0., ..., 0., 0., 0.]])"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#在第一列（第二维度的第一个元素位置）添加偏置向量\n",
    "X = np.insert(raw_x, 0, values=1, axis=1)\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000, 401)"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 对y进行热编码处理：one-hot编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot_encoder(raw_y):\n",
    "    \n",
    "    result = []\n",
    "    \n",
    "    for i in raw_y: #raw_y存放的是1～10的数字\n",
    "        y_temp = np.zeros(10)\n",
    "        y_temp[i-1] = 1  #十个数字中只有一个是1\n",
    "#         print(type(y_temp))\n",
    "        result.append(y_temp)\n",
    "        \n",
    "    return np.array(result) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = one_hot_encoder(raw_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5000, 10), (5000, 1))"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape,raw_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'__header__': b'MATLAB 5.0 MAT-file, Platform: GLNXA64, Created on: Tue Oct 18 14:57:02 2011',\n",
       " '__version__': '1.0',\n",
       " '__globals__': [],\n",
       " 'Theta1': array([[-2.25623899e-02, -1.05624163e-08,  2.19414684e-09, ...,\n",
       "         -1.30529929e-05, -5.04175101e-06,  2.80464449e-09],\n",
       "        [-9.83811294e-02,  7.66168682e-09, -9.75873689e-09, ...,\n",
       "         -5.60134007e-05,  2.00940969e-07,  3.54422854e-09],\n",
       "        [ 1.16156052e-01, -8.77654466e-09,  8.16037764e-09, ...,\n",
       "         -1.20951657e-04, -2.33669661e-06, -7.50668099e-09],\n",
       "        ...,\n",
       "        [-1.83220638e-01, -8.89272060e-09, -9.81968100e-09, ...,\n",
       "          2.35311186e-05, -3.25484493e-06,  9.02499060e-09],\n",
       "        [-7.02096331e-01,  3.05178374e-10,  2.56061008e-09, ...,\n",
       "         -8.61759744e-04,  9.43449909e-05,  3.83761998e-09],\n",
       "        [-3.50933229e-01,  8.85876862e-09, -6.57515140e-10, ...,\n",
       "         -1.80365926e-06, -8.14464807e-06,  8.79454531e-09]]),\n",
       " 'Theta2': array([[-0.76100352, -1.21244498, -0.10187131, -2.36850085, -1.05778129,\n",
       "         -2.20823629,  0.56383834,  1.21105294,  2.21030997,  0.44456156,\n",
       "         -1.18244872,  1.04289112, -1.60558756,  1.30419943,  1.37175046,\n",
       "          1.74825095, -0.23365648, -1.52014483,  1.15324176,  0.10368082,\n",
       "         -0.37207719, -0.61530019, -0.1256836 , -2.27193038, -0.71836208,\n",
       "         -1.29690315],\n",
       "        [-0.61785176,  0.61559207, -1.26550639,  1.85745418, -0.91853319,\n",
       "         -0.05502589, -0.38589806,  1.29520853, -1.56843297, -0.97026419,\n",
       "         -2.18334895, -2.85033578, -2.07733086,  1.63163164,  0.3490229 ,\n",
       "          1.82789117, -2.44174379, -0.8563034 , -0.2982564 , -2.07947873,\n",
       "         -1.2933238 ,  0.89982032,  0.28306578,  2.31180525, -2.46444086,\n",
       "          1.45656548],\n",
       "        [-0.68934072, -1.94538151,  2.01360618, -3.12316188, -0.2361763 ,\n",
       "          1.38680947,  0.90982429, -1.54774416, -0.79830896, -0.65599834,\n",
       "          0.7353833 , -2.58593294,  0.47210839,  0.55349499,  2.51255453,\n",
       "         -2.4167454 , -1.63898627,  1.2027302 , -1.20245851, -1.83445959,\n",
       "         -1.88013027, -0.34056098,  0.23692483, -1.06137919,  1.02759232,\n",
       "         -0.47690832],\n",
       "        [-0.67832479,  0.46299226,  0.58492321, -0.1650184 ,  1.93264192,\n",
       "         -0.22965765, -1.84731492,  0.49011768,  1.07146054, -3.31905643,\n",
       "          1.54113507,  0.37371947, -0.86484681, -2.58273522,  0.97062447,\n",
       "         -0.51021867, -0.68427897, -1.64713607,  0.21153145, -0.27422442,\n",
       "          1.72599755,  1.32418658, -2.63984479, -0.08055871, -2.03510803,\n",
       "         -1.46123776],\n",
       "        [-0.59664339, -2.04481799,  2.05698407,  1.95100909,  0.17637699,\n",
       "         -2.16141218, -0.40394736,  1.80157532, -1.56278739, -0.25253004,\n",
       "          0.23586497,  0.71656699,  1.07689092, -0.35457279, -1.67743058,\n",
       "         -0.12939255, -0.67488849,  1.14066535,  1.32431237,  3.21158484,\n",
       "         -2.15888898, -2.60164082, -3.2226466 , -1.89612906, -0.87488068,\n",
       "          2.51038628],\n",
       "        [-0.87794907,  0.4344112 , -0.93161049,  0.18390778, -0.36078216,\n",
       "          0.61958137,  0.38624948, -2.65150343,  2.29710773, -2.08818098,\n",
       "         -1.86382323,  1.06057836,  0.77562146,  2.1346861 , -1.14973702,\n",
       "         -0.52081426,  0.99743429, -1.48309353, -2.3139424 ,  0.29517333,\n",
       "         -0.38704879, -2.20607697,  0.30702191, -1.17646114, -1.63462966,\n",
       "         -0.82467661],\n",
       "        [-0.52746527,  1.21564288, -1.50095981, -2.03195359, -1.52366734,\n",
       "         -2.43732079, -2.37570311, -1.39987277, -0.88735315, -0.63278873,\n",
       "          1.50450176, -1.580763  ,  0.58599217, -0.77540416,  0.94257331,\n",
       "          2.10919653,  0.54479132,  0.43773612, -1.28024228, -0.04360994,\n",
       "          1.4774997 , -1.13276949, -0.72846904,  0.04734716,  1.6574566 ,\n",
       "          1.68540944],\n",
       "        [-0.7490154 , -0.72249056, -3.15228173,  0.36577778,  0.19811362,\n",
       "         -0.73059946,  1.65263918, -2.300357  , -1.87468162,  0.98095387,\n",
       "         -1.58825159,  1.35434142,  2.17895331, -1.99239762, -2.00371362,\n",
       "         -0.388613  , -2.33992976, -2.91719062,  0.99398645, -2.70476768,\n",
       "         -1.27139772,  1.86091461, -1.20519404, -0.38014194,  0.7087181 ,\n",
       "         -2.11014003],\n",
       "        [-0.6665468 ,  0.53601845,  1.30307573, -1.03372714, -4.03084753,\n",
       "          0.58173469, -2.65717902,  0.80379994, -1.09241928,  2.49910058,\n",
       "          0.362008  ,  0.66195337, -0.92160534, -0.83123666, -2.00200952,\n",
       "         -2.94897501,  0.64564202, -1.10114694,  0.74510309,  0.58506717,\n",
       "         -1.99545251,  0.62591105,  1.80596103, -0.22309315, -1.40442136,\n",
       "         -2.1319153 ],\n",
       "        [-0.46089119, -1.43944954, -1.21809509,  0.71093011,  0.45216919,\n",
       "         -0.35953381,  0.62284954, -0.67005297, -0.7069138 ,  0.06311351,\n",
       "         -1.23199074, -1.74645233, -2.71960897, -2.21437178, -1.69307505,\n",
       "         -0.90927394,  0.87852311,  1.18664814, -1.87041262,  0.39796295,\n",
       "          1.72113872, -1.36934055,  0.8580668 , -0.24779579,  1.28009118,\n",
       "         -1.32752042]])}"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#存放权重的初始化参数\n",
    "path1 = '/home/alex/test/NG_exercise/ex4_neuralNetwork(BP)/ex4weights.mat'\n",
    "theta = sio.loadmat(path1)\n",
    "theta  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "theta1 = theta['Theta1']\n",
    "theta2 = theta['Theta2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((25, 401), (10, 26))"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(theta1.shape,theta2.shape)\n",
    "# theta1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 序列化权重参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "def serialize(a, b):\n",
    "    c = np.append(a, b)\n",
    "#     c = np.append(a.flatten(), b.flatten())\n",
    "\n",
    "    print(type(c))\n",
    "    return c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'numpy.ndarray'>\n"
     ]
    }
   ],
   "source": [
    "theta_serialize = serialize(theta1, theta2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10285,)"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theta_serialize.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-2.25623899e-02, -1.05624163e-08,  2.19414684e-09, ...,\n",
       "       -2.47795788e-01,  1.28009118e+00, -1.32752042e+00])"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theta_serialize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 解序列化权重参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "def deserialize(theta_serialize):\n",
    "    theta1 = theta_serialize[:25*401].reshape(25,401)\n",
    "    theta2 = theta_serialize[25*401:].reshape(10,26)\n",
    "    return theta1,theta2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "theta1,theta2 = deserialize(theta_serialize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((25, 401), (10, 26))"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theta1.shape, theta2.shape\n",
    "# theta1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-2.25623899e-02, -1.05624163e-08,  2.19414684e-09, ...,\n",
       "        -1.30529929e-05, -5.04175101e-06,  2.80464449e-09],\n",
       "       [-9.83811294e-02,  7.66168682e-09, -9.75873689e-09, ...,\n",
       "        -5.60134007e-05,  2.00940969e-07,  3.54422854e-09],\n",
       "       [ 1.16156052e-01, -8.77654466e-09,  8.16037764e-09, ...,\n",
       "        -1.20951657e-04, -2.33669661e-06, -7.50668099e-09],\n",
       "       ...,\n",
       "       [-1.83220638e-01, -8.89272060e-09, -9.81968100e-09, ...,\n",
       "         2.35311186e-05, -3.25484493e-06,  9.02499060e-09],\n",
       "       [-7.02096331e-01,  3.05178374e-10,  2.56061008e-09, ...,\n",
       "        -8.61759744e-04,  9.43449909e-05,  3.83761998e-09],\n",
       "       [-3.50933229e-01,  8.85876862e-09, -6.57515140e-10, ...,\n",
       "        -1.80365926e-06, -8.14464807e-06,  8.79454531e-09]])"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theta1"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "####不懂上边的两块代码，为什么序列化后又阶序列化？\n",
    "scipy.optimize.minimize的参数要求是一维数组，因此序列化"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 前向传播\n",
    "计算损失用的"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(z):\n",
    "    return np.exp(z) / (1 + np.exp(z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "#从输入向输出计算估计,h是网络的输出，求偏差用的\n",
    "def feed_forward(theta_serialize, X):\n",
    "    theta1, theta2 = deserialize(theta_serialize)\n",
    "    a1 = X              #5000*401\n",
    "    z2 = a1@theta1.T    #5000*401 @ 401*25\n",
    "    a2 = sigmoid(z2)    #5000*25\n",
    "    a2 = np.insert(a2, 0, values=1, axis=1) #添加一维偏置\n",
    "    z3 = a2@theta2.T    #5000*26  @ 26*10\n",
    "    h = sigmoid(z3)     #5000*10  #这是我们得到的估计数值\n",
    "    return a1,z2,a2,z3,h    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 损失函数，\n",
    "不带正则化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cost(theta_serialize, X, y):\n",
    "    a1,z2,a2,z3,h = feed_forward(theta_serialize, X)\n",
    "    j = -np.sum(y*np.log(h) + (1-y)*np.log(1-h)) / len(X)\n",
    "    return j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2876291651613189"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cost(theta_serialize, X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 损失函数，\n",
    "带正则化，防止过拟合"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "#三层神经网络，有两层权重，故sum1,sum2\n",
    "#偏置向量不放在正则化项里，就算放进去也不会有很大差距\n",
    "def reg_cost(theta_serialize, x, y, lamda):\n",
    "    sum1 = np.sum(np.power(theta1[:,1:], 2))\n",
    "    sum2 = np.sum(np.power(theta2[:,1:], 2))\n",
    "    reg = (sum1 + sum2) * lamda / (2*len(x))  #lamda控制theta\n",
    "    return reg + cost(theta_serialize, X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.38376985909092365"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lamda = 1\n",
    "reg_cost(theta_serialize, X, y, lamda)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 反向传播"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "1.无正则化的梯度,误差反向传递，求梯度用\n",
    "  偏置项不求梯度。\n",
    "  链式求导法则。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid_gradient(z):\n",
    "    return sigmoid(z) * (1 - sigmoid(z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "#这段梯度下降不是很明白\n",
    "#误差反向传播\n",
    "def gradient(theta_serialize, X, y):\n",
    "    theta1, theta2 = deserialize(theta_serialize)\n",
    "    a1,z2,a2,z3,h = feed_forward(theta_serialize, X)\n",
    "    d3 = h - y #最后一层的误差\n",
    "    d2 = d3 @ theta2[:,1:] * sigmoid_gradient(z2)  #(BP2公式)\n",
    "    D2 = (d3.T @ a2) / len(X)  #代价函数对于第二层的偏导数  (BP4公式)\n",
    "    D1 = (d2.T @ a1) / len(X)  #代价函数对于第一层的偏导数  (BP4)\n",
    "    return serialize(D1, D2)   #"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "2.带正则化的梯度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reg_gredient(theta_serialize, X, y, lamda):\n",
    "    D = gradient(theta_serialize, X, y)\n",
    "    D1,D2 = deserialize(D)\n",
    "    #加上正则化项，防止过拟合\n",
    "    theta1,theta2 = deserialize(theta_serialize)\n",
    "    D1[:,1:] = D1[:,1:] + theta1[:,1:] * lamda / len(X)\n",
    "    D2[:,1:] = D2[:,1:] + theta2[:,1:] * lamda / len(X)\n",
    "    \n",
    "    return serialize(D1, D2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 神经网络优化\n",
    "优化权重theta1，theta2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.optimize import minimize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "#以前是自己写梯度下降来求theta或者最小二乘法得到局部最优theta\n",
    "#这里用现成的模块\n",
    "def nn_training(X, y):\n",
    "    #生成10285个±0.5之间的浮点数\n",
    "    init_theta = np.random.uniform(-0.5, 0.5, 10285)\n",
    "    #求极值\n",
    "    res = minimize(fun = reg_cost,\n",
    "                   x0 = init_theta,\n",
    "                   args = (X, y, lamda),\n",
    "                   method = 'TNC',\n",
    "                   jac = reg_gredient,\n",
    "                   options = {'maxiter':300})\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "     fun: 1.5212680320027947\n",
      "     jac: array([ 1.72154042e-04, -4.45587223e-06,  9.65731117e-07, ...,\n",
      "       -1.05268194e-03, -9.88580416e-04, -1.69509604e-03])\n",
      " message: 'Max. number of function evaluations reached'\n",
      "    nfev: 300\n",
      "     nit: 20\n",
      "  status: 3\n",
      " success: False\n",
      "       x: array([ 1.30796186e+00, -2.22793612e-03,  4.82865559e-04, ...,\n",
      "       -1.38971162e+00,  8.85857550e-01,  1.64412759e-01])\n"
     ]
    }
   ],
   "source": [
    "lamda = 10\n",
    "res = nn_training(X, y)\n",
    "print(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_y = data['y'].reshape(5000,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[2.20569925e-04 1.04758231e-02 8.18003551e-03 ... 4.76418435e-03\n",
      "  4.93944083e-03 9.73736034e-01]\n",
      " [2.85401965e-04 6.80839088e-03 1.35604293e-02 ... 4.00948047e-03\n",
      "  4.05264823e-03 9.54794578e-01]\n",
      " [3.94898514e-04 9.33136442e-03 1.33338875e-02 ... 5.38362752e-02\n",
      "  6.76175421e-03 8.27683144e-01]\n",
      " ...\n",
      " [3.46092640e-02 2.15776417e-03 3.78886142e-02 ... 1.53051069e-02\n",
      "  3.88758598e-01 1.81493953e-04]\n",
      " [2.57619007e-03 6.81741146e-04 2.23810673e-03 ... 5.04773477e-02\n",
      "  8.43722519e-01 1.57271252e-03]\n",
      " [6.46592372e-04 7.34093330e-03 1.48666897e-03 ... 1.47006207e-02\n",
      "  2.29290464e-01 1.49645775e-01]]\n",
      "y_pred [1 1 1 ... 9 9 9]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.94"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "_,_,_,_,h = feed_forward(res.x, X)\n",
    "print(h)\n",
    "\n",
    "#求第一维中取最大的数字\n",
    "y_pred = np.argmax(h, axis=1) + 1\n",
    "print('y_pred', y_pred[500:,])\n",
    "\n",
    "acc = np.mean(y_pred == raw_y)\n",
    "\n",
    "acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 可视化隐藏层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_hidden_layer(theta):\n",
    "    theta1,_ = deserialize(theta)\n",
    "    hidden_layer = theta1[:,1:] # 25×400  #偏置向量排除\n",
    "    \n",
    "    \n",
    "    fig,ax = plt.subplots(ncols=5, nrows=5,figsize=(8,8), \n",
    "                           sharex=True, sharey=True\n",
    "                         )\n",
    "    \n",
    "    for r in range(5):\n",
    "        for c in range(5):\n",
    "            #绘制第一维度中的数据，即0到24层\n",
    "            ax[r,c].imshow(hidden_layer[5 * r + c].reshape(20,20).T,\n",
    "                           cmap='gray_r'\n",
    "                           )\n",
    "            \n",
    "    \n",
    "#     plt.xticks([])\n",
    "    plt.yticks([])\n",
    "    \n",
    "    plt.show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdEAAAHVCAYAAACuUsGNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJztvWvMXNV5930NZ2Mbn7DB4AMY22DMGZMA4ZCmhJCUpFGImjfth0dtpERR1SZV1GPUSvlQ9Uu/VFEbqVJfPc+H9k3TJEqeqAkJKRBwgBiDDcZgbGxjbGODzfl8nPdDkt3/+s09a89es2fuwf3/JIu97jWz99rXOjHXf11rdbrdbhhjjDGmOUdNdwGMMcaY9yqeRI0xxphCPIkaY4wxhXgSNcYYYwrxJGqMMcYU4knUGGOMKcSTqDHGGFOIJ1FjjDGmEE+ixhhjTCHHNPnw8ccf3505c+aoytIKnU5n4M+Ocrem55577nC32104zD1mzZrVnT9//pR5LHuT9x4l01WuvXv3Dm3vGTNmdGfPnt1WkY5oDh06NLS9IyJmzpzZt42blH379rUypixYsGDKvKZ9Vfs6v1s3DkziTnks4549ewayd6NJdObMmfGRj3ykSucMMYzRmnyXnx1mEh2mjOSb3/zmnoFv1of58+fHV77ylSnz2pys6r7bxC7DlGuYNvPlL395aHvPnj07brrppr7lyTGp/1MzKr7xjW8Mbe+IX7bxP/mTP6nSaremNh1mYNbvHnXUUX3zBskvfW4dX/nKV4a2+YIFC+KrX/3qlHmeRNMyfv7znx/I3nbnGmOMMYV4EjXGGGMKaeTOjej/M7zOFZDLf+ONN7KfPf7445O0ulNeffXVJO+dd95J0u+++27fex999NFJ3rHHHtv3OZNGU9eLfp7vdcwxaTOgzfTzJ5xwQpL3+uuvD/zct956K8ljXQ3jNm6LUjfsMO5bvovan3XFuslR135zLrbpcr/lnlNXBrUN7cQ0+77yzDPPZJ/Ddqv3OvHEE5O83NgVkW83o5AEOp1OX5d50zoepnxqs2HaVpP+EJGXC0qZ3FnCGGOMmXA8iRpjjDGFNHbnKsOspMt9dtasWUl6zpw5SVrDbPbu3Zvkbd26NUnTVfzaa69V13RNcuk30yeddFJ1/fbbbyd5073arM7eLK+iNonodZGovemeoiuYqNuGdfHiiy8m6ffCCr5B0bLzvZu4tfldbYMREc8991zfMrDfsF/RpclyKJO42jjnoj3uuOOSvLq29NJLL1XXBw8eTPIOHz6cpNmXdJw47bTTkrw6iSg3ho6i/Xe73azbvgn6LnV9l2lta3WyDutZ+0Cd3PHmm29m83PPHRT/EjXGGGMK8SRqjDHGFOJJ1BhjjCmksSY66BLhJhopdR5uLXj66acn6WXLllXXl1xySZJ38cUXJ+lDhw4l6Q0bNlTXO3bsSPK45RvDZ7Sc1Droex8FakP69l955ZUk/cILLyRpfRfWhepB/GxEWh9z585N8ubNm5ekzzzzzCStITC02eLFi5M03+Hll1+urql9jMPeOXIaT0RaPoZM8D1pQ/0ubcYQCup3qsmdeuqp2e9S23v++eera+rkuZCQYek3NtSNIbS5pvmubC/sH08++WR1vWXLliRv//79SZrbFGo75RjCsYztuImuOGrqtMjczk1sp6wbtVFEuh7llFNOSfJowxkzZiRp1ag5d7CeqWfnQvKsiRpjjDFjxpOoMcYYU4gnUWOMMaaQobb9y+mc9KfTR65+bmoMixYtStInn3xyktbPM2/58uVJes+edCN+9eNfdtllSR41ImqFGhNJ33ouDrMtcvGDTz/9dDatUGMm1CrVZowTpf2pnWks6IEDB5I8aoHUVfTerItcTOM4oH7C9qBaDN+bsZ2M31SNSHXKiIjdu3cnadbHNddcU13TvtSHGKerz8ppd23Tb0yp051zW37S5uyf1NJ0zPnsZz+b5LGNsxzPPvtsdc0YYK7JoDao5aCO23RLu2HJ6foRvWOOjofUfqn78120v2zfvj3JowbK+UFtSj2V6wCYr22c7ad0DPcvUWOMMaYQT6LGGGNMIZ5EjTHGmEJaixNtqqGoDsTYQ+oKjNO67777qmv6takv0c+t2sfKlSuzZaY2q+V4+OGHk7xRxC1yn0uFcXzU5Kj5qE3PO++8JK9uf1aNqaMOTg2CcYsPPvhg389SAz3nnHOStGpE999/f5I3Kg16UM2fNqPeqHbQGMSIXr16yZIlSXr16tXVNfdvZmwztSiNsdu1a1f2uew7qltRB5yOo9A4DlAfpr6ofbDOxtTZzj333Or6oosuSvLYTu+9994krVoiYxypS+vYFdH7jgr39h41LAvHGPY5Xf/AslITVd04IuLHP/5xdc2+zXGCsf8XXnhhdc16zq2riEjXENQd4zgo/iVqjDHGFOJJ1BhjjClkqBAXhUuY6QrjsmV1u9I1xm3S6ArQZf50I/Dn+wUXXJCk1a2zcOHCJI+uF7rSNPSDLjqWuQ14Cn1u278614+6RvlZho/QjaMukzVr1iR5dG3xaDp9LsON6PpiuMaqVauqa7qJ6dZsi0GPhqLri64hvQ/DfnhEGUOO1J3F9svwI5b3iSeeqK43btyY5NH9WbeFozLKEBdFxxe6D+skE83nZ9k/6c5V1/fNN9+c5LGvLV26NElr26QN2S7Y19Su4wgr4pii9ubYzmMOOW5o+2HfpR1ow0svvbS65nsyLEW3eWW5eOQdw7YYNqRjG/uwt/0zxhhjxownUWOMMaYQT6LGGGNMIY01UUV9yPR5c6kx81VTZHhAnRagWiU1B/q1qaXps7jsndoV06oJ0Pc+ipALhrjoNUMb6N+nVnbGGWdU19QrqNFRR9Al5Qw3+vd///ckTS1KwwXuvvvuJI9L29lm1P7j3IZuKtiuqMVQx9EwIWq/PIaL31XN7Td/8zeTPD3qLCLinnvuSdKPPfZYdf3UU08leVwvwHLpOgVqXOPY1jIiDRehnbgtHfuvbgdH/Y7bgfLdf/KTn1TXXBNw/vnnJ2mu4dD1HuyHDBfjUYHaxmnzUcAxRW1K+9YdM6b2Zn/kGML6UJtSq68bl3Xu0HEtIr/9ZkSvzqtYEzXGGGPGjCdRY4wxphBPosYYY0whQ237l4MaisavRaR+cPqtuR3fjTfemKQ1potxijwC6Y477kjSa9eura4feeSRJI/bilFHUS2R/vNxbIumtqcGQd1AtzGLSLUD6sTU9xhDp5//+te/nuRRd6PNHn300eqaW3StWLEiSVPP0O8yvm5c29D1ex7tTQ1IdTLmsQ8x5lj1Oo2Vjehts4x/1Dhd9kFuiUmdSt+R+te4NeiIXo2O8bSXX355kta2yPphzPcDDzyQpPV9afNPfvKTSZpx6zrGsEz8LMcYrWuuCRiFDp2LPec4wPbC9RDaJ/lZ9mXq/to/qDlTu+faCf0ujxWkDdmOuTdAG/iXqDHGGFOIJ1FjjDGmEE+ixhhjTCGt7Z3L+Bv6/nk0jt6HR44xFo5+bNUZ+Fz69ekT37x585RlmOo59OOr/50xXaPQ6HL6BfUiahKM8VLtjHVB+1PD45FZObgfq8Z8/cZv/EaSxyOPWO+qwzF2b9yaKDXBurUBqmcxdlDj6yJ69V7V7/ieqhNHRGzbti1Jq35NbYkaKGO3tZzsV4OuhSihX12yPVCrZzvmUXpK3X7R1113XXVdty8y46xVM+UaAeqIHBfVzizjKG3+a3J797LfU2/Udsoxg23vyiuvTNJqf+rVtDf7vo7DHPcY/8u2lYuLLcW/RI0xxphCPIkaY4wxhQwV4qI/lel6qEurm5VbN/G4LS5X15AYbg1Gdy7dCupOyblWInrdCrrVHl0BLOOooZuCR83RXaVhFJdddlmSd9VVVyXpH/7wh0n6O9/5TnXNkAq60Ghvdbfo9oERvVug8Z3UnUT7jmMbuibtm2XXtkObUWKgK0zb1tatW5O8hx9+OEkzPEZdkdwakmWme07bEPsRXb9tom5ELSPdm7QT3eLa1mjjui0PNdyKNuXxc3Qhqgud7nZuD8pxQ6UpSiG5LepK4bZ/uRCX3PgXkYYccRxmf6U0ljsqkKFwdONru2CIC8vBetZxg3221H3uX6LGGGNMIZ5EjTHGmEI8iRpjjDGFtOZ0p77C8AXy0EMPVde6jV9E7zZQ27dvT9L0mSvc9o/bbl199dXVNTUX6hf0kat/nZrcuI9CI9SAmNZl4tTo9uzZk6TXr1+fpDds2DDlfaZ6DtuBaiEMDWCYEO2vz6ImN+5tFgm1LYZFaPgFtSW2FR5Fp+/K9rxz587svfS5PLKrbqtKfSfed5Tb/vWrS7Yt9ldqk/r5ui30mK/rI+qOQtOxKyINm+NYxnZLXTcXJsL6GwW5fkSb5ba+ZN2wvbBvq85JLZhbxDJsSG1GvZrHqrGedcxpErqXw79EjTHGmEI8iRpjjDGFeBI1xhhjCmlNE2VME2MAVYuMSP3gd999d5J33333JWlqeDwOSrnpppv6PicijQGjJnfNNdckafrXNc34pHFs+6f6Sp0mR61A31U1zojercl4L9WVqWteccUVSXrZsmVJWu3PrQipebGeNda1rS26mpCrU+axragOSm2L7Y5HfOn2k9RTH3vssexzVS+qi/sj2r6oaY3rKDS1K/VErm+gVqnaGbVkxpQyflDjSNmmGXvI/qH1W6f7cytDjc3lfUdBbkxhfySME9V3Yywn49bZB1SPpMbM7Rw5bqjuzLU3tDfXG+g8xf5QOob7l6gxxhhTiCdRY4wxphBPosYYY0whrR2FRs2EMU/cr/UTn/hEdf3tb387yfu3f/u3JM14sLPOOqu6pu5DfYm+efXFM4Zo1apVSZoajO5jSo1uHHGLWl76/llW7p27bt26Ke8zVZp6quoI3AOTsVaXXnppklaNiGWkrtzEpuPQ6PppRxG9eoru5xyRau+MN6WNeBSUasO0AfUhxjSqNkXdj5oX9yzVvjRdmqhCu1Er4zoMjVVkbDljDdnGtT7Zt3gv7q2r4wLHEGqFHMtUK2RfGvdxf3XaPdue6vHc75zjP9up9hfWK9excNzQemYej7PkO2g56uKmB8W/RI0xxphCPIkaY4wxhQwV4qLuFm4RRdcc89WF9YEPfCDJ49ZmPF5I3Qg8zuxnP/tZkr722muT9JIlS6prHo9E9yKXyevWVXyfUZxCz23/1H1FNzZdQbSDumFXr16d5NEtlnOR79ixI/tZugx1+TpdnnSfLF26tO+9piPERWH91m3zqK4vvifdtwxt0KPoFi1alOSx7uhWVpccj01jeAVdi+qypat3lEehKWpnupDZxm+55ZYkvXv37ur68ccfT/K2bNmSpOnq1j7AUA6G63HbUXUNs8x1dtPPj+N4P5I7kqzu+EGVjLZt25bkcVzgd7WuOGawTdMlrnXFMZv1mnuHttzl/iVqjDHGFOJJ1BhjjCnEk6gxxhhTSGvb/nF5NLdw0+OCItLjt6iVcXk0l0+rtnPw4MEkj0uaqSmpHsWtwBgWQn+7LsWm1sEwkVGgNuay8OXLlydp6gp33XVXdc3l/VxSTh1Bl5EzZIhbov34xz/ue2/aiPVMbVD1mnEv9yesb7azs88+O0mrNrN3794kjzo+26Fu7bd27dokb8WKFUmabVb7A4+nohZL7Y861nRDm1OXZqiJbgfHo7fYpqnhaZ9g39q3b1+SprasOjV1Zq43YB/IjSmjWGeRg2XlmM7+qWtKNMwnoteG/K6Oyzz6jO2WW8bqVn7UWlnPdUdEtoF/iRpjjDGFeBI1xhhjCvEkaowxxhTSWBPtp1PRF81YKm7PpJod4zWvvPLKbFo1VGob9K9TG1Qdhdt5UefKxSZSPxi3ZkfdgFol9UaNeeXRZ9ySi/dWbZiaDuuZsXy6DRdjuKjBUd9TvYPxd+O2N59PPZHau74b64ZaHu2v/YE2ovatW2BGRGzatKm65jFpLAfb+3Rs7ZejLtacqG0YJ8q1E7mju6ibUc/j1qJqx7qYa9a1psetgRLWP8dObp2o6yH4XmxrXHeh78rjyjh2cX2N6s4cjzgPcc3GKGJxJ6vXGGOMMe8hPIkaY4wxhXgSNcYYYwppbe9c+tOpZ1CTUN80tRv6wDdu3Jikdf9b+s/px9fYMZaZ/nH69Rm3Rb1DmW6NjmnqOLny8b1oU9WIqHlS86HuqTas22+YbUb1jenW6+rKythP6s65e7HNqs5DjZ+aDzVS1bNZRtZzrj1PAiw/7ca1FBrXyLZ24403Jmlq2grjdnm81iWXXJKkVdPjZ/kOjD3PaXTj0Ej1GXXP47isYzr3vWaMKfuvxizz+DjuG6Bx0xFpG+d9WXfU/XPx/KX29i9RY4wxphBPosYYY0whjf056hbUn790H9IVmnM38uc7jyCjS1aXT9MVzG236G7UNMtIl+d0u287nU5it362j+h1CRJ1s9KFRPvTNakuRG7nRfeIHrkWkbqCWUban3U13Uv+m5ALJ6FLjzbkdnxqB4YbsU3SPajtJeeyfC/A9sA+l5NbzjvvvOx3mxyFRimK9aXyB9s4x0W6OXNbW47jeEWFUgHH0txxgGyH3HaR479uhZmT3CJ65wO1/7x587LfpTs3N4aWjun+JWqMMcYU4knUGGOMKcSTqDHGGFNIa9v+1UGdQdPUNuq0Sk1T26DvPef3pp7H5zQJqxiVfjeojes+p/oSdTWWnfZXXYcaA79Lm2q6LqTivaSBsqx8N9WX+FlqTXWhKEqTPlf32Umxd78xpa787J/U9JTcdnuEYxV1Tn6XIWFKbuyKSPsa34f9cNSwPeTWk0SkoT11Y2dujOE2i2z/GtJIWOd1R6ONos37l6gxxhhTiCdRY4wxphBPosYYY0whnSYaS6fTORQReyLi5Ig4XPPxcTNpZVre7XYX1n+sP7Z3I450e0dMVrmGtnfExNt80sp0pLfxSSvTQPZuNIlWX+p0Nna73XVFxRoRk1imtpjEd5vEMrXFpL7bpJarDSbx3SaxTG0xie82iWUaBLtzjTHGmEI8iRpjjDGFlE6i/9xqKdphEsvUFpP4bpNYpraY1Heb1HK1wSS+2ySWqS0m8d0msUy1FGmixhhjjLE71xhjjCnGk6gxxhhTiCdRY4wxphBPosYYY0whnkSNMcaYQjyJGmOMMYV4EjXGGGMK8SRqjDHGFOJJ1BhjjCnkmCYfnjFjRnf27NmjKssRxaFDhw4Pe2zRiSee2J07d25bRTqiOXDgwND2njFjRnfOnDltFemI5qmnnhra3hERM2fO7M6bN2/KPO6m1ul0svdqeKzjwJ9tWo5RsX///qFtTnvru/xP372O9bpv376B7N1oEp09e3Z8+tOfrtJHHfXfP2Tffffd7Hf1s4N8flB4XzLMc4Yp8ze+8Y09xQ/+FXPnzo0vfOELVTrXyOs6tpadnx1mcOJ3aSP9bt1zmN/ku1/72teGtvecOXPi937v96bMa9rOcp9v2leafLcJufZd975///d/P7S9IyLmzZsXf/iHfzhl3nRNonVterom0b/6q78a2ubz5s2LL33pS1V6EifRd955J0mzLY7K/rzvn/7pnw5kb7tzjTHGmEI8iRpjjDGFNHLnkiaupdxnjzkmLUadK6kuX6GLQstR577I5Y/KPd2kDDlyLiiWvYk9Z86cmaTrXG5vvvlmdf3WW29l7/32229n7/VeQu1/9NFHJ3m0N91Xmj7hhBOyz6H99Vm8L+1LmrSDNtF6btKPcm2P756TGSJS29TZie1S64g2ZN1Pir46CDl5hfnHHnts9rM6DjD/+OOPT/Lq+ovOF3wOxxjWe06yKB1r/UvUGGOMKcSTqDHGGFPIUO7cYTjuuOOqa7qs+PM+55phSMKrr76apF977bW+937jjTeSvJdffjn73EmGrojcijbmnXjiiUla6ybil6uEf838+fOzn+W9X3nller62WefTfKee+65JP36668naa276XB7DePe1Pqge5AuJuary5whZewbrDu1k9o+IuLQoUNJmvbW747T3v1WYTd1fWp/pY1feumlvp+NiDh8+HB1/fTTTyd5bOMzZsxI0to/WD8nn3xyks5JV3zfUayW7XQ6fd3nde7bHDn3eESv/TW9dOnSJG/JkiVJeuHCNMpEx6Dnn38+ydu3b1+SfuaZZ5K09gm6oEvxL1FjjDGmEE+ixhhjTCGeRI0xxphCGmuig+5S1CT0gfrE/v37kzR917NmzZryeqrnUG/VfC6HpkbEtOod1EUmDZZPdR3am1sLLlu2LEmvWbOmuj7ppJOSPGp21KBVg7jrrruSvN27dyfpF154IUlrXVH7a0vPIP3ad92ye+brd2kT6vh8F62fHTt2JHm8F9s7NTmF6wWob2s5qN0xPQ5oY4ZJ8H20r+faf0SvZnf//fdX19TqzzrrrCR9+umnJ2mtr7rxh++QW6swjh2EcjpzXSgWx0eF7YVa5YMPPlhdsx5POeWUJL169eokvWLFiup68eLFSR51f47x+g7ss0wPin+JGmOMMYV4EjXGGGMK8SRqjDHGFNKa0EF/ObUbxmBq/A41CPriL7zwwiStmh1jihg3RN12586d1fXjjz+e5LEc1E1U/1u0aFGSR61w1NSdNEGtMqfbUAvg1n6qK1Cjo42oAWk9q+4UEbFly5YkzbhdtSljyeq2wyuln85fd5JETvOnfXn0F22oNqM+dOqppyZp6tmqCWlbj4jYsyc9lIL6nGq1XGswCSd81G3xpjGZ7J/Uf6nDqXZPbf7MM8/MPvehhx6qrpcvX57ksa6p1epzGdfLMrZBt9tN7NjkpCTaX8vLsvKzHFO0XXPc5foH5t98883VNfsl43LZX1SrZZmYHhT/EjXGGGMK8SRqjDHGFOJJ1BhjjCmksSba7ygZxgxRA2X8jsakUdehzkn9S2OOmEcf+cMPP5ykdc9G+tq5t2tun0tqvqX+9Dr67SVKnY06Qu4YLOoy1AEfe+yxJL1+/frq+sUXX0zyqB8RfS41Ot6L+pG2IbYvxlq2hdZxbu9k1j/1RW13zHviiSeSNG144403Vtcf+9jHkrwDBw4kadpw27Zt1TXbBPcgZTlUi6UOyD46KrSNM7aTdmSZtMzUkmmnBQsWJOnzzz+/umY8c+6ouoiISy65ZMr7RPTuw0ttX++ldReRj8MshXvnNoFjq/ZP1g3HcNpMY29Zjxx3OS5/73vfq66pT3OvXK7h0HaxatWqJC8XY53Dv0SNMcaYQjyJGmOMMYW0FuJStwSeW2XpT/jTTjstyWO4CN1O6n5kSAtdMfy5r/emG4EhFtxSSl0SdG0Mc2xWjn7L0enOpXuRLlq1A0Mq6GphKIS6zOm2PnjwYJJ+8sknox90odG9yDaix1PRXcR6bYt+IS60EV2w3Kpy+/bt1TVdsHQ1st2pnTZs2JAt365du/qWk+2b7ir2HXXz6dZqEaNr3yR3FBrHBaYZ3qDQbnSr/vznP6+u//M//zPJu+KKK7Lf1TKzTGzzrBN12e7duzfJoyQ2CrTsHFPYxxjGomF0/C4lo1wYC6U+yhCbN29O0uqap0uW7nO6xHVsa6uN+5eoMcYYU4gnUWOMMaYQT6LGGGNMIa1polyWzCXO3A5Lt4yq2zKQPnO9V91WfTxmStMXXHBBkkeffy5NjSV3LNwoqDu2jfWhNqQ9ud0Ybfr+97+/uqZeweXy1IRUV77ooouSPNYNdRXVXLjMfRz2Vk2OIRLUfhlqtXLlyimvI3rfm+1MNSBu38jv8t4aYsGQIupU3BJN+w63/RvVUWjchk51KYa4cOs+2kbtyPo699xzkzTHCQ1T+a//+q8kj1r9unXrkvRTTz1VXdPmDMViu9Fysh/mQqzaQu1dF/rCNq4aNLVe6qlc/6B9/fbbb0/yqCOfc845SZp1qWgIY0Tvmg3Vmev61qD4l6gxxhhTiCdRY4wxphBPosYYY0whQwkdqkvRn0wNlPGFurUc4+SofzGtcaVnn312kse4oXvuuSdJq86iW09F9Mbz0Z+umgx1xVHF0fXTKaiBMqZOdZqI9Ngx6gbUK9asWZOkVZOgrqbxkBG9Mb+//du/XV0z1pMaENuQxjXyu6VblzVBbUp7UwO6/PLLk7RqX2z7F198cZLmvXWrMsaf0v6f+cxnkrRqcixjXWyzPovbp41Kg+Y2dP300Yje9sH2oxoXbc51FmvXrk3SOm5ofHJEr+7M/qFH/PF4vwcffDBJ59ZsUKPjGoE2yB2FRnuzPLn4ZsbUc4zheKlbtzLWmVtd3nTTTUn61ltvra6p1VO33bRpU5LWuYTvW3rcn3+JGmOMMYV4EjXGGGMK8SRqjDHGFNJa8Bf35aRewfgo1V+oCVGTOOOMM5K0anhXXXVVksf4JO6DqXFC1Byo+1CDUf8735e6bltomVQ7Ylnpz2eMWW6vWepH1DVVO6DmTN1448aNSfoHP/hBdc09R7nXKWNM9bnUPkYVQ6fPVJtSX2SM5XnnnZek1U6Mb7zyyiuTNPXeBx54oLpmDB3jr1lXjz/+eHXNWE/CNqMaGGOFRwU1Om3j1A8Z+5nb65rxzNxTlfWl+9ZSd6aNt27dmqT12Dtqf9TuecyganiMfxzHfsVqe44p1ERz36V9eS/GQt98883VNfVT1g3rUvsexwWOcxynNQa1rf23/UvUGGOMKcSTqDHGGFNIY3euuhhyy965NJ/Ha6lblW4aulO4xFxda3Q5qDsronfLrn/913+trtUNE9HrqmQIjLoG6EYYlTtX3UH9riN6XeKsG3WB031Idy5DXtRNxlCBn/3sZ0maR9HpcUMf//jHIwfL3M+1GjGeEBd9BpfOU2K48MILk7S2FbpV6YrWY7giIn7xi19U1wzvohv5tttuS9J6RBvDDvgOrEt933EefdavLtlOOU7Q5axjTt0RZAxnUxc6pSfKOrSNyk+UovhuHDc0bKROXmqLfqEcTbcy1fbE4yrpiuexe9pOedQcQ2m4laLWx6JFi5I82nv37t1JWstJe7PMg+JfosYYY0whnkSNMcaYQjyJGmOMMYU01kT7+cnpZ6duwKXGql3yuCBui0Z9Q33iXDL+yCNHVzX6AAAgAElEQVSPJGn6xFWL5daEDEWgzqk+cy67HpeG9Gt4TBTrhfZWTYiaHL9LfTWnfWhoQESvJnTppZdOWYaIXq1Q6yZidDpzDrWFaiZ14U/U5vXzbKO33HJLkr7jjjuStIYJUa9etmxZkqauqeECrAtqpAwpUs2Rtuc6hbZoEuLCd922bVuSVo2UYSkf+tCHkjRDh3SbTNYlQ7MYkpHTzhnql+sDfL+6EKVSBtWgqWPye/quDL2ilk89W8dPjmVs8/PmzUvSelSdbpHJMkX02lT7BD/L9KD4l6gxxhhTiCdRY4wxphBPosYYY0whrW37V3fMFTU61XY+9alPJXncyo/3Vu2SGui3vvWtJM14MNUkNIYxoncbOsYR5fSaccR0qe5K3YBlpQahunLdMXXUM77//e9X1+vXr0/yuJ3X9ddfn6QvueSS6pptgDGP1A61DVFzLz22qI5+sal1WtdPfvKTJK3aC9sgbcjtx/TezNNjtyJ6jwK89tprq2tqWjn78vOj2laRNIkTpVbPMmo75j0Za8i4ddVBP/GJTyR57NuPPvpoktbjABm7movFZZrxkaMYU3JHz9G+tBHb8dVXX11dv//970/yeC8ePbd69erqmu/N4yu57kLHXo03jUjjfSN6+62OQdRiS/EvUWOMMaYQT6LGGGNMIZ5EjTHGmEKG0kRVP2KMn8ZdRfTGrKmW84EPfCDJ41E49K/rHpm33npr9rn0iasOSs2BPnLGPml8GLWOUdHv+DPuwUl9Mbfn6gUXXJDkUVtiLNszzzxTXTO28MMf/nCSpkanGgRjD6n3MRZR2xTLOCpNVG2s19RzqUEzXk33ZWasct27qH5Nm7A9s660bzDekRooj6/S+mCM4jj2Ko5IxxTGXnMdAvfFVq2SejCPL+OYov3je9/7XpJHTY7rMLQvcr9tlpmaqdY9x59RxJ4zLldhu+SYfd999yVp3Tf7mmuuSfIY68/1J9qeuN8543A5t2g7pX05/pM1a9ZU123Z279EjTHGmEI8iRpjjDGFDHUUmv4k53J0/jTmNnm6fZxu4xSRug8jepcx61FRP/3pT5M8ukt4WrwutWZ4Bl12dG/oVm/87jjILUfne/I4Mw15oVuP22rRZatuwQ9+8IN98yJ6605tyrqhm4ZpddWM6yi0flv90fXM9k7UJUXXOvsCtxBU1xe3ojznnHOSNI/tUpmBz6ELms/Vd2T/Hfe2llNBVzbbnqYPHjyY5HGrxdzRXPwuP0uXrdqV2w2yjHSR6hhDG4/Dhd7Encz2o3bZsGFDkkepjGF02o45vnN7SoZ16RjDMtL+LLPWh925xhhjzDTjSdQYY4wpxJOoMcYYU0hrR6FRr+DSfG6pl9u678EHH0zS1P80VII6GpdSU0NSDaBOZ+PWedNNv3CXiN5wBd1WKyI9zokaHXUaamka7sDn1oWpqO5A7YP1ynzVi8YVYqHP1NAFvif1XdpQdWeGoTA0g+FSqlHzs6y7ffv2JelFixZV17QZ9Wrqugybmg60T7KtUbNi+1GNl8cnMgTswgsvTNJXXnlldU0NlNvdUbPTz7PM7A/U7PS7DNcYFf2OnqNGTpux7LregVuJUm9k/9DxiqFvDAnTbRUj0vUo7B916wC0PtraStS/RI0xxphCPIkaY4wxhXgSNcYYYwoZKk5U/f/0p1OPYWzinDlzqmtqENT3GL/ZrzxTwXhOjYGkbsLnUBNVzWJccXT9ji3i86jR7dq1K0nntGDqfbz3qlWrqmtqG/xurq6ofbBN5I6Xmw69Tp9fp5dwmzzdMpLfpfbLuETV+anHsa/okYIRqSbEPlgXu6fUfbctctvQ1cW1sj1p29M1ABERX/ziF5P0ypUrk7S+H9doUBukFqvrQRjXy1hzbmmnjEv3H/T5fBced6af53tt3LgxSd9+++1JWuucR6Gprh/R2w50DOd6A8KxjBp1G/iXqDHGGFOIJ1FjjDGmEE+ixhhjTCGtHYVGPYt+bMak6fE31FuY5ndVg+CxXjyKS7VX3ptaDDXQnA40Ko2I9NOLqBHS/jyai9qlwrqirqnP4l6tvC81aNU3qJswPpLvOt1xi/3i6ZgX0auv63tT16zT9tSmusd0RG8cHDUe1TlZjzzaj/1KtdhRHTVHOp1OXy2Q7ZL6F7U0rQPuHV2nr+o6DB7jxXbKPnDo0KHqmm2c78b+om18XHvn9rsv9VvG/tMuGi9Lm1Bzpl10LwBq82yn3H9Y64Pf5dqEXFx7W2O4f4kaY4wxhXgSNcYYYwppbds/ut7o3qKrIHeUFN0NdCuoq4nPpTuRLiD9OZ87MX3S4TJ7ut+Yr/bmZ+lGYt2o64VhKXSL0W2mLvI6d23dNm/KqNyN/UK42DYYhsV36XefiF43K7dTU5ty20p1HUb0ui213zVxG5MmdTEMDHFp4sLk1nJKnQud45Gm2f7Z5jlu5O7LMrIcuXcfh2SU2/YvV9aIdKzlcYp0tXM8UtmNRy/S9ct2mrMLZS0+dxT4l6gxxhhTiCdRY4wxphBPosYYY0whI4shoH+fmpGmqaPVHVGm+gy1GvrP9+7d2/fe9J9TA5gE+ml/ddoRdWTV4ersS81B66puCXnuXnVL+NkO+m15ONV3R0FOB6T2lduykGVlyATbrN6r7ohB6nU8Km3QMkaMTvfMwRCXnEZIO/FoN333uvaSC1li3eba5VT5/e7blHG08Zy92V645afq83VjSM6GXF9A+F1Ns03UlWMUOrN/iRpjjDGFeBI1xhhjCvEkaowxxhTSaeKz73Q6hyJiT0ScHBGHaz4+biatTMu73e7C+o/1x/ZuxJFu74jJKtfQ9o6YeJtPWpmO9DY+aWUayN6NJtHqS53Oxm63u66oWCNiEsvUFpP4bpNYpraY1Heb1HK1wSS+2ySWqS0m8d0msUyDYHeuMcYYU4gnUWOMMaaQ0kn0n1stRTtMYpnaYhLfbRLL1BaT+m6TWq42mMR3m8QytcUkvtsklqmWIk3UGGOMMXbnGmOMMcV4EjXGGGMK8SRqjDHGFOJJ1BhjjCnEk6gxxhhTiCdRY4wxphBPosYYY0whnkSNMcaYQjyJGmOMMYUc0+TDM2bM6J500kmjKssRxdNPP3142GOLZsyY0Z09e3ZbRTqiOXTo0ND2njlzZnfevHltFemIZv/+/UPbO6J3TMntoNbpdLL30u/WfXZSaFLmNsaU448/vjtz5sxhbvE/hueee24gezeaRE866aT43d/93fJSjQB2ujY7WpN787P/8A//sCd78wGYPXt2/M7v/M6UeSxLXVk1/913303yjjqqPYdEExuRYcr1j//4j0Pbe968efHHf/zHVVrfpa7sdfXRhNxz62w0ronkz//8z4e2d8Qvx5TPfOYzVXoSJtFh+lZTmpT561//+tA2nzlzZlx33XUDfbbhWdMTea/cvevu8+1vf3sge9uda4wxxhTiSdQYY4wppJE7twl1Lg9NN3Un6r2POSZ9haOPPjpJv/32232fS9dYE5fddJ9+M8zz+d5NXIR1333nnXf6fvf444/PPodptTfv26YLuh/DuBYVtknel++maX63rhzHHXfcQPdtyntBY9Qy5vr9VOm33nqrb17du+sYxPrKtelB7j1J5Mb0uvEoZ5e6uiG5/GG+W4p/iRpjjDGFeBI1xhhjChmZO7fJz3u6+U488cQkTZeI5vOzr7zySpKmC0LdNs8991yS9/rrryfpNleUjZucy/DNN99M8l577bUk/cYbbyRpdVfRfU53Lu2g9zr22GOTPC61ZzvQe43DfduEJm4julFpb6a1jdIms2bNStIzZsxI0qecckp1TZfm888/n6RZz/r5NlegltJ0lazajX35hRdeSNKvvvpqktZ2/PLLLyd56iKPiJgzZ07f5+p1RMSSJUuSNEMEtV0P425vQr+6bOpq1jGcfZtp2kXtz75NO3CM0TZPNzHrle+qz2prTJmskckYY4x5D+FJ1BhjjCnEk6gxxhhTyMg0UUJdgX5uhX7uuXPnJumTTz65uqavnZooNbzDhw9X10888USSV6eFLFiwoLqmFjXdIS91oSYvvfRSdU29iND+ixYtqq7POuusJI91s2vXriStNqUGV6dBqyZDHZd10xb9wpjq9LlcCA4/S53z2WefTdJaV3XrBaixaXtn3VCn0udE9GqmSq6/DkO32+1r5zqbs69rGdnWeC/aVT/P9s+tIE844YQkre2Y7ZIaNp9L3Xo6aRrao22eNuF3qVVq/9BxNaK3XlkfupaF6ypYDj5X79VW2Jx/iRpjjDGFeBI1xhhjCvEkaowxxhTSWBPtp1/Ubd3EWDjVyqjVUEdYvXp1kj7ttNOq66VLlyZ51DmZVt1k//79Sd5TTz2VpKmn6rFkLOO4Yrx+Td2WhdSEXnzxxer6wIEDSd7ixYuz91aNYseOHUneihUrBr4X9Tzan3Wl5aQGPaojywbVtuu2d1Pti+9N7ZHf1b7B+EbagbqytlmNGY3o1UgXLkxPedJ3Yn+t09FL6XQ6yftre6Fd6uKMFV03EdHbl6m7aR2x7+RiOyPSdrxs2bIkj+2Emt2hQ4eqa2rU4yC3VSJ1TdpQ0yx7Lg43Il1nUbcdaC7emRo02zz7s5aLeexbg+JfosYYY0whnkSNMcaYQjyJGmOMMYU01kT7xXFRE6TmwDg/zafmQF+86gYRqQ/9nHPOSfKoSdxxxx1J+oEHHqiuqalQX6WGpGm+7zj2zs3pRYyXJervp56r+kRExN69e5O0anQ7d+7smxcRccMNNyRp1aZ+/vOfZ8tM/W/Pnv8+WJ57kM6fPz/GSV0MHbUZ1XGosVELZmztli1bquvdu3cneay7VatWJWnV3GizSy+9NEkzPi93pNco46D77TPMMrCtsQ5yOpuuCYjo1SbVrvwstbJTTz01Sau2TN2W6yx4Lx1TqDuPKoa031hF3ZI25Hio4yd1furtnA+efPLJ6prjAnVN2kHTbCN8N2r7WrdsA0wPin+JGmOMMYV4EjXGGGMKGSrERa/5k7tuCz2Fbgxug5Y78mj9+vVJ3m233Zak6WbQY4y4hR3DNeh60XJwCTfd1aMgd4wPXS90laqrpi7UQV0tEambg+9J+9Jto/emC5xuMbaRzZs3V9fPPPNMkkdXZVsMuu0f3XZs/5pPG7G90yWloVRsg5Qv2M+03llXrFeWQ/tG3ZF3o0KfyzbO9kEZQsNUKA3wu+wvGl5F+eiSSy5J0nT7aXgHw5nWrFkTObSdcNwbVdictmu1Mdsh02zzWleUBmgjygEqQ5x++unZ5+aOaKs7Co3yh7pz2f752UHxL1FjjDGmEE+ixhhjTCGeRI0xxphChjoKTXUSalZcWkz9UXUi6jrU6KjPqFZ2//33J3lcLk3t7Oyzz66u6cdnmu+kOgr1pnFpRr+mTjegzTSMiFoktRhqpqr5/OIXv0jyrrjiiiT9kY98JElryBG3BGRIEe2t4R3cqrD02KI6+mn+tGfdGgC1N7egowZEvU77xk9/+tMkT/XSiN5wDC0z9VTqRQzxyoVJjeuoP23HtDnDmriGIdc/aXO2H7XNrbfemuSxrjmmaLt++umnkzxqpHqMV0SqxY5rTOkXpshQw7r2o+sfqIGyLxPVszlmc1y4+uqrk7TWx/Lly5M8jmWch9T+7Ds+Cs0YY4wZM55EjTHGmEI8iRpjjDGFDLXtn+okPAqH/nXGdKk/mtqMxqtFpDpmRKod0E9P7ZU6nJafGii1D8aS6TtR2xjVtn+DbtFFfz5tqvoS34t6Bu3A488U2pAxkZpPPY+aC+O0zj333Oqa7zcqTbSfXsTnUb+inqjveuaZZyZ5tBk1atVXGbPIeqc2rtvMPf74433LFJG3aV0s66jIaa9sH7l4Z9qJ24FyXNCtFzds2JDkMfaZaJnZDh599NEkTR1ONVH2y1xs/Shgf+S4QN3/8OHD1TXb0r59+5I09WzNf+ihh5I8ttN77703SWusNMdH9i3GRusRim1pzv4laowxxhTiSdQYY4wpxJOoMcYYU8hQcaLqB6cfmxw8eDBJ87gbhXreBz/4wSSt2iT1u7q4LP08tT7GFDEeTH3ojGGrO4qslH77t9Kfz/emzqbaC2PZqC1R+1D9b+3atUne+eefn6QffPDBJK0xd3wuj+ainq2aDPWiccPjzKhXnXbaaUma9aHUHUW3bdu26pptkloe26i2Ecbb8R24F6r2O+49O8o46H7rLHL7P0+V1thE9nvqmow7vuuuu6prHgVIOz722GNJWnVrfpc2Zt/SNR3T0cbV3tTBc2tCIlJdk0f2EfYPtSG1Vz6HmqnWHecVrj/gOK310Za9/UvUGGOMKcSTqDHGGFPIUCEues2f5Ax12LNnT5LWn/d07XI7L11KHdG7fFqha5JunQceeKDvfbk8muVSV8yojiki/UJc+HyWnUcwnXHGGdX197///SSPW3TRraouxZ07dyZ5dIuxHeS2hrznnnuSNJfJq3uJrrtRhRQN6j4ndJ9r2ffv35/kMRyMrtM77rijuqY7kKECuSPi6raxZBvKyRWUTdqkX1gL3bkcUzZt2pSkVbKg+5bbw7E+dXzi1pYf/vCHkzTrS92PHFMIxycNK2HYFOu+LQY9zpL1wnFZpZx/+Zd/SfIYXsUQKe0DbKd0565bty5J58K42Ga45WbuyMrSrS39S9QYY4wpxJOoMcYYU4gnUWOMMaaQxpqo+o1Vw6KGwuXD1Ch0my36xPnZ2267LUn/x3/8R3WtGmdExCc+8YkkzS2jVM+g5qBbQkX0hgSoRsbv8rNt0U+jI9QCGGKhGukjjzyS5DHUJxeuw/CjnBYYkbaRuuX+1M1V+xiXBt0P6iUMPeH2Yqp18bvUSGkztZNq2RG9uhX1VYZjKNTjqA+pvs33G+UWdP3aNd+VayFYRm1P1H+1LUX0tnEdj1g/HMtoYz1+i3XNsC72AR03WZdcX9AW/UKK2B7YLqkvnnfeeX2/e/fddydp1pWOT1yDwedQT9V2TJ2ccwe/q+MI12CUhnH5l6gxxhhTiCdRY4wxphBPosYYY0whQ237p1DXrDvWR/3PjAvi8WY8TkiPKqKORs3hQx/6UJJWPz+fmzuSKiKNAaNeMKq4RSUXx0QtYPPmzUladR5uo8g0NVLdWovaJONEaReNa+QxUNQrWM+qX49KHyKDbkFHDZx61qpVq6prHjHFeqRNdRtA1dsienUctjvViLgVJ/Ujao7MHxdqD20TfNdcrGFEuqaBawKoyTHGVNuxHqkW0atL86hG3aaRGmguHjUi1fdyW0W2Sb9xhOMdy86YTI1J/qM/+qMkj2ndVjEi4v7776+uWY/UpKm36jaL7Jd1a0N0XmK/c5yoMcYYM2Y8iRpjjDGFeBI1xhhjChlKE83ttbls2bIkffrppyfpc845p7qmfqp7YEb0ahDqi6fetHLlyiTNPTNVG6FOSx85dTjVp+inH+VRUb9Gy0f9ok4v2rp1a3VNG7FuNm7cmKRVX6VudtFFFyVp7iuqGhDrkZoXNVPVU/ndce+dy7LWaaRbtmyprmlfxtry3qrVs52xvdPeWmbVjqb6LNceaBvmcW3jQstAO9HmjNdUW1FXY5r1pfoxxwEe88W02pl6KuuWY1vufdmn26Jf3+G6Curx3Idc7UAtmPuOc3xU+9MmfG9+V9sxxyMeDUhNVN89t3d0E/xL1BhjjCnEk6gxxhhTyFBHoeWO0eEWYzzZPHeiOl0DTF9wwQXVNX+S081Jl4S6bVhmnpKeOzaHzx13iAufT1cX3Ui6rZmeKh/R6xajvdWFSDcNwyh4VJq6XlgXdLXThurmp1wwKvd5v3qs2/aO7U7vQxds3VZ9aie6Hbk1JetZXYt0z7G987tq41G5EoeB4Wsso4bZsX3s3bs3SXNbQJUOGPrGMC6GvKidWdcarhTRK3Pp50d19FkOHUdy/S+it/3odqwMi2M75XygUMbhd1mXui0gXeC0N8uc29K0dAz3L1FjjDGmEE+ixhhjTCGeRI0xxphChjoKrZ8+GtGrf1GH0+9S76KPnEvxFy9eXF1TM6KuSX+66hl1ZaS+oeWg/3xUmmi/49eo21DfIqofMeSC2x1ymX5O+2Y4DMMqtMx8LvUL6oqqt9K+ozoard/WX3V6EfO13VF/4xaZzFedh22wLgxL783Pcus7ljm35d50wL5ddwycrmFg/2B9nX/++Un6fe97X3XN+uH4w3tpO6bdWH8c6/SdqPGOY52Flpc2I1z/oH2bbYvjAPuVjsNce0J7M9xKy0z70v4cU7RN0b6lbX76e4oxxhjzHsWTqDHGGFOIJ1FjjDGmkKHiRBVqj/Rrc8sx9U3zu9QccnFDTz75ZJLH43r0SKqIVEfhc5pu7TYOSreho1agugI1UN2CMaI3blF1m1tvvTXJo8ZDe1MHVWhPah+q97GNjEqzy+n8CuPTGD+rsc+M/6O2xHhO3V6N8dW0Qy5emeWnfsrvqibH745SnxvU5nXvrjY/77zzkrzrr78+STPWU/vL9u3bkzzGPrPt5bZI5FZ57KfaNkqP4hqG3JhCnTN3NF1dX+aYonHXjMGmHaiFa99j3+GeA7n+0dYY4l+ixhhjTCGeRI0xxphCPIkaY4wxhTTWRFWXyOkk9EXTd507kobaDfVV1eGok1Df030WI1J/Oo/Xom+e8XvjiNvKkdNMWDZqA6pRUEfmcULbtm1L0qoX8TmrV69O0qwP1TNYJtYztcNJiFXsB7Vg6pwK2yD7AuPkVDdjHuuOMdVaV9T82X6Yz2Ojct9tk366P6GNGdeodUItnmsEGMeomh7Xb/C7bMf6XObRphzrRhXvnGPQWH+2Pb6bfreurXGtitqUYzafw/pQrZZ6Kb/L8UjHmLbG88kdpYwxxpgJx5OoMcYYU0hrIS6ELlgugc5t0UUXB5dLaxl0C8CI3p/3DI/RZdnc9o9LurkcfVBX9qjQ59PVWedOUfcJ64IuEG6dqPeiCzbnviWsR7rPc8ebMW9UR3UN6rasC7dQ6DqkeyonddRtfcd7a6gN7Uub1R3vNi4G7Uu0MV3qms8tPHlUV66d1m3pybpXly3z6sLipmNM6fcc2pPHl7Fv6Of5nvwsxyu1vx7TOBW5Iyn5LrltFUlbEoV/iRpjjDGFeBI1xhhjCvEkaowxxhQy1FFoub/TV81t0jRdt90U/dr6eWp01Jd2796dpHPbbFEbpGaU0yzGfWwRn1fn31fthbrBMKEktFETm/G5uXea7nCXuvrle6uNWTfU6Zmv2h77ArXvJUuWJGm10zC6cdP2NQyD3pvjANcsNOmDbE/aP+rKw++qZtd0u8TpGFMGPe6v7mg2TXMc5ndpM23jrFeOT+xbufUH/G6dnp377KD4l6gxxhhTiCdRY4wxphBPosYYY0whnSZaR6fTORQReyLi5Ig4PKpCFTJpZVre7XYXDnMD27sRR7q9IyarXEPbO2LibT5pZTrS2/iklWkgezeaRKsvdTobu93uuqJijYhJLFNbTOK7TWKZ2mJS321Sy9UGk/huk1imtpjEd5vEMg2C3bnGGGNMIZ5EjTHGmEJKJ9F/brUU7TCJZWqLSXy3SSxTW0zqu01qudpgEt9tEsvUFpP4bpNYplqKNFFjjDHG2J1rjDHGFONJ1BhjjCnEk6gxxhhTiCdRY4wxphBPosYYY0whnkSNMcaYQjyJGmOMMYV4EjXGGGMK8SRqjDHGFHJMkw/PnDmzO3/+/KIHcWekTqcz8Gdz5O7TlCZlrPvu/v37Dw97bNGsWbPGYu8m92p6n2G+24S9e/cObe+ZM2d2586d21aRjmiefPLJoe0dMZk2b7PvtEkbNp81a1Z3wYIFA3227r2b9O02x/Rx7bL3xBNPDGTvRpPo/Pnz40tf+lJRgcY1iR599NFJ+t133x343m1Oon/2Z3+2Z+Av92H+/Pnxla98pei7/9Mm0S9/+ctD23vu3LnxxS9+sY3iHPH89V//9dD2jmhm87pxoa329c4777R237rvan5dn23D5gsWLIivfvWrA322yWR21FFH9c2rS/M5dc/VMb2tcW2qe33hC18YyN525xpjjDGFeBI1xhhjCmnkzu12u8XuOX5W71PncqWL9phj/rvYL774YpJ33HHHZcvx9ttvV9fHHnts9jlNGLduUueKaLM8ahfWFRnGTT8p2tN0k3NX1bnJFLrYJtW+o9C4Zs6cmaTZt1999dW+3126dGmSfuONN7LP0vK/9tpr2e++9dZbfe9TV9ejYBjJKueKbjI+6Zgc0WsjpnX8Z71yTM/ZtC17+5eoMcYYU4gnUWOMMaaQRu7cTqcz8M//up/3+jP8hBNOSPKOP/74JM0l2SeeeGJ1feDAgSTvpZde6vsc5jddSj1J7rCmZdHP081HG3F1on6ez6X7im6yGTNmVNes17rnDrOCb7rJyRV8T7qzXn/99eo658qK6HUXaj5dW3PmzEnSbAfTRT+34DB1zHbJOqBdlyxZUl1feumlSR7DzGjzk046qbresydd0Ll58+YkfejQoST98ssvV9d1UtQoaOLCzI2HdfbNuWx536eeeipJ6xgSETFr1qzqWueCiIg333wzSeekQFLaHyajFxljjDHvQTyJGmOMMYV4EjXGGGMKaaSJktyOFdQ5Z8+enaQXLVr034WAn5o+cPq9VYPgd6k5PP3000n62Wefra7379+f5FFDOuWUU5K0arPDhMOMAuoK1N1UK+B7Ur/gvfS71D5Yr6eddlqSVn2Pz6UGRD0jp5vUhdqMGtqXOplqcgzDeuWVV5L0qaeemqRfeOGF6lrtN9VzDh8+nKRPPvnk6pp1wzab04tYN6PUoAfVQev0O20v1NXqvnvw4MHqetOmTUke64d2/ehHPzplGSJ664ttnO1oHAxqb+qYLKuuf8i1fz4zItWg68Z7ziW6roV5dc/VNvVsPsIAACAASURBVM45q3RM8S9RY4wxphBPosYYY0whnkSNMcaYQoba9k9hDCB1hLPPPrtvPvUL6kCMvdq1a1d1ndOEInrjRtUPrrpsRK9W9cwzzyRp9fsvXrw4yRuFRprbZrFOk3vuueeStOpy1ByoK8+bNy9Ja11Rg6CN+FyNseNzWM88EkvrSnXCiPy2baOA7Z72fv7555O06u1PPPFEkkf7XnDBBUn6+uuvr64Zo8i4aOr6Dz/8cHX95JNPJnn79u1L0tQ9dQ0AtW2NzWubfrocNSrqiRxzdBzZu3dvksdxgPq8tmu2y9tuuy1Jc62E1u9jjz2W5FFXZMy7jiPslyzjKNA+Rj2X9tb1JBFp22NdMSaZ7Um3ZWR/oH2pmT7yyCPVNfshy8ExXd+X41EuhjSHf4kaY4wxhXgSNcYYYwrxJGqMMcYUMtTeuaoT1cUL0keu2sFDDz2U5NXFt6nuSd2AcULU2VTf01iliIjt27cn6W3btiVp1Zh4X5Z51FC30Ti3iF79S7WZlStXJnm5PWsj0v09GfOo+nREr/63du3a6nrFihVJHvVVahIaD8z3ZT2PArUDta26vVnPPffc6prHcl1yySVJ+vTTT0/SZ511VnXNtQTkzjvvTNKqH1FL2rp1a5LmegJ937rYvTbR5/a7jqg/zkx1UOrBdfqitj22S/YPxp5/73vfq64ff/zxJI9rQ7RdRKR2ZRlHqUP/Gn23Op2f+qL2CbYPrk1hvj63Ll6c9lftlmsy+By2Y9XR6/TUQfEvUWOMMaYQT6LGGGNMIUNt+6euXbpauKUSf3brUn0u26fLkD/n1VVMtzHdJTt37kzSGirBpdR00dFVmTsqZxyn0OdcL3Q30t2s7iq6WuhupOtd645bot13333Ze6mbm3l00dJ9pa7guhPrR4E+g67mui0i1Q37vve9L8ljX2GohoYSsN+sX78+STN0Q11dy5cvT/L4DgwbYn1MB7kwA7r5ctLNmjVrkjy6rum60zCLpUuXJnlnnHFGkv7Rj36UpNXtSVmHfW3hwoVJuslxYqOG4yz7WC4khH2XbZr9Re1AFyxlB45lOo5wfOc4TPvTVa9wDB0U/xI1xhhjCvEkaowxxhTiSdQYY4wpZKht/9QnTn8+t+rj1n4a1vLggw8meVxCrqEOEanu8JnPfCbJ+/SnP52k6QO///77q2sN3Yio3xpMtREunR6FRseQItWEuAUXy0ptWOuKZV22bFmS5tJ2rTvajEfP7dixI0mrdss2Qt2EmrPqGbT3OI6Q0qX3XIbPLSMvv/zyJK06DregoybK9q71w+9S86GepNvKUUs6//zzkzQ1f25ZNx2obdjvqdlyPcS6deuqa26vx9ArcuONN05ZhoiIf/qnf0rS1L/PPPPM6praIOsnt51i3bqSUaB9kpog+2vuXbh1X257w4i0P2/cuDHJ27BhQ5K+9NJLk/Q555xTXTNMkWMK+4COobR36ZjiX6LGGGNMIZ5EjTHGmEI8iRpjjDGFDLXtn17Tn65HMkVEfP/730/SqvXwu4ztYdyi+tOptX7729/O3kvhdoP33ntvkqYmoNohNbJRaHTUoFUj4fNUl4no3VpOtUnGHvJe3E5N77169eok78ILL0zS3G5QtVpqsdS4qElrunRLrmFQu9QdF0e9SLedo+bJ+ExuL6nb/uWOpYvo1YrV3qodRUQ88MADkUPj78YR9zwVuXpmmXJH59W1aY4LuvVi7hjBiN4YVD2ujlo540ap2WksKNeRjGKrxU6nk9hJ7U2bUZPOHTPJvswxm3bRtviDH/wgyeO9qKdqH+BRgRxj2J60HzMOt3Rdi3+JGmOMMYV4EjXGGGMK8SRqjDHGFDJUnKj6m6k5MC6Lx4zpd6kxMG6RepRqqN/61reSPOoZ1113Xd9y1e3Z2+R4s3HEieo146OoD7E+NLaNmuj+/fuT9MUXX5ykdR9Yxq7xKC7ub6o6CmMe77nnniTNGFTVDqlBj2rv3H5aIPUTHgt19913J2lth7TvbbfdlqTZNz71qU9V14zt5PqBn/zkJ0la44O5ryj7BvM19pjta1yo/RmvTJ2NbU2PA2RffuSRR5I0+4vqfxpvGhFx9dVXJ2mOORrryX13qcmxHSt1+/u2QbfbTe6r/Zl7W1O/pf6oNuZ3uVaC99Ij47h+hjHvXN+hfYDrKDg+MS5Xv9uW7u9fosYYY0whnkSNMcaYQoYKccktj+a2T9dee22S1qNw1F0Y0RsSwKXe6nbikWB0/XIrM3Wn6LL2iF43Al12uk0Ut4waRUgA3edaPoY20H27ZcuWJK3vws/SFcO6UzcaXWx00+RCnVivdLnR1aXlpGtrVFuiafvWNk13LqUAhkdpG9Vwl4jeNsmtKfUIM7qr1GUZ0bv8f/PmzVOWYarnNGnfo9xmsZ9ExL5d1z81VIjhGHT/33LLLUla2zVDg7iVJdui9kUedcbxiO+kx7mxT1PeGAW54yw5LnBLQ4ZbKQy5o6ShaYYp8ug59nXtP6wLjl2UBjV8pq0xxb9EjTHGmEI8iRpjjDGFeBI1xhhjCmmkiZKcP53LkukjV7i0m1oZfeT6LIah8Ln026vPnCEXXA6d2x6P+gD1pzagBt2vLBG9W2VRs9PwBm7dt2rVqp7nKqpZUU9lmMGmTZuStB5zVxemQu1bt7yjHjaOo9AUlpX6CdOqSVN7YQgFw1hWrlxZXd9+++1JHrV3anDUnRW2CWqkqn1TAx7HsVyE7YUaIdu8apfUIhnew3ar2uSPf/zjJO+HP/xhkj7vvPOStIZ+UPujZs2xTnVc9q0mIXZN6Bc2l/tcRG8b0LGUWiT1aqZ1rQTfc+3atUmammnu+DjqypxLWI428C9RY4wxphBPosYYY0whnkSNMcaYQoba9i+nTXL7K2osuv0etcfdu3cnafriVe/gc3Tbs4iIQ4cOJWnVWRiDp0dQRfT611U3GYVvndDeCnU26kU8xkjfhboZ9QzWleqPPLZLbRLRq0FrTBc1Z8aYcis21UT5HMajjhrG99HebKMa+3zFFVckedR8qM+pHfjd7373u0maMaeqK7NM1O2pM2u/YvvhZ9ukn95KXZMxylu3bk3SaleOA9QmWQfaP3gkGXVMtj2Np6SOmFsjEJFqdoxpHIUmmjsKje2QeiL1d70PtUnagVqxbkN6+eWXJ3kXXHBB38/yXszjfMByaN3Vab6D4l+ixhhjTCGeRI0xxphCPIkaY4wxhQwVJ6r+5TqfOGO6cnvYUruhNrBixYrqmn56PT4rojeubv369X3LSJ84dS/VDPg+ozi2KBcnSnszxpLaZG7fXcbQUXtSO7A8jJeldvihD32ouqaN+FxqQvpc6lDjOApNNTnq548++miS1qOdItJ34d7Q3NeVuuZDDz1UXXO9AJ9L7VLtRE2RNuN3tZ/VfXdUaB+s00S1L0ekfZ8628c//vEkzXUBGovI/Yl5BBvbqWqiPALyvvvuS9LUHRctWlRdc5wbhQ7No9CU3FqIiF5tWNs1+yftwPUPqvVznOWRa7mYcL4Lx3++k9qfbbp0DPcvUWOMMaYQT6LGGGNMIUMdhaauL/7kZkiChivw83TbcJs/HsmjIQD86c+f73RBqIuE23fRRUuXT5MtskaBurr4PNqQ7kd9b9pk586dSZo2XLJkSXXNY6Lo+qJ7V0+4p4uHrna6pNX1Mgp3+VT0OwqN7iq6rekaVXf7nXfemf0sj0rTZ9HVy+eyzWr9MI/9iP1M3czcgm6U9BtH2Mbp3mR4g7q+abeLLrooSTOsS7dApLxE6LrXkAuG3dS1E5Wj6PIcVZtXe+uYwv7IMURlhoi0fXF857jMd9P+wbqi3EF3uo51DDXMhR9FpDZlmUqPs/QvUWOMMaYQT6LGGGNMIZ5EjTHGmEJaOwqNS7e5NJ++a9XdGFqiy74jen3i+/btq67pp+dnqaOov52aJ5daU89Qfzo1yHGgehH1EuqYREOKqGMS1qVqlRs2bEjyuKT8hhtuSNI5HZdHcfEdVHMZV4hFP1jf3KKQadW6eAQZt36jjqxtVMO5InptRu1SbUiNmToVdSxtI+MI4fo1Wre5bej4PtzaT0NE2F7uuuuuJJ071uviiy/ue9+IiNtuuy1JaxgL+1Zdm9c0NbpxHPen9qZ+yFA4sn379uqa7Z/bLHKNjL4r11mwbmgHrSvqmGyntD9DA9vAv0SNMcaYQjyJGmOMMYV4EjXGGGMKGeooNIV6FjUk6hmqGTFOkdv+8Rgd1UoY60b/OXVN1Sx4X2oSudim6SCnC1ILoCatdqDmQBtxO77NmzdX19Q2WK/8rtqwLg6OcWnjjFWsg3VPDYgaon5e42wjejVQtmHVLnkcFXUrHvWncXJcW1AXy619eFxxuRGprpVr42xrtJvC9Q2/+MUvkjSPW9T6vPXWW5M86nv3339/3+eeeeaZSZrtRnXniP568KjgUWg523OcYL72dfZ7zhPr1q1L0hqjz+MVaV9q0toO2O/YRjgPUWdXStdd+JeoMcYYU4gnUWOMMaYQT6LGGGNMIa3tnUv/eZ2+uGbNmr7PYXwSjztTXUFjRiN6/etE9SZqbqM4emhU0N4sOzUJ3dOT+4ZSZ2Msm+ZTr6Y2SJuqBsE4RWqg1CtyGlHpPpelUC9hG83tiUpdjFpebg9h6qnLly9P0jwebMeOHdV1Tn+L6NVTVSOd7rjciPo2zvaksC3xaEDWn+rF1FP1mLSI3r11tZwsMzW6HGwHo2jjXNeSG8PZ/xjrr32d6yo4DuzduzdJ6xjD/W5ZDuqaueP+aDPeW/tiXUzpoPiXqDHGGFOIJ1FjjDGmkNa2/aMLhCEvdC098MAD1TVdvWvXrk3SXJqvbkD+BD/rrLOSNJfxaznotqxb0j0d9HOf1y1HpxtJ3VmsC7oEGYKhdqJ7kSEYPNZL647PoauFbSbnzhp33dCVyC3z6NZWVyPfk25IutNZlwqP4aLUoWEtrBv2I6Ynob3n6px1QDe55tO9yP7AdqzP5ZhBu3C80rGPbZifnbQ2rs+ga5Ttg/nablkX/C63B9W2yftyXGYYC/NzsM30G0+Hwb9EjTHGmEI8iRpjjDGFeBI1xhhjChlq27+cf5l+bC4553Jphf51akT6LN6XGgR94rmjosa51dmgDOq353tSo9MQDNpTj96a6plaz3X2zR3RVrc12CRo0v3sXadBN9mykLoxn6k2pf5DvY5bZmq52AcZ1jHuMKFBaLK1JckdHUa7ER1zeB/Wde5edWFyHNtUL+e7T3f9sKxsPzoWsKy0IetO87lGow7VUGmzJsfH1YX0DHyfom8ZY4wxxpOoMcYYU4onUWOMMaaQThO/e6fTORQReyLi5Ig4PKpCFTJpZVre7XYX1n+sP7Z3I450e0dMVrmGtnfExNt80sp0pLfxSSvTQPZuNIlWX+p0Nna73XX1nxwfk1imtpjEd5vEMrXFpL7bpJarDSbx3SaxTG0xie82iWUaBLtzjTHGmEI8iRpjjDGFlE6i/9xqKdphEsvUFpP4bpNYpraY1Heb1HK1wSS+2ySWqS0m8d0msUy1FGmixhhjjLE71xhjjCnGk6gxxhhTiCdRY4wxphBPosYYY0whnkSNMcaYQjyJGmOMMYV4EjXGGGMK8SRqjDHGFOJJ1BhjjCnkmCYfnjVrVnfBggWjKssRxRNPPHF42GOLZs+ebXsPyJ49e4a294knntidO3duW0U6ojlw4MDQ9o74ZRtfuPC/b9PpdKrrut3U9LODfH5QeN9haHNHuN27d7fSxufMmdNWkY5oDh48OJC9G02iCxYsiL/8y7+cMm9UDXoq9N5HHTW6H9PDvMMXv/jFPcM+f8GCBfE3f/M3Rd9l2dVOzGO6ySDSpN6HeU4dn/vc54a299y5c+Pzn/98lc4N6HXv3WQyaEITmzUtcxO+9rWvDW3viIiFCxfG3/7t31bpXDslufdp2tZyY8rRRx+dpN99992B7jNVehg++9nPDm3zOXPmxO///u+3UZzk3Zr0hyb3bfrdNvm7v/u7gextd64xxhhTiCdRY4wxppBG7tyIcjcVf5KrS+Sdd95J8t5+++0kTXeK5j///PNJ3rHHHpuk6ZrRND973HHHZe81HbRl72HcSvpd1hVdW0xr3dW53pk/HW6cQZ/ZppvumGPSbsj2noP1kXNpTiqlbTznOuUYQtjXtQwvvvhikjdjxowk/dZbbyXpN954o7o+4YQTkjzWLdOTTJ1bNddXcuM90+z3dW2gSf8YB/4laowxxhTiSdQYY4wpZGy+BbqdFP7Uf/3115P08ccfn6TV3fL4448neXTT8LnqomX4yMknn9y3jLz3pKwg6/d8lk/tQJu8+eab2bRC1wvrSl1bEam9GT4yc+bMJJ1zdbFMo1yV3QZaH3TxMZ2rD9qEn6UdtI3S7XX48OEknXN5ss9NB3UuQearLWbPnp3ksa8zrfd+4oknkryXXnopSdOur732WnXNuqUklHsHuomnm7oxTd+F/T43DkSk9mcbrpPk1E6vvvpq3zKNi8keiYwxxpgJxpOoMcYYU4gnUWOMMaaQkWmi9KdTyznxxBOnvI7o1RVmzZqVpFW7XLZsWZJHn7jqFRERTz/9dHW9aNGi7HNy0Oc/qnCCQUMWqA1Qx1E78D4vvPBCkqb+qDob64qa3cqVK5O0fn7+/PlJHrWOV155JUnrO7HM49CPcjuyUBdjm9V3o05Pmz377LNJWrVjhlewXmkX1T1ZxiahA3VhZuOA5aUd2RZ1XKDmSZvzXvpdro3QMWOqtNqKeWzj1Jp1C76moR6jhuXJ6Y1sw2wv1Ki17tiXqYnu3r07SS9evLi6ZjgS12hwTNdysYyl6yz8S9QYY4wpxJOoMcYYU4gnUWOMMaaQxprooH56+rX5vXnz5vX97ssvv5ykGQv63HPPVdf79+9P8hiLSC1k9erV1fWFF16Y/S41AI0f0zJE9GqSo0B1OeqW3P7w0KFDSVr9/7QJdQXqF6r3nXTSSX3LFNGrFat+ceqppyZ51PsYn6dxpLQ3ddy26KeDUtOv0whV52FdUW9kvupL1O2pA1Lb0zbLNsD1AdSPtD543+mIy2U7pb6lR6hFRCxdurS65ruynbI9bdq0qbpmX+K4wP6i/YNtnGMZ263WF+Omp5vcFp4RaTvmuKD9PqL3vZ988snqmvHLtCHrTtdWsO9s27YtSXNc1nuxb5XGRvuXqDHGGFOIJ1FjjDGmEE+ixhhjTCGtHYVG/zlj/hgLpD5xxr5RG6PuuWPHjur60UcfTfKovV533XVJ+qqrrupbZmoufAdqI8qoNNF+Gh21R7J8+fIkrVoa9Wjq16effnqSfuaZZ6rre++9N8mr0wa1nmlv2pM6imqzW7ZsSfKoW40CtTd1TGqGzFfdjO2I+hF1M+p5OajPHTx4sG+ZGMuq9RqRakKnnHJK37xRou2d5WW7pU67b9++6po2pc62ZMmSJK36K/VSxkCyv+i9GAvN9RyPPPJIktb6o949jthctTc1WT6fGrr2AY5HHMO1XUakff3iiy9O8jhXMOZXtW+WibDvaZrjjzVRY4wxZsx4EjXGGGMKGSrEJbclHZcl072iP8N37dqV5NGdQjeDuiq5TPl973tfkj733HOTtLootm/fnuTRNcZl2frzv+5IqrZQO+oz+DxuVUbXtNrpnHPOSfLoaqG78amnnur73Z07dyZpumT181u3bk3ycsfURaTvxDKxzG2RC2vJkTvOj+2IoRt0Ta9Zs6a6potPXZYRvS5btT9lENqMLjiVVejqamKLYVD750IqInolFH0ftiWVFSJ6xyP9PF31bNN1W5oq3AZQ+1JE2q7ZLsYdVsT35phOm6rbde/evUnegQMHkjTb4mWXXVZd053L9D333JOk77///uqabYJue5Yjt5VoKf4laowxxhTiSdQYY4wpxJOoMcYYU8hQIS5Ntjbbs2dPks4dM5U7pigi1Uwvv/zyJO/aa69N0vTV33HHHdU1NRVdOh3RqwHoO+a2W2sTtZNqJqtWrUo+R/2I+q7akHrpihUr+n42Ij2uiaEmfA6X9GuYgoYmRfRqpGeffXaS1m3rGAY1Kvrp/Fx2z/KwDWv75xJ9bnNGzV+1Sy7R51Z+Z555ZpLWcAs+l3XFNqv9gXoc+0KbqJ31udQa2ZcfeOCBJK3aMuuH7TIXsvS5z30uyfut3/qtJE1dU9ux9pWI3jUD1GZVw6MmOg4dOreVKMdHhhyprsz32rx5c5Km/n7NNddU1zfccEO2jHyu9he26bq1Kjr2cQwvxb9EjTHGmEI8iRpjjDGFeBI1xhhjCmmsiSq5mC5qSIzJ0Vg45tUdFaXawbp165I8xuTdcsstSVp987wvt31ijJ6Wk/Gpo0JtrBoWy8qt4lh21RX02Kep7kVt8kc/+tGU1xG9cYtnnHFGktYYL8bm8bPUalX/Y5xoWzFeOfrF6Eb0aoTUNVXLZN1Q1+QWabodH+PrmmwZWBczTT1Jn8ut7kZpb7VzTqPjlnnf/OY3k7TaldoYdTVql5pPrZh6H7V8tRvjEhmLzq0WdaxjmxpXbO6voQbKNNuErk3h8YmMJz/ttNOS9Cc/+cnqetmyZUned77znSRNu+j4T32a4z/LpfVeFxc7KP4laowxxhTiSdQYY4wpxJOoMcYYU8hQmqhCXY1xc9Rj1GdO7YN6I7WbfsexRUTcfPPNSXrhwoVJWn3z/C51XPrXVU/lETyM8Ro1jDWk7sajrFSb4XtRH6IOpzG+27ZtS/IefvjhJM3jqrQuuR8r43JnzJiRpLU+GGfG9tUW/dpWXUwZ8/U4udwezBG9GqnWLXUorj3I7T/M9lx3JJtqgSzzuPS5XLw196HdvXt3kta641GAHFM4Xp133nnVNXV+6uFs49r3aGPajbp/biwbB7l4fWqR1EjVDrQnNWiOwzrmcK903ovjkY5luT2rp0LLxfctHVP8S9QYY4wpxJOoMcYYU8hQR6GpC5PuTLrmuMRZ03WuMh5voy6t++67L8njvXJH5fAIKrpE6VZQ10zd0W+jQN+FbiM+n24OdQsy/IVhEj/72c+StLpz+dm1a9cmabrNNFyGrsi6Jf0aNjIdri61KcvKpfN0H+pxTtz6jVuiMTxm/vz51fXq1auTPLoWdZu/iNSGDKege5dQIphu2KbZtrjFp26ByOO06BbneKXud7bxXLuMSPsTP8stNGlj3Z6Q3x3HmKKwj3EMycldHKNpI84H+m4MC+L2oA899FCSVpc4n8t6plTYrwzD4F+ixhhjTCGeRI0xxphCPIkaY4wxhbQWl1G3zRb97aoV8NiiXbt2JWn64q+44orq+mMf+1iSxyXODI9RqJ/Sb88y6+epkY1Dv8jpgnwX6jq6dRm1SS5dv/3225P0Y489Vl1zqz7WHfXWs846q7qmfXm0FfXrXLjGqDTSQY9CY33z2DHV2Khbal5E79aVt956a3VNvYghErntBxk6wM+yr2i7YD2yjYwDlvcDH/hAkl65cmWS1jGH4TDUxqjZ6bvzeDl+lyEuGqrFvkUNm9vU6TtQ89Uj8UaFlrfuOEf2AW0/PGqOn+XRjaoNUwNl2BxDwjQ8hqFvDNvi+KRjTFvrWvxL1BhjjCnEk6gxxhhTiCdRY4wxppDGmmi/o7moZ1Gjo2akMZqMS2TsJ7WP66+/vu9zLrrooiT96KOP9r03feLc/o4xXho3ypjScaDlpeZJvXHx4sVJWrdI++53v5vkMeZR4xQjUh30qquuSvK4nRfj87Rc3EqOujn1DdVcqDWNG+qHjDGm9rJhw4bqmu2IfUGPBYxI63b9+vVJHttsLk6UWhL1VMbYafumFjvKON1+W8+xXXLrUOpud955Z3V97733Jnl6JF9Er8213VJXW7FiRZLmOgttt9QVL7300ux3tS0wrndU6yzU3tqv2F6o5/LIPtWsOR5R5+S75baYZHw+61k1aMZgU7tnP9V5inMHNfhB8S9RY4wxphBPosYYY0whnkSNMcaYQoaKE83pJNSwqDPocTY8eoj6KveJ1RgwlmHLli1JWmPuIlLfPWPhuJ9m7tgi6hXjiFtUrYAancaBRvTqcB/96Eera+7fSd0gd2wUNTjGh9FmCmPk2CZo/+nQQftp/rk9OCN6Y5s1lpDvzWPq2L61fqhPU8ekJr1p06bqmro97Ul9TvUkfrcuhnAY+ml/LC/flfqXaqa0KfUu6tIax8i+zP7CvqX6K9swY05pRy3nuGyu9tZxhPHLHAeoc+oRZdQx9ajLiN661BhY7jHAembdqc24roJtgmsV9PPUfEvxL1FjjDGmEE+ixhhjTCFDHYWmP4frtrfiFmS6bJmuF7oGuLT6L/7iL6prugTpCqarQF24dOlwOy8ugVb3yijdW4q6XtRVRDcG3bkaYhERcdNNN1XXf/AHf5Dk0V3CulLXO48lIlzKrqESdFfRbUY3Z5Nt90ZBP1d6RK/bjvbX487Yri677LLsvdQlTjcYwxAYHqBHDNYdZUU3pbqs2W+mA5aXNma+ti+GsPBIObZFdWuybdXVtY59evRfRK8d6UJXm9f1j1Gg4yXDVHh8JV20Gr5GSYjyB92qOua///3vT/IoYbCdappjFcds2lDr0kehGWOMMdOMJ1FjjDGmEE+ixhhjTCGtHYVG6N/n0USqDTDEhboP76W6G33t9ONTG1G9iduK8XgkbkOn/nf608ehkerybYa46FZYEb2hJhr6w6XrXNpODe/ZZ5+trhliQW2WoR6qF7GMrGfaVHWVto4takJuSzS+N7VL1WaoNVG337NnT980NTVqQGzDbP8K64blUo2Rzx2Hvfkc9j+GU7EOVNtnqENdOJtuD8qximsnuA5D7039jn2JdtU6YLsYB6r1c22KHoEY0avnqv25NSKPcaOOrN+t226VNtN7cRzk2oXc1q0OcTHGGGOmGU+iALjLkwAABedJREFUxhhjTCGeRI0xxphChtJEVb+gZkWNKKep0J9OXzU1CT1eaNmyZdl70Sd+6NCh6pr6BTUiajLKdGh0CmM7maZWpjbNab1TofoFNVHq1YwPUw2Cx7Wxnqm5aP647UuoV1GLoU6m7Y7vdddddyXpXLwj35v1rNvVRaRrBFjP7JO813Qc7xfRPx6SNq5D44y3bt2a5Gms81T31nhbavfUNdmOdRzhGMKYR2qxWvfTPabUjeF8F40N5ThLdIvAiHRrP2rbdXq8fj63jiKit1/qmNKWvf1L1BhjjCnEk6gxxhhTiCdRY4wxppCRxYlS76L+ohoE90ylRkT0XtQ27rzzziRN/ULj9/gc7kPKuFH97jj2tSTqs6dewThV7mGrZa+LcWWMnWoUjOmiHaibaOwZ99fkO1ALGcWxRU1QO1ETZXwy94pWHZR5vBfbmb4rdUrG/6rGz89zH1HqceM6zq+OfmsrWB6mc3GVjOtl/CztqLZhvCT1PNpV+wQ1Oa434HiksI2Poz767YUe0TtmU7tUG2sseUSvHk+defHixdU145c5DueeS7gWhG18FLG4/iVqjDHGFOJJ1BhjjCmksTu3n+ul7vggpnVJdN0RTVw2rq5K/tQndGvqlnc8QX0Yl+G4l6PzeSw77a12ovuWad5LtwWkq4XP4b3UrUMXD7fooptsOly4irbvuvAQuom0ftR1NdVnc6E/uaOcInptps9lXVD6yLXZ6Qop0ueyvHT3U3ZQNyzdtZdffnn2ufosjj90EdJlq9TJLKx71ue4UXvTntyqj+i7sr2wb9MFq3JIXWgV27hus8i6oX1zYYpt4V+ixhhjTCGeRI0xxphCPIkaY4wxhTR2yPfTQandUM+iRpFb6r18+fK+z4xItZHcNnNTfVep0/P4Dpo/rvCAfvau06yoDaheRO2D+l7u3rQ3NSA+V/UM1g3tP10hFv3KkGvffBemdbuxupCi3HvXhQGRnCY6CfZtAu3Gd+f7advkVovUOXPhM8xj/+C4oOWifle3/iBXpumGa1Vy5avbQo91p2keU0edk/ZXnXMcISx1+JeoMcYYU4gnUWOMMaYQT6LGGGNMIZ0mfvhOp3MoIvZExMkRcbjm4+Nm0sq0vNvtLqz/WH9s70Yc6faOmKxyDW3viIm3+aSV6Uhv45NWpoHs3WgSrb7U6Wzsdrvrioo1IiaxTG0xie82iWVqi0l9t0ktVxtM4rtNYpnaYhLfbRLLNAh25xpjjDGFeBI1xhhjCimdRP+51VK0wySWqS0m8d0msUxtManvNqnlaoNJfLdJLFNbTOK7TWKZainSRI0xxhhjd64xxhhTjCdRY4wxppBGk2in07mh0+k82ul0Hut0On8xqkINUI7/t9PpPN3pdB6Sv83vdDq3dDqdHb/677zpKl9b2N7jxzYfL7b3+JkEmx9J9h54Eu10OkdHxD9GxEcj4tyI+Gyn0zl3VAWr4X9HxA34219ExH91u91VEfFfv0q/Z7G9x49tPl5s7/EzQTb/33GE2LvJL9H3RcRj3W53V7fbfTMivhkRvz2aYuXpdrt3RMSz+PNvR8T/+dX1/4mIT461UO1je48f23y82N7jZyJsfiTZu8kkenpE7JX0vl/9bVI4pdvtHvjV9cGIOGU6C9MCtvf4sc3Hi+09fibZ5u9Jex+RC4u6v4zbcezOmLC9x49tPl5s7/HyXrJ3k0l0f0QslfSSX/1tUniq0+ksjoj41X+fnubyDIvtPX5s8/Fie4+fSbb5e9LeTSbReyNiVafTObPT6RwXEf9PRPzf0RSriP8bEf/rV9f/KyK+P41laQPbe/zY5uPF9h4/k2zz96a9u93uwP8i4mMRsT0idkbEV5t8t81/EfH/RcSBiHgrfunT/1xELIhfrujaERE/jYj501W+Ft/T9rbNj2ib297/M21+JNnb2/4ZY4wxhRyRC4uMMcaYceBJ1BhjjCnEk6gxxhhTiCdRY4wxphBPosYYY0whnkSNMcaYQjyJGmOMMYX8/7735kWL5FJWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x576 with 25 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_hidden_layer(res.x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}