pytorch-neuralnet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dtype = torch.FloatTensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 64  # batch size\n",
    "input_dim = 1000  # input dimension\n",
    "hidden_dim = 100  # hidden dimension\n",
    "output_dim = 10   # output dimension"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# input and output data\n",
    "x = torch.randn(batch_size, input_dim).type(dtype)\n",
    "y = torch.randn(batch_size, output_dim).type(dtype)\n",
    "\n",
    "# Randomly initialize weights\n",
    "w1 = torch.randn(input_dim, hidden_dim).type(dtype)\n",
    "w2 = torch.randn(hidden_dim, output_dim).type(dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 32629818.697378688\n",
      "1 29639069.575577572\n",
      "2 31565690.009640925\n",
      "3 32405940.646661058\n",
      "4 28353118.857581124\n",
      "5 19716795.435692035\n",
      "6 11201645.904863745\n",
      "7 5685191.090651594\n",
      "8 2966191.8320646994\n",
      "9 1738024.3777328655\n",
      "10 1172917.1127420184\n",
      "11 880926.3565512658\n",
      "12 706738.9071125507\n",
      "13 587910.9585375916\n",
      "14 498969.72646291554\n",
      "15 428615.83280366845\n",
      "16 371242.8594270777\n",
      "17 323790.5308198726\n",
      "18 283923.64515687595\n",
      "19 250095.35423040146\n",
      "20 221167.2744523422\n",
      "21 196276.01938981423\n",
      "22 174746.74711279152\n",
      "23 156047.52482570393\n",
      "24 139733.42658157414\n",
      "25 125445.28035887978\n",
      "26 112878.81285783146\n",
      "27 101798.48348675203\n",
      "28 92003.75664738257\n",
      "29 83311.31713879586\n",
      "30 75575.60022745095\n",
      "31 68666.66589825135\n",
      "32 62489.17666272074\n",
      "33 56951.80910727063\n",
      "34 51980.75553461498\n",
      "35 47510.75894754102\n",
      "36 43479.82513540902\n",
      "37 39837.380138760855\n",
      "38 36542.52530213431\n",
      "39 33555.43675281806\n",
      "40 30842.561831531537\n",
      "41 28375.731322248335\n",
      "42 26129.751894926565\n",
      "43 24084.823317346883\n",
      "44 22218.837466398836\n",
      "45 20513.572041687992\n",
      "46 18953.33338333509\n",
      "47 17524.219303275022\n",
      "48 16213.708884728185\n",
      "49 15010.767257574342\n",
      "50 13905.711091326353\n",
      "51 12889.572802737814\n",
      "52 11954.274535187538\n",
      "53 11093.171326032529\n",
      "54 10299.392485844357\n",
      "55 9567.085562762775\n",
      "56 8891.650826125522\n",
      "57 8267.786773299618\n",
      "58 7690.828139435796\n",
      "59 7157.164089029557\n",
      "60 6663.431152480471\n",
      "61 6206.196798370807\n",
      "62 5783.009570439554\n",
      "63 5391.47626925268\n",
      "64 5028.633310276156\n",
      "65 4691.628450736914\n",
      "66 4378.792330894028\n",
      "67 4088.2412672548626\n",
      "68 3818.3102610834426\n",
      "69 3567.1975278100645\n",
      "70 3333.5676418589196\n",
      "71 3116.1701625408073\n",
      "72 2914.031304376486\n",
      "73 2725.600903193999\n",
      "74 2549.9787838747447\n",
      "75 2386.310282487982\n",
      "76 2233.701344652094\n",
      "77 2091.4125454847217\n",
      "78 1958.5999742554886\n",
      "79 1834.6776509950455\n",
      "80 1718.9830498583508\n",
      "81 1610.983521583141\n",
      "82 1510.0912003127253\n",
      "83 1415.7735695454076\n",
      "84 1327.6214805742811\n",
      "85 1245.2454411933065\n",
      "86 1168.2161941016197\n",
      "87 1096.140198740261\n",
      "88 1028.7021129580562\n",
      "89 965.5872559559375\n",
      "90 906.5213452980915\n",
      "91 851.2655740819075\n",
      "92 799.4859238318734\n",
      "93 750.9647598490145\n",
      "94 705.5062209538993\n",
      "95 662.9114909059923\n",
      "96 622.9961567690857\n",
      "97 585.5966706391068\n",
      "98 550.514365465373\n",
      "99 517.6052082399638\n",
      "100 486.7359740033458\n",
      "101 457.7904002178666\n",
      "102 430.63168534464603\n",
      "103 405.1450431935482\n",
      "104 381.2086738600266\n",
      "105 358.7386355314529\n",
      "106 337.6392177439775\n",
      "107 317.827041578363\n",
      "108 299.2165793538022\n",
      "109 281.7349850036232\n",
      "110 265.2977589568561\n",
      "111 249.85779474205236\n",
      "112 235.35131325213996\n",
      "113 221.70761661031554\n",
      "114 208.88495114636652\n",
      "115 196.8329562624861\n",
      "116 185.49406970428095\n",
      "117 174.8239097577777\n",
      "118 164.78646571478487\n",
      "119 155.3470508290864\n",
      "120 146.46362387718955\n",
      "121 138.104260472291\n",
      "122 130.235681635986\n",
      "123 122.82983407469283\n",
      "124 115.85545194165161\n",
      "125 109.28955728986625\n",
      "126 103.10763441763129\n",
      "127 97.28427473722775\n",
      "128 91.8004346473441\n",
      "129 86.63438480615869\n",
      "130 81.76753204128747\n",
      "131 77.17885645990211\n",
      "132 72.85747236935633\n",
      "133 68.78302454038655\n",
      "134 64.94374705627948\n",
      "135 61.326693372595344\n",
      "136 57.911863206048906\n",
      "137 54.69366890273261\n",
      "138 51.65933359211981\n",
      "139 48.802619261715876\n",
      "140 46.108864795148044\n",
      "141 43.567092756097146\n",
      "142 41.169897377899936\n",
      "143 38.90655351384767\n",
      "144 36.77074570509191\n",
      "145 34.756563378867455\n",
      "146 32.85490728822802\n",
      "147 31.059347305383568\n",
      "148 29.364347473864115\n",
      "149 27.764135729294196\n",
      "150 26.252464012948906\n",
      "151 24.825180531351478\n",
      "152 23.477574208519925\n",
      "153 22.205326702990092\n",
      "154 21.00231501686605\n",
      "155 19.866867777295525\n",
      "156 18.794528974590264\n",
      "157 17.7805544199825\n",
      "158 16.822143826692002\n",
      "159 15.917148127675746\n",
      "160 15.061899161868133\n",
      "161 14.253653791034075\n",
      "162 13.489231225472693\n",
      "163 12.766362678809074\n",
      "164 12.083790627021664\n",
      "165 11.438454745542188\n",
      "166 10.827993431479143\n",
      "167 10.25037399345847\n",
      "168 9.705035071180218\n",
      "169 9.188687692361693\n",
      "170 8.700196386149438\n",
      "171 8.238542364129945\n",
      "172 7.801752721503695\n",
      "173 7.38881430601535\n",
      "174 6.997662616714994\n",
      "175 6.6279253838890035\n",
      "176 6.277786779018275\n",
      "177 5.946332138337347\n",
      "178 5.633370084654963\n",
      "179 5.336654904835012\n",
      "180 5.056053588204211\n",
      "181 4.7904472811506515\n",
      "182 4.538814731148396\n",
      "183 4.300897907427071\n",
      "184 4.0753648290078\n",
      "185 3.862105325519906\n",
      "186 3.6600942103924057\n",
      "187 3.468749563180399\n",
      "188 3.2878173362429557\n",
      "189 3.1163203936977055\n",
      "190 2.9538731395007716\n",
      "191 2.800317056905321\n",
      "192 2.654549803869447\n",
      "193 2.5164190713968395\n",
      "194 2.3857120229311057\n",
      "195 2.2620357867760674\n",
      "196 2.144721266550807\n",
      "197 2.0336509619582195\n",
      "198 1.9284526563050015\n",
      "199 1.8286132314316816\n",
      "200 1.7341331038644086\n",
      "201 1.6445096513078536\n",
      "202 1.5596833309810743\n",
      "203 1.4794102856381883\n",
      "204 1.4031537238274838\n",
      "205 1.3308867507876059\n",
      "206 1.262485322173843\n",
      "207 1.1975737362547187\n",
      "208 1.135973410644608\n",
      "209 1.0777292835426149\n",
      "210 1.0224781480518814\n",
      "211 0.9701691650637123\n",
      "212 0.9202823571571237\n",
      "213 0.8731464727092764\n",
      "214 0.8285611963301314\n",
      "215 0.7862856462935248\n",
      "216 0.746042984688561\n",
      "217 0.7079652596869581\n",
      "218 0.6718697311951172\n",
      "219 0.637548478090425\n",
      "220 0.6051642094193639\n",
      "221 0.5742156132323304\n",
      "222 0.5450679443707571\n",
      "223 0.5173539948548811\n",
      "224 0.4911145003248656\n",
      "225 0.4661075056435857\n",
      "226 0.44243755248064165\n",
      "227 0.4200124313615561\n",
      "228 0.39872566774070695\n",
      "229 0.3784866533685056\n",
      "230 0.3592891041648487\n",
      "231 0.3411523752865051\n",
      "232 0.323876820175947\n",
      "233 0.30752567910214346\n",
      "234 0.29196344753006187\n",
      "235 0.2772402265708446\n",
      "236 0.26325538481783006\n",
      "237 0.24994306768656704\n",
      "238 0.23738769012979333\n",
      "239 0.22542557410546726\n",
      "240 0.21408134178840432\n",
      "241 0.20330805914324718\n",
      "242 0.1930610932376169\n",
      "243 0.18331727724398528\n",
      "244 0.17414145164584438\n",
      "245 0.1653614540542639\n",
      "246 0.15711398492108963\n",
      "247 0.14923577513489983\n",
      "248 0.14172309885551337\n",
      "249 0.13460119668734705\n",
      "250 0.1279168331355532\n",
      "251 0.1214901033389766\n",
      "252 0.11540405783303888\n",
      "253 0.10963289454861125\n",
      "254 0.10416984926070033\n",
      "255 0.09895399082152956\n",
      "256 0.09400748299429029\n",
      "257 0.08932095909081972\n",
      "258 0.08485992264881126\n",
      "259 0.08062137357921118\n",
      "260 0.07659338593130022\n",
      "261 0.07279765026809365\n",
      "262 0.06915704624320207\n",
      "263 0.06570047269920481\n",
      "264 0.062428374702772516\n",
      "265 0.05931431279682703\n",
      "266 0.056340329418330976\n",
      "267 0.053553212350360035\n",
      "268 0.05089696331727048\n",
      "269 0.04838682699601779\n",
      "270 0.046003034876209936\n",
      "271 0.04372479740490198\n",
      "272 0.041553487404577\n",
      "273 0.03949482626029899\n",
      "274 0.03753449600468356\n",
      "275 0.03567851223722629\n",
      "276 0.03391568452111704\n",
      "277 0.03223883967718516\n",
      "278 0.030644691532153878\n",
      "279 0.02913714105824844\n",
      "280 0.027715774497310774\n",
      "281 0.026341518736007485\n",
      "282 0.02505037235652158\n",
      "283 0.02381521081962379\n",
      "284 0.022642213090122265\n",
      "285 0.0215354648735005\n",
      "286 0.020471738391171446\n",
      "287 0.01945922336583772\n",
      "288 0.018512211239926635\n",
      "289 0.017600352821822263\n",
      "290 0.01674266610057118\n",
      "291 0.015918131548992964\n",
      "292 0.015150117619400216\n",
      "293 0.014413140174930561\n",
      "294 0.013721049426397625\n",
      "295 0.013056230421295112\n",
      "296 0.012411728951689405\n",
      "297 0.011818430966700544\n",
      "298 0.01124718123176971\n",
      "299 0.010705424577186127\n",
      "300 0.010193940623009518\n",
      "301 0.009697309773781665\n",
      "302 0.009235104787036166\n",
      "303 0.00879359509293054\n",
      "304 0.008376243929928417\n",
      "305 0.007974225912390331\n",
      "306 0.00759275515953961\n",
      "307 0.00723470819357086\n",
      "308 0.006898224032105382\n",
      "309 0.006580016674428091\n",
      "310 0.006266159467233856\n",
      "311 0.0059776009739221925\n",
      "312 0.005693181084596155\n",
      "313 0.00542515333598971\n",
      "314 0.00517873554228121\n",
      "315 0.004938810443868458\n",
      "316 0.004715956877626715\n",
      "317 0.004495610730976873\n",
      "318 0.004292598919978241\n",
      "319 0.00409314371292302\n",
      "320 0.003908590182878147\n",
      "321 0.003731232057965661\n",
      "322 0.0035674954361317734\n",
      "323 0.003405584342209611\n",
      "324 0.0032535367396657744\n",
      "325 0.003107777554112845\n",
      "326 0.0029707863014958313\n",
      "327 0.0028448511141111466\n",
      "328 0.002720185264538044\n",
      "329 0.0026029653769859135\n",
      "330 0.002490552822499492\n",
      "331 0.0023822620067807665\n",
      "332 0.0022823212339411203\n",
      "333 0.0021858283516462684\n",
      "334 0.00209378675351779\n",
      "335 0.0020035175368196545\n",
      "336 0.001918758194044523\n",
      "337 0.0018437310095706927\n",
      "338 0.0017678178523339028\n",
      "339 0.0016944441216069084\n",
      "340 0.0016277045404275992\n",
      "341 0.001563773533440893\n",
      "342 0.001499169953982118\n",
      "343 0.0014396486723363688\n",
      "344 0.0013849308862586285\n",
      "345 0.001329525362701256\n",
      "346 0.0012796900403980738\n",
      "347 0.001230649695860453\n",
      "348 0.0011836421010552045\n",
      "349 0.0011396050962968474\n",
      "350 0.0010972927880526506\n",
      "351 0.0010574158504541575\n",
      "352 0.0010186789818402708\n",
      "353 0.0009812189489110112\n",
      "354 0.0009467795440341042\n",
      "355 0.0009130925810105911\n",
      "356 0.0008805800170306743\n",
      "357 0.0008503996905409178\n",
      "358 0.0008202581352160232\n",
      "359 0.0007923361279851271\n",
      "360 0.0007667601345567321\n",
      "361 0.000739016588807484\n",
      "362 0.000714074507744189\n",
      "363 0.0006906291594844932\n",
      "364 0.0006680067288572511\n",
      "365 0.0006470650446624426\n",
      "366 0.0006259113614797696\n",
      "367 0.0006050798022088633\n",
      "368 0.0005850457400099068\n",
      "369 0.0005655234579631796\n",
      "370 0.0005485102792854302\n",
      "371 0.0005312509062795241\n",
      "372 0.0005148227192373622\n",
      "373 0.0004988432861490338\n",
      "374 0.0004825683449090762\n",
      "375 0.0004677007199710183\n",
      "376 0.00045420706845122893\n",
      "377 0.0004412325664452134\n",
      "378 0.00042823355556625975\n",
      "379 0.000415752813489656\n",
      "380 0.00040441700067462075\n",
      "381 0.00039277088852297803\n",
      "382 0.00038115967829810415\n",
      "383 0.000369816095666331\n",
      "384 0.00035970345148961594\n",
      "385 0.00034902487980556854\n",
      "386 0.00034036774557283725\n",
      "387 0.000331413524038926\n",
      "388 0.0003212680853117711\n",
      "389 0.00031388068561055693\n",
      "390 0.00030537443136441356\n",
      "391 0.0002970122764897193\n",
      "392 0.000289781415254986\n",
      "393 0.00028216626315269444\n",
      "394 0.0002752173055022289\n",
      "395 0.0002679985379584471\n",
      "396 0.0002610742667568111\n",
      "397 0.00025443616558085136\n",
      "398 0.0002484165404376404\n",
      "399 0.00024232250823891244\n",
      "400 0.00023666266693342375\n",
      "401 0.00023052216061564268\n",
      "402 0.00022516231923365437\n",
      "403 0.0002199543746803645\n",
      "404 0.0002138327730801387\n",
      "405 0.00020954200737915984\n",
      "406 0.00020497739151675853\n",
      "407 0.00020070109383737988\n",
      "408 0.00019585536273572512\n",
      "409 0.00019144086562800436\n",
      "410 0.00018726922232253917\n",
      "411 0.00018285811679054353\n",
      "412 0.0001784006782367155\n",
      "413 0.00017481700844512726\n",
      "414 0.00017109117179819888\n",
      "415 0.00016776609959312516\n",
      "416 0.0001636898072640558\n",
      "417 0.0001604839751997572\n",
      "418 0.00015728999412900624\n",
      "419 0.0001535666493119453\n",
      "420 0.00015055205651932912\n",
      "421 0.00014737245742696414\n",
      "422 0.00014412323746815225\n",
      "423 0.00014123677586769245\n",
      "424 0.00013804275894169216\n",
      "425 0.0001350933146107569\n",
      "426 0.00013243541573831896\n",
      "427 0.0001295649958237416\n",
      "428 0.00012713432469396713\n",
      "429 0.00012542006675420048\n",
      "430 0.00012268021361822035\n",
      "431 0.00012041285433131779\n",
      "432 0.00011828453966750141\n",
      "433 0.00011577220198906446\n",
      "434 0.00011395576488293716\n",
      "435 0.00011135294819066\n",
      "436 0.00010948620855778801\n",
      "437 0.00010735500175763912\n",
      "438 0.00010573167045438425\n",
      "439 0.000103851128486699\n",
      "440 0.00010185453613520412\n",
      "441 0.0001001585547484446\n",
      "442 9.785061315693844e-05\n",
      "443 9.628966692983021e-05\n",
      "444 9.445637846856869e-05\n",
      "445 9.308691359016796e-05\n",
      "446 9.139180250433698e-05\n",
      "447 9.017859377746795e-05\n",
      "448 8.860754003782445e-05\n",
      "449 8.709001201391564e-05\n",
      "450 8.57833372728406e-05\n",
      "451 8.415640719609818e-05\n",
      "452 8.280771802801623e-05\n",
      "453 8.132138603964001e-05\n",
      "454 8.013981262031122e-05\n",
      "455 7.867988684963356e-05\n",
      "456 7.751986866141869e-05\n",
      "457 7.637107941267651e-05\n",
      "458 7.542656244877155e-05\n",
      "459 7.425139223632415e-05\n",
      "460 7.301539963905368e-05\n",
      "461 7.178581900936856e-05\n",
      "462 7.046868860266509e-05\n",
      "463 6.921078264145175e-05\n",
      "464 6.813588966742001e-05\n",
      "465 6.693611603265925e-05\n",
      "466 6.60653750182566e-05\n",
      "467 6.520484609180699e-05\n",
      "468 6.422274495766767e-05\n",
      "469 6.351570945957452e-05\n",
      "470 6.247728068374536e-05\n",
      "471 6.172147405597572e-05\n",
      "472 6.076333535837142e-05\n",
      "473 5.9962545935376976e-05\n",
      "474 5.8956449758229557e-05\n",
      "475 5.820601749118026e-05\n",
      "476 5.74597422896006e-05\n",
      "477 5.66893339878638e-05\n",
      "478 5.5775570650395956e-05\n",
      "479 5.4945310256113156e-05\n",
      "480 5.438136685029238e-05\n",
      "481 5.3780541473622684e-05\n",
      "482 5.291752806165298e-05\n",
      "483 5.23341749977535e-05\n",
      "484 5.149997439564402e-05\n",
      "485 5.083420953388984e-05\n",
      "486 5.0097776035318287e-05\n",
      "487 4.935521810034427e-05\n",
      "488 4.86532733639522e-05\n",
      "489 4.8153233962511255e-05\n",
      "490 4.753355416928576e-05\n",
      "491 4.6809983953344236e-05\n",
      "492 4.62422420247216e-05\n",
      "493 4.5849464325620204e-05\n",
      "494 4.511483377134251e-05\n",
      "495 4.463982127595259e-05\n",
      "496 4.403376351539215e-05\n",
      "497 4.3333901625905447e-05\n",
      "498 4.2808408821370936e-05\n",
      "499 4.251143387845602e-05\n"
     ]
    }
   ],
   "source": [
    "learning_rate = 1e-6\n",
    "for t in range(500):\n",
    "    # Forward pass: predict y\n",
    "    h = x.mm(w1)\n",
    "    h_relu = h.clamp(min=0)\n",
    "    y_pred = h_relu.mm(w2)\n",
    "    \n",
    "    # Compute and print loss\n",
    "    loss = (y_pred - y).pow(2).sum()\n",
    "    print(t, loss)\n",
    "    \n",
    "    # Backprop to compute gradients of w1 and w2 with respect to loss\n",
    "    grad_y_pred = 2.0 * (y_pred - y)\n",
    "    grad_w2 = h_relu.t().mm(grad_y_pred)\n",
    "    grad_h_relu = grad_y_pred.mm(w2.t())\n",
    "    grad_h = grad_h_relu.clone()\n",
    "    grad_h[h < 0] = 0\n",
    "    grad_w1 = x.t().mm(grad_h)\n",
    "\n",
    "    # Update weights using gradient descent\n",
    "    w1 -= learning_rate * grad_w1\n",
    "    w2 -= learning_rate * grad_w2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2+"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}