convolutional networks.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zjr/anaconda3/lib/python3.7/site-packages/theano/tensor/nnet/conv.py:98: UserWarning: theano.tensor.nnet.conv.conv2d is deprecated. Use theano.tensor.nnet.conv2d instead.\n",
      "  warnings.warn(\"theano.tensor.nnet.conv.conv2d is deprecated.\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training mini-batch number 0\n",
      "Training mini-batch number 1000\n",
      "Training mini-batch number 2000\n",
      "Training mini-batch number 3000\n",
      "Training mini-batch number 4000\n",
      "Epoch 0: validation accuracy 94.17%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 93.62%\n",
      "Training mini-batch number 5000\n",
      "Training mini-batch number 6000\n",
      "Training mini-batch number 7000\n",
      "Training mini-batch number 8000\n",
      "Training mini-batch number 9000\n",
      "Epoch 1: validation accuracy 96.39%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 96.04%\n",
      "Training mini-batch number 10000\n",
      "Training mini-batch number 11000\n",
      "Training mini-batch number 12000\n",
      "Training mini-batch number 13000\n",
      "Training mini-batch number 14000\n",
      "Epoch 2: validation accuracy 97.21%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 96.88%\n",
      "Training mini-batch number 15000\n",
      "Training mini-batch number 16000\n",
      "Training mini-batch number 17000\n",
      "Training mini-batch number 18000\n",
      "Training mini-batch number 19000\n",
      "Epoch 3: validation accuracy 97.80%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.45%\n",
      "Training mini-batch number 20000\n",
      "Training mini-batch number 21000\n",
      "Training mini-batch number 22000\n",
      "Training mini-batch number 23000\n",
      "Training mini-batch number 24000\n",
      "Epoch 4: validation accuracy 98.08%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.87%\n",
      "Training mini-batch number 25000\n",
      "Training mini-batch number 26000\n",
      "Training mini-batch number 27000\n",
      "Training mini-batch number 28000\n",
      "Training mini-batch number 29000\n",
      "Epoch 5: validation accuracy 98.23%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.04%\n",
      "Training mini-batch number 30000\n",
      "Training mini-batch number 31000\n",
      "Training mini-batch number 32000\n",
      "Training mini-batch number 33000\n",
      "Training mini-batch number 34000\n",
      "Epoch 6: validation accuracy 98.38%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.19%\n",
      "Training mini-batch number 35000\n",
      "Training mini-batch number 36000\n",
      "Training mini-batch number 37000\n",
      "Training mini-batch number 38000\n",
      "Training mini-batch number 39000\n",
      "Epoch 7: validation accuracy 98.41%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.30%\n",
      "Training mini-batch number 40000\n",
      "Training mini-batch number 41000\n",
      "Training mini-batch number 42000\n",
      "Training mini-batch number 43000\n",
      "Training mini-batch number 44000\n",
      "Epoch 8: validation accuracy 98.45%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.43%\n",
      "Training mini-batch number 45000\n",
      "Training mini-batch number 46000\n",
      "Training mini-batch number 47000\n",
      "Training mini-batch number 48000\n",
      "Training mini-batch number 49000\n",
      "Epoch 9: validation accuracy 98.52%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.54%\n",
      "Training mini-batch number 50000\n",
      "Training mini-batch number 51000\n",
      "Training mini-batch number 52000\n",
      "Training mini-batch number 53000\n",
      "Training mini-batch number 54000\n",
      "Epoch 10: validation accuracy 98.55%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.59%\n",
      "Training mini-batch number 55000\n",
      "Training mini-batch number 56000\n",
      "Training mini-batch number 57000\n",
      "Training mini-batch number 58000\n",
      "Training mini-batch number 59000\n",
      "Epoch 11: validation accuracy 98.54%\n",
      "Training mini-batch number 60000\n",
      "Training mini-batch number 61000\n",
      "Training mini-batch number 62000\n",
      "Training mini-batch number 63000\n",
      "Training mini-batch number 64000\n",
      "Epoch 12: validation accuracy 98.56%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.65%\n",
      "Training mini-batch number 65000\n",
      "Training mini-batch number 66000\n",
      "Training mini-batch number 67000\n",
      "Training mini-batch number 68000\n",
      "Training mini-batch number 69000\n",
      "Epoch 13: validation accuracy 98.57%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.66%\n",
      "Training mini-batch number 70000\n",
      "Training mini-batch number 71000\n",
      "Training mini-batch number 72000\n",
      "Training mini-batch number 73000\n",
      "Training mini-batch number 74000\n",
      "Epoch 14: validation accuracy 98.63%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.69%\n",
      "Training mini-batch number 75000\n",
      "Training mini-batch number 76000\n",
      "Training mini-batch number 77000\n",
      "Training mini-batch number 78000\n",
      "Training mini-batch number 79000\n",
      "Epoch 15: validation accuracy 98.70%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.72%\n",
      "Training mini-batch number 80000\n",
      "Training mini-batch number 81000\n",
      "Training mini-batch number 82000\n",
      "Training mini-batch number 83000\n",
      "Training mini-batch number 84000\n",
      "Epoch 16: validation accuracy 98.69%\n",
      "Training mini-batch number 85000\n",
      "Training mini-batch number 86000\n",
      "Training mini-batch number 87000\n",
      "Training mini-batch number 88000\n",
      "Training mini-batch number 89000\n",
      "Epoch 17: validation accuracy 98.70%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.72%\n",
      "Training mini-batch number 90000\n",
      "Training mini-batch number 91000\n",
      "Training mini-batch number 92000\n",
      "Training mini-batch number 93000\n",
      "Training mini-batch number 94000\n",
      "Epoch 18: validation accuracy 98.66%\n",
      "Training mini-batch number 95000\n",
      "Training mini-batch number 96000\n",
      "Training mini-batch number 97000\n",
      "Training mini-batch number 98000\n",
      "Training mini-batch number 99000\n",
      "Epoch 19: validation accuracy 98.66%\n",
      "Training mini-batch number 100000\n",
      "Training mini-batch number 101000\n",
      "Training mini-batch number 102000\n",
      "Training mini-batch number 103000\n",
      "Training mini-batch number 104000\n",
      "Epoch 20: validation accuracy 98.68%\n",
      "Training mini-batch number 105000\n",
      "Training mini-batch number 106000\n",
      "Training mini-batch number 107000\n",
      "Training mini-batch number 108000\n",
      "Training mini-batch number 109000\n",
      "Epoch 21: validation accuracy 98.68%\n",
      "Training mini-batch number 110000\n",
      "Training mini-batch number 111000\n",
      "Training mini-batch number 112000\n",
      "Training mini-batch number 113000\n",
      "Training mini-batch number 114000\n",
      "Epoch 22: validation accuracy 98.68%\n",
      "Training mini-batch number 115000\n",
      "Training mini-batch number 116000\n",
      "Training mini-batch number 117000\n",
      "Training mini-batch number 118000\n",
      "Training mini-batch number 119000\n",
      "Epoch 23: validation accuracy 98.69%\n",
      "Training mini-batch number 120000\n",
      "Training mini-batch number 121000\n",
      "Training mini-batch number 122000\n",
      "Training mini-batch number 123000\n",
      "Training mini-batch number 124000\n",
      "Epoch 24: validation accuracy 98.70%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.77%\n",
      "Training mini-batch number 125000\n",
      "Training mini-batch number 126000\n",
      "Training mini-batch number 127000\n",
      "Training mini-batch number 128000\n",
      "Training mini-batch number 129000\n",
      "Epoch 25: validation accuracy 98.72%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.77%\n",
      "Training mini-batch number 130000\n",
      "Training mini-batch number 131000\n",
      "Training mini-batch number 132000\n",
      "Training mini-batch number 133000\n",
      "Training mini-batch number 134000\n",
      "Epoch 26: validation accuracy 98.71%\n",
      "Training mini-batch number 135000\n",
      "Training mini-batch number 136000\n",
      "Training mini-batch number 137000\n",
      "Training mini-batch number 138000\n",
      "Training mini-batch number 139000\n",
      "Epoch 27: validation accuracy 98.70%\n",
      "Training mini-batch number 140000\n",
      "Training mini-batch number 141000\n",
      "Training mini-batch number 142000\n",
      "Training mini-batch number 143000\n",
      "Training mini-batch number 144000\n",
      "Epoch 28: validation accuracy 98.69%\n",
      "Training mini-batch number 145000\n",
      "Training mini-batch number 146000\n",
      "Training mini-batch number 147000\n",
      "Training mini-batch number 148000\n",
      "Training mini-batch number 149000\n",
      "Epoch 29: validation accuracy 98.70%\n",
      "Finished training network.\n",
      "Best validation accuracy of 98.72% obtained at iteration 129999\n",
      "Corresponding test accuracy of 98.77%\n",
      "Training mini-batch number 0\n",
      "Training mini-batch number 1000\n",
      "Training mini-batch number 2000\n",
      "Training mini-batch number 3000\n",
      "Training mini-batch number 4000\n",
      "Epoch 0: validation accuracy 92.54%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 92.70%\n",
      "Training mini-batch number 5000\n",
      "Training mini-batch number 6000\n",
      "Training mini-batch number 7000\n",
      "Training mini-batch number 8000\n",
      "Training mini-batch number 9000\n",
      "Epoch 1: validation accuracy 96.78%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 96.56%\n",
      "Training mini-batch number 10000\n",
      "Training mini-batch number 11000\n",
      "Training mini-batch number 12000\n",
      "Training mini-batch number 13000\n",
      "Training mini-batch number 14000\n",
      "Epoch 2: validation accuracy 97.57%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.23%\n",
      "Training mini-batch number 15000\n",
      "Training mini-batch number 16000\n",
      "Training mini-batch number 17000\n",
      "Training mini-batch number 18000\n",
      "Training mini-batch number 19000\n",
      "Epoch 3: validation accuracy 97.77%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.58%\n",
      "Training mini-batch number 20000\n",
      "Training mini-batch number 21000\n",
      "Training mini-batch number 22000\n",
      "Training mini-batch number 23000\n",
      "Training mini-batch number 24000\n",
      "Epoch 4: validation accuracy 98.07%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.89%\n",
      "Training mini-batch number 25000\n",
      "Training mini-batch number 26000\n",
      "Training mini-batch number 27000\n",
      "Training mini-batch number 28000\n",
      "Training mini-batch number 29000\n",
      "Epoch 5: validation accuracy 98.19%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.09%\n",
      "Training mini-batch number 30000\n",
      "Training mini-batch number 31000\n",
      "Training mini-batch number 32000\n",
      "Training mini-batch number 33000\n",
      "Training mini-batch number 34000\n",
      "Epoch 6: validation accuracy 98.33%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.17%\n",
      "Training mini-batch number 35000\n",
      "Training mini-batch number 36000\n",
      "Training mini-batch number 37000\n",
      "Training mini-batch number 38000\n",
      "Training mini-batch number 39000\n",
      "Epoch 7: validation accuracy 98.35%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.34%\n",
      "Training mini-batch number 40000\n",
      "Training mini-batch number 41000\n",
      "Training mini-batch number 42000\n",
      "Training mini-batch number 43000\n",
      "Training mini-batch number 44000\n",
      "Epoch 8: validation accuracy 98.37%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.37%\n",
      "Training mini-batch number 45000\n",
      "Training mini-batch number 46000\n",
      "Training mini-batch number 47000\n",
      "Training mini-batch number 48000\n",
      "Training mini-batch number 49000\n",
      "Epoch 9: validation accuracy 98.48%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.47%\n",
      "Training mini-batch number 50000\n",
      "Training mini-batch number 51000\n",
      "Training mini-batch number 52000\n",
      "Training mini-batch number 53000\n",
      "Training mini-batch number 54000\n",
      "Epoch 10: validation accuracy 98.54%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.56%\n",
      "Training mini-batch number 55000\n",
      "Training mini-batch number 56000\n",
      "Training mini-batch number 57000\n",
      "Training mini-batch number 58000\n",
      "Training mini-batch number 59000\n",
      "Epoch 11: validation accuracy 98.66%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.57%\n",
      "Training mini-batch number 60000\n",
      "Training mini-batch number 61000\n",
      "Training mini-batch number 62000\n",
      "Training mini-batch number 63000\n",
      "Training mini-batch number 64000\n",
      "Epoch 12: validation accuracy 98.66%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.67%\n",
      "Training mini-batch number 65000\n",
      "Training mini-batch number 66000\n",
      "Training mini-batch number 67000\n",
      "Training mini-batch number 68000\n",
      "Training mini-batch number 69000\n",
      "Epoch 13: validation accuracy 98.69%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.67%\n",
      "Training mini-batch number 70000\n",
      "Training mini-batch number 71000\n",
      "Training mini-batch number 72000\n",
      "Training mini-batch number 73000\n",
      "Training mini-batch number 74000\n",
      "Epoch 14: validation accuracy 98.70%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.75%\n",
      "Training mini-batch number 75000\n",
      "Training mini-batch number 76000\n",
      "Training mini-batch number 77000\n",
      "Training mini-batch number 78000\n",
      "Training mini-batch number 79000\n",
      "Epoch 15: validation accuracy 98.68%\n",
      "Training mini-batch number 80000\n",
      "Training mini-batch number 81000\n",
      "Training mini-batch number 82000\n",
      "Training mini-batch number 83000\n",
      "Training mini-batch number 84000\n",
      "Epoch 16: validation accuracy 98.70%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.77%\n",
      "Training mini-batch number 85000\n",
      "Training mini-batch number 86000\n",
      "Training mini-batch number 87000\n",
      "Training mini-batch number 88000\n",
      "Training mini-batch number 89000\n",
      "Epoch 17: validation accuracy 98.70%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.81%\n",
      "Training mini-batch number 90000\n",
      "Training mini-batch number 91000\n",
      "Training mini-batch number 92000\n",
      "Training mini-batch number 93000\n",
      "Training mini-batch number 94000\n",
      "Epoch 18: validation accuracy 98.77%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.81%\n",
      "Training mini-batch number 95000\n",
      "Training mini-batch number 96000\n",
      "Training mini-batch number 97000\n",
      "Training mini-batch number 98000\n",
      "Training mini-batch number 99000\n",
      "Epoch 19: validation accuracy 98.79%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.82%\n",
      "Training mini-batch number 100000\n",
      "Training mini-batch number 101000\n",
      "Training mini-batch number 102000\n",
      "Training mini-batch number 103000\n",
      "Training mini-batch number 104000\n",
      "Epoch 20: validation accuracy 98.81%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.79%\n",
      "Training mini-batch number 105000\n",
      "Training mini-batch number 106000\n",
      "Training mini-batch number 107000\n",
      "Training mini-batch number 108000\n",
      "Training mini-batch number 109000\n",
      "Epoch 21: validation accuracy 98.83%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.80%\n",
      "Training mini-batch number 110000\n",
      "Training mini-batch number 111000\n",
      "Training mini-batch number 112000\n",
      "Training mini-batch number 113000\n",
      "Training mini-batch number 114000\n",
      "Epoch 22: validation accuracy 98.82%\n",
      "Training mini-batch number 115000\n",
      "Training mini-batch number 116000\n",
      "Training mini-batch number 117000\n",
      "Training mini-batch number 118000\n",
      "Training mini-batch number 119000\n",
      "Epoch 23: validation accuracy 98.84%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.88%\n",
      "Training mini-batch number 120000\n",
      "Training mini-batch number 121000\n",
      "Training mini-batch number 122000\n",
      "Training mini-batch number 123000\n",
      "Training mini-batch number 124000\n",
      "Epoch 24: validation accuracy 98.87%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.88%\n",
      "Training mini-batch number 125000\n",
      "Training mini-batch number 126000\n",
      "Training mini-batch number 127000\n",
      "Training mini-batch number 128000\n",
      "Training mini-batch number 129000\n",
      "Epoch 25: validation accuracy 98.88%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.91%\n",
      "Training mini-batch number 130000\n",
      "Training mini-batch number 131000\n",
      "Training mini-batch number 132000\n",
      "Training mini-batch number 133000\n",
      "Training mini-batch number 134000\n",
      "Epoch 26: validation accuracy 98.89%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.93%\n",
      "Training mini-batch number 135000\n",
      "Training mini-batch number 136000\n",
      "Training mini-batch number 137000\n",
      "Training mini-batch number 138000\n",
      "Training mini-batch number 139000\n",
      "Epoch 27: validation accuracy 98.89%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.94%\n",
      "Training mini-batch number 140000\n",
      "Training mini-batch number 141000\n",
      "Training mini-batch number 142000\n",
      "Training mini-batch number 143000\n",
      "Training mini-batch number 144000\n",
      "Epoch 28: validation accuracy 98.91%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.95%\n",
      "Training mini-batch number 145000\n",
      "Training mini-batch number 146000\n",
      "Training mini-batch number 147000\n",
      "Training mini-batch number 148000\n",
      "Training mini-batch number 149000\n",
      "Epoch 29: validation accuracy 98.94%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.96%\n",
      "Finished training network.\n",
      "Best validation accuracy of 98.94% obtained at iteration 149999\n",
      "Corresponding test accuracy of 98.96%\n",
      "Training mini-batch number 0\n",
      "Training mini-batch number 1000\n",
      "Training mini-batch number 2000\n",
      "Training mini-batch number 3000\n",
      "Training mini-batch number 4000\n",
      "Epoch 0: validation accuracy 97.16%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 96.95%\n",
      "Training mini-batch number 5000\n",
      "Training mini-batch number 6000\n",
      "Training mini-batch number 7000\n",
      "Training mini-batch number 8000\n",
      "Training mini-batch number 9000\n",
      "Epoch 1: validation accuracy 97.87%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.68%\n",
      "Training mini-batch number 10000\n",
      "Training mini-batch number 11000\n",
      "Training mini-batch number 12000\n",
      "Training mini-batch number 13000\n",
      "Training mini-batch number 14000\n",
      "Epoch 2: validation accuracy 98.28%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.14%\n",
      "Training mini-batch number 15000\n",
      "Training mini-batch number 16000\n",
      "Training mini-batch number 17000\n",
      "Training mini-batch number 18000\n",
      "Training mini-batch number 19000\n",
      "Epoch 3: validation accuracy 98.13%\n",
      "Training mini-batch number 20000\n",
      "Training mini-batch number 21000\n",
      "Training mini-batch number 22000\n",
      "Training mini-batch number 23000\n",
      "Training mini-batch number 24000\n",
      "Epoch 4: validation accuracy 98.26%\n",
      "Training mini-batch number 25000\n",
      "Training mini-batch number 26000\n",
      "Training mini-batch number 27000\n",
      "Training mini-batch number 28000\n",
      "Training mini-batch number 29000\n",
      "Epoch 5: validation accuracy 98.45%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.31%\n",
      "Training mini-batch number 30000\n",
      "Training mini-batch number 31000\n",
      "Training mini-batch number 32000\n",
      "Training mini-batch number 33000\n",
      "Training mini-batch number 34000\n",
      "Epoch 6: validation accuracy 98.39%\n",
      "Training mini-batch number 35000\n",
      "Training mini-batch number 36000\n",
      "Training mini-batch number 37000\n",
      "Training mini-batch number 38000\n",
      "Training mini-batch number 39000\n",
      "Epoch 7: validation accuracy 98.34%\n",
      "Training mini-batch number 40000\n",
      "Training mini-batch number 41000\n",
      "Training mini-batch number 42000\n",
      "Training mini-batch number 43000\n",
      "Training mini-batch number 44000\n",
      "Epoch 8: validation accuracy 98.56%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.39%\n",
      "Training mini-batch number 45000\n",
      "Training mini-batch number 46000\n",
      "Training mini-batch number 47000\n",
      "Training mini-batch number 48000\n",
      "Training mini-batch number 49000\n",
      "Epoch 9: validation accuracy 98.45%\n",
      "Training mini-batch number 50000\n",
      "Training mini-batch number 51000\n",
      "Training mini-batch number 52000\n",
      "Training mini-batch number 53000\n",
      "Training mini-batch number 54000\n",
      "Epoch 10: validation accuracy 98.64%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.40%\n",
      "Training mini-batch number 55000\n",
      "Training mini-batch number 56000\n",
      "Training mini-batch number 57000\n",
      "Training mini-batch number 58000\n",
      "Training mini-batch number 59000\n",
      "Epoch 11: validation accuracy 98.73%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.68%\n",
      "Training mini-batch number 60000\n",
      "Training mini-batch number 61000\n",
      "Training mini-batch number 62000\n",
      "Training mini-batch number 63000\n",
      "Training mini-batch number 64000\n",
      "Epoch 12: validation accuracy 98.46%\n",
      "Training mini-batch number 65000\n",
      "Training mini-batch number 66000\n",
      "Training mini-batch number 67000\n",
      "Training mini-batch number 68000\n",
      "Training mini-batch number 69000\n",
      "Epoch 13: validation accuracy 98.75%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.56%\n",
      "Training mini-batch number 70000\n",
      "Training mini-batch number 71000\n",
      "Training mini-batch number 72000\n",
      "Training mini-batch number 73000\n",
      "Training mini-batch number 74000\n",
      "Epoch 14: validation accuracy 98.66%\n",
      "Training mini-batch number 75000\n",
      "Training mini-batch number 76000\n",
      "Training mini-batch number 77000\n",
      "Training mini-batch number 78000\n",
      "Training mini-batch number 79000\n",
      "Epoch 15: validation accuracy 98.77%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.69%\n",
      "Training mini-batch number 80000\n",
      "Training mini-batch number 81000\n",
      "Training mini-batch number 82000\n",
      "Training mini-batch number 83000\n",
      "Training mini-batch number 84000\n",
      "Epoch 16: validation accuracy 98.79%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.72%\n",
      "Training mini-batch number 85000\n",
      "Training mini-batch number 86000\n",
      "Training mini-batch number 87000\n",
      "Training mini-batch number 88000\n",
      "Training mini-batch number 89000\n",
      "Epoch 17: validation accuracy 98.99%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.73%\n",
      "Training mini-batch number 90000\n",
      "Training mini-batch number 91000\n",
      "Training mini-batch number 92000\n",
      "Training mini-batch number 93000\n",
      "Training mini-batch number 94000\n",
      "Epoch 18: validation accuracy 98.98%\n",
      "Training mini-batch number 95000\n",
      "Training mini-batch number 96000\n",
      "Training mini-batch number 97000\n",
      "Training mini-batch number 98000\n",
      "Training mini-batch number 99000\n",
      "Epoch 19: validation accuracy 98.90%\n",
      "Training mini-batch number 100000\n",
      "Training mini-batch number 101000\n",
      "Training mini-batch number 102000\n",
      "Training mini-batch number 103000\n",
      "Training mini-batch number 104000\n",
      "Epoch 20: validation accuracy 98.76%\n",
      "Training mini-batch number 105000\n",
      "Training mini-batch number 106000\n",
      "Training mini-batch number 107000\n",
      "Training mini-batch number 108000\n",
      "Training mini-batch number 109000\n",
      "Epoch 21: validation accuracy 98.81%\n",
      "Training mini-batch number 110000\n",
      "Training mini-batch number 111000\n",
      "Training mini-batch number 112000\n",
      "Training mini-batch number 113000\n",
      "Training mini-batch number 114000\n",
      "Epoch 22: validation accuracy 99.05%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 98.93%\n",
      "Training mini-batch number 115000\n",
      "Training mini-batch number 116000\n",
      "Training mini-batch number 117000\n",
      "Training mini-batch number 118000\n",
      "Training mini-batch number 119000\n",
      "Epoch 23: validation accuracy 98.89%\n",
      "Training mini-batch number 120000\n",
      "Training mini-batch number 121000\n",
      "Training mini-batch number 122000\n",
      "Training mini-batch number 123000\n",
      "Training mini-batch number 124000\n",
      "Epoch 24: validation accuracy 98.91%\n",
      "Training mini-batch number 125000\n",
      "Training mini-batch number 126000\n",
      "Training mini-batch number 127000\n",
      "Training mini-batch number 128000\n",
      "Training mini-batch number 129000\n",
      "Epoch 25: validation accuracy 98.70%\n",
      "Training mini-batch number 130000\n",
      "Training mini-batch number 131000\n",
      "Training mini-batch number 132000\n",
      "Training mini-batch number 133000\n",
      "Training mini-batch number 134000\n",
      "Epoch 26: validation accuracy 98.99%\n",
      "Training mini-batch number 135000\n",
      "Training mini-batch number 136000\n",
      "Training mini-batch number 137000\n",
      "Training mini-batch number 138000\n",
      "Training mini-batch number 139000\n",
      "Epoch 27: validation accuracy 99.00%\n",
      "Training mini-batch number 140000\n",
      "Training mini-batch number 141000\n",
      "Training mini-batch number 142000\n",
      "Training mini-batch number 143000\n",
      "Training mini-batch number 144000\n",
      "Epoch 28: validation accuracy 98.99%\n",
      "Training mini-batch number 145000\n",
      "Training mini-batch number 146000\n",
      "Training mini-batch number 147000\n",
      "Training mini-batch number 148000\n",
      "Training mini-batch number 149000\n",
      "Epoch 29: validation accuracy 98.95%\n",
      "Finished training network.\n",
      "Best validation accuracy of 99.05% obtained at iteration 114999\n",
      "Corresponding test accuracy of 98.93%\n"
     ]
    }
   ],
   "source": [
    "import network3\n",
    "from network3 import Network, ConvPoolLayer, FullyConnectedLayer, SoftmaxLayer # import进行模块导入\n",
    "#Softmax函数将多个标量映射为一个概率分布，其输出的每一个值范围在(0,1)，作为神经网络的最后一层，输出层实现多分类\n",
    "# read data:\n",
    "training_data, validation_data, test_data = network3.load_data_shared()\n",
    "# mini-batch size:\n",
    "mini_batch_size = 10\n",
    "\n",
    "# chapter 6 - 一个隐藏层，中间全连接\n",
    "\n",
    "net = Network([\n",
    "    FullyConnectedLayer(n_in=784, n_out=100),\n",
    "    SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size)\n",
    "\n",
    "\n",
    "# chapter 6 - 加入卷积池化层，5x5 local receptive fields（局部感受野）,\n",
    "# 20 feature maps（不同卷积核提取出的不同特征）, max-pooling layer 2x2（即对2x2邻域内特征点取最大，能很好的保留纹理特征）\n",
    "\n",
    "net1 = Network([\n",
    "    ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),\n",
    "                  filter_shape=(20, 1, 5, 5),\n",
    "                  poolsize=(2, 2)),\n",
    "    FullyConnectedLayer(n_in=20*12*12, n_out=100),\n",
    "    SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size)\n",
    "net1.SGD(training_data, 30, mini_batch_size, 0.1, validation_data, test_data)\n",
    "\n",
    "# chapter 6 - inserting a second convolutional-pooling layer to the previous example => better accuracy\n",
    "\n",
    "net2 = Network([\n",
    "    ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),\n",
    "                  filter_shape=(20, 1, 5, 5),\n",
    "                  poolsize=(2, 2)),\n",
    "    ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),\n",
    "                  filter_shape=(40, 20, 5, 5),\n",
    "                  poolsize=(2, 2)),\n",
    "    FullyConnectedLayer(n_in=40*4*4, n_out=100),\n",
    "    SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size)\n",
    "net2.SGD(training_data, 30, mini_batch_size, 0.1, validation_data, test_data)\n",
    "\n",
    "\n",
    "# chapter 6 -  rectified linear units and some l2 regularization (lmbda=0.1) => even better accuracy\n",
    "# sigmoid 函数的优点：输出映射在(0,1)内，单调连续，适合用作输出层，求导比较容易；缺点：软饱和性，一旦输入落入饱和区， f'(x)就会变得接近于 0，很容易产生梯度消失。\n",
    "# tanh 函数也具有软饱和性。因为它的输出以 0 为中心，收敛速度比 sigmoid 要快。\n",
    "# 改变激活函数，ReLU相对于Sigmoid一方面大大降低运算（生物运算也有成本）；另一方面在输入信号较强时，仍然能够保留信号之间的差别。\n",
    "\n",
    "from network3 import ReLU\n",
    "net3 = Network([\n",
    "    ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),\n",
    "                  filter_shape=(20, 1, 5, 5),\n",
    "                  poolsize=(2, 2),\n",
    "                  activation_fn=ReLU),\n",
    "    ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),\n",
    "                  filter_shape=(40, 20, 5, 5),\n",
    "                  poolsize=(2, 2),\n",
    "                  activation_fn=ReLU),\n",
    "    FullyConnectedLayer(n_in=40*4*4, n_out=100, activation_fn=ReLU),\n",
    "    SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size)\n",
    "net3.SGD(training_data, 30, mini_batch_size, 0.03, validation_data, test_data, lmbda=0.1)\n",
    "#卷积实际上就是加法和乘法的组合，训练过程等同与训练每一个卷积层的滤波器，，让其对于特定特征具有提取功能，选取卷积核就是在对数据的不同特征进行选取，\n",
    "#全连接层之前的作用是提取特征，全连接层的作用是分类\n",
    "#反向传播步骤：进行前馈传导运算，求输出层激活值；对每个输出单元计算残差；节点处残差计算；计算偏导"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<network3.Network object at 0x7f7db7840790>\n",
      "<network3.Network object at 0x7f7dbe785450>\n",
      "<network3.Network object at 0x7f7dbe6dc050>\n",
      "<network3.Network object at 0x7f7db6a25b10>\n"
     ]
    }
   ],
   "source": [
    "print(net)\n",
    "print(net1)\n",
    "print(net2)\n",
    "print(net3)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}