notebooks/10_residual_network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(\"\"\"Tutorial on how to create a simple residual network with Tensorflow.\n",
    "forked from Parag K. Mital, Jan. 2016\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.examples.tutorials.mnist.input_data as input_data\n",
    "from collections import namedtuple\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def conv2d(x, n_filters, f_h=5, f_w=5,\n",
    "           stride_h=2, stride_w=2,\n",
    "           stddev=0.02,\n",
    "           activation=None,\n",
    "           bias=True,\n",
    "           padding='SAME',\n",
    "           name=\"Conv2D\"):\n",
    "    \"\"\"2D Convolution with options for kernel size, stride, and init deviation.\n",
    "    Parameters\n",
    "    ----------\n",
    "    x : Tensor\n",
    "        Input tensor to convolve.\n",
    "    n_filters : int\n",
    "        Number of filters to apply.\n",
    "    f_h : int, optional\n",
    "        Filter height.\n",
    "    f_w : int, optional\n",
    "        Filter  width.\n",
    "    stride_h : int, optional\n",
    "        Stride in rows.\n",
    "    stride_w : int, optional\n",
    "        Stride in cols.\n",
    "    stddev : float, optional\n",
    "        Initialization's standard deviation.\n",
    "    activation : arguments, optional\n",
    "        Function which applies a nonlinearity\n",
    "    padding : str, optional\n",
    "        'SAME' or 'VALID'\n",
    "    name : str, optional\n",
    "        Variable scope to use.\n",
    "    Returns\n",
    "    -------\n",
    "    x : Tensor\n",
    "        Convolved input.\n",
    "    \"\"\"\n",
    "    with tf.variable_scope(name):\n",
    "        w = tf.get_variable('w', [f_h, f_w, x.get_shape()[-1], n_filters], initializer=tf.truncated_normal_initializer(stddev=stddev))\n",
    "        conv = tf.nn.conv2d(x, w, strides=[1, stride_h, stride_w, 1], padding=padding)\n",
    "        if bias:\n",
    "            b = tf.get_variable('b', [n_filters], initializer=tf.truncated_normal_initializer(stddev=stddev))\n",
    "            conv = tf.nn.bias_add(conv, b)\n",
    "        if activation:\n",
    "            conv = activation(conv)\n",
    "        return conv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def linear(x, n_units, scope=None, stddev=0.02, activation=lambda x: x):\n",
    "    \"\"\"Fully-connected network.\n",
    "    Parameters\n",
    "    ----------\n",
    "    x : Tensor\n",
    "        Input tensor to the network.\n",
    "    n_units : int\n",
    "        Number of units to connect to.\n",
    "    scope : str, optional\n",
    "        Variable scope to use.\n",
    "    stddev : float, optional\n",
    "        Initialization's standard deviation.\n",
    "    activation : arguments, optional\n",
    "        Function which applies a nonlinearity\n",
    "    Returns\n",
    "    -------\n",
    "    x : Tensor\n",
    "        Fully-connected output.\n",
    "    \"\"\"\n",
    "    shape = x.get_shape().as_list()\n",
    "\n",
    "    with tf.variable_scope(scope or \"Linear\"):\n",
    "        matrix = tf.get_variable(\"Matrix\", [shape[-1], n_units], tf.float32, tf.random_normal_initializer(stddev=stddev))\n",
    "        return activation(tf.matmul(x, matrix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def residual_network(x, n_outputs, activation=tf.nn.relu):\n",
    "    \"\"\"Builds a residual network.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    x : Tensor\n",
    "        Input to the network\n",
    "    n_outputs : int\n",
    "        Number of outputs of final softmax\n",
    "    activation : arguments, optional\n",
    "        Nonlinearity function to apply after each convolution\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    net : Tensor\n",
    "        Description\n",
    "\n",
    "    Raises\n",
    "    ------\n",
    "    ValueError\n",
    "        If a 2D Tensor is input, the Tensor must be square or else\n",
    "        the network can't be converted to a 4D Tensor.\n",
    "    \"\"\"\n",
    "\n",
    "    LayerBlock = namedtuple('LayerBlock', ['num_repeats', 'num_filters', 'bottleneck_size'])\n",
    "    blocks = [LayerBlock(3, 128, 32),\n",
    "              LayerBlock(3, 256, 64),\n",
    "              LayerBlock(3, 512, 128),\n",
    "              LayerBlock(3, 1024, 256)]\n",
    "\n",
    "    input_shape = x.get_shape().as_list()\n",
    "    if len(input_shape) == 2:\n",
    "        ndim = int(math.sqrt(input_shape[1]))\n",
    "        if ndim * ndim != input_shape[1]:\n",
    "            raise ValueError('input_shape should be square')\n",
    "        x = tf.reshape(x, [-1, ndim, ndim, 1]) # {(n, H, W, 1) | H == W} else (n, H, W, C)\n",
    "\n",
    "    # First convolution expands to 64 channels and downsamples\n",
    "    net = conv2d(x, 64, f_h=7, f_w=7, name='conv1', activation=activation) # (n, H/2, W/2, 64)\n",
    "\n",
    "    # Max pool and downsampling\n",
    "    net = tf.nn.max_pool(net, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME') # (n, H/4, W/4, 64)\n",
    "\n",
    "    # Setup first chain of resnets\n",
    "    net = conv2d(net, blocks[0].num_filters, f_h=1, f_w=1, stride_h=1, stride_w=1, \n",
    "                 padding='VALID', name='conv2') # (n, H/4, W/4, 128)\n",
    "\n",
    "    # Loop through all res blocks\n",
    "    for block_i, block in enumerate(blocks):\n",
    "        for repeat_i in range(block.num_repeats):\n",
    "            \n",
    "            name = 'block_%d/repeat_%d' % (block_i, repeat_i)\n",
    "            # net: (n, H/4, W/4, 128) (n, H/4, W/4, 256) (n, H/4, W/4, 512) (n, H/4, W/4, 1024)\n",
    "            \n",
    "            conv = conv2d(net, block.bottleneck_size, f_h=1, f_w=1, stride_h=1, stride_w=1,\n",
    "                          activation=activation, padding='VALID', name=name + '/conv_in')\n",
    "            # (n, H/4, W/4, 32) (n, H/4, W/4, 64) (n, H/4, W/4, 128) (n, H/4, W/4, 256)\n",
    "            \n",
    "            conv = conv2d(conv, block.bottleneck_size, f_h=3, f_w=3, stride_h=1, stride_w=1,\n",
    "                          activation=activation, padding='SAME',name=name + '/conv_bottleneck')\n",
    "            # (n, H/4, W/4, 32) (n, H/4, W/4, 64) (n, H/4, W/4, 128) (n, H/4, W/4, 256)\n",
    "            \n",
    "            conv = conv2d(conv, block.num_filters, f_h=1, f_w=1, stride_h=1, stride_w=1,\n",
    "                          activation=activation, padding='VALID', name=name + '/conv_out')\n",
    "            # (n, H/4, W/4, 128) (n, H/4, W/4, 256) (n, H/4, W/4, 512) (n, H/4, W/4, 1024)\n",
    "            \n",
    "            net = conv + net\n",
    "        try:\n",
    "            # upscale to the next block size\n",
    "            next_block = blocks[block_i + 1]\n",
    "            net = conv2d(net, next_block.num_filters, f_h=1, f_w=1, stride_h=1, stride_w=1, \n",
    "                         bias=False, padding='SAME', name='block_%d/conv_upscale' % block_i)\n",
    "            # (n, H/4, W/4, 256) (n, H/4, W/4, 512) (n, H/4, W/4, 1024)\n",
    "        except IndexError:\n",
    "            pass\n",
    "\n",
    "    \n",
    "    net_shape = net.get_shape().as_list() # (n, H/4, W/4, 1024)    \n",
    "    net = tf.nn.avg_pool(net, ksize=[1, net_shape[1], net_shape[2], 1],\n",
    "                         strides=[1, 1, 1, 1], padding='VALID')\n",
    "    \n",
    "    net_shape = net.get_shape().as_list() # (n, 1, 1, 1024)\n",
    "    \n",
    "    net = tf.reshape(net, [-1, net_shape[1] * net_shape[2] * net_shape[3]]) # (n, 1024)\n",
    "    \n",
    "    net = linear(net, n_outputs, activation=tf.nn.softmax) # (n, n_outputs)\n",
    "\n",
    "    return net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def test_mnist():\n",
    "    \"\"\"Test the resnet on MNIST.\"\"\"\n",
    "\n",
    "    mnist = input_data.read_data_sets('MNIST_data/', one_hot=True)\n",
    "    x = tf.placeholder(tf.float32, [None, 784])\n",
    "    y = tf.placeholder(tf.float32, [None, 10])\n",
    "    y_pred = residual_network(x, 10)\n",
    "\n",
    "    # Define loss/eval/training functions\n",
    "    cross_entropy = -tf.reduce_sum(y * tf.log(y_pred))\n",
    "    optimizer = tf.train.AdamOptimizer().minimize(cross_entropy)\n",
    "\n",
    "    # Monitor accuracy\n",
    "    correct_prediction = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float'))\n",
    "\n",
    "    # We now create a new session to actually perform the initialization the\n",
    "    # variables:\n",
    "    sess = tf.Session()\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "\n",
    "    # We'll train in minibatches and report accuracy:\n",
    "    batch_size = 50\n",
    "    n_epochs = 5\n",
    "    for epoch_i in range(n_epochs):\n",
    "        # Training\n",
    "        train_accuracy = 0\n",
    "        for batch_i in range(mnist.train.num_examples // batch_size):\n",
    "            batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "            train_accuracy += sess.run([optimizer, accuracy], feed_dict={x: batch_xs, y: batch_ys})[1]\n",
    "            print(train_accuracy)\n",
    "        train_accuracy /= (mnist.train.num_examples // batch_size)\n",
    "\n",
    "        # Validation\n",
    "        valid_accuracy = 0\n",
    "        for batch_i in range(mnist.validation.num_examples // batch_size):\n",
    "            batch_xs, batch_ys = mnist.validation.next_batch(batch_size)\n",
    "            valid_accuracy += sess.run(accuracy, feed_dict={x: batch_xs, y: batch_ys})\n",
    "        valid_accuracy /= (mnist.validation.num_examples // batch_size)\n",
    "        print('epoch:', epoch_i, ', train:', train_accuracy, ', valid:', valid_accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_mnist()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tensorflow]",
   "language": "python",
   "name": "conda-env-tensorflow-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}