Yolo Step-by-Step.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Outline of Steps**\n",
    "    + Initialization\n",
    "        + Download COCO detection data from http://cocodataset.org/#download\n",
    "            + http://images.cocodataset.org/zips/train2014.zip <= train images\n",
    "            + http://images.cocodataset.org/zips/val2014.zip <= validation images\n",
    "            + http://images.cocodataset.org/annotations/annotations_trainval2014.zip <= train and validation annotations\n",
    "        + Run this script to convert annotations in COCO format to VOC format\n",
    "            + https://gist.github.com/chicham/6ed3842d0d2014987186#file-coco2pascal-py\n",
    "        + Download pre-trained weights from https://pjreddie.com/darknet/yolo/\n",
    "            + https://pjreddie.com/media/files/yolo.weights\n",
    "        + Specify the directory of train annotations (train_annot_folder) and train images (train_image_folder)\n",
    "        + Specify the directory of validation annotations (valid_annot_folder) and validation images (valid_image_folder)\n",
    "        + Specity the path of pre-trained weights by setting variable *wt_path*\n",
    "    + Construct equivalent network in Keras\n",
    "        + Network arch from https://github.com/pjreddie/darknet/blob/master/cfg/yolo-voc.cfg\n",
    "    + Load the pretrained weights\n",
    "    + Perform training \n",
    "    + Perform detection on an image with newly trained weights\n",
    "    + Perform detection on an video with newly trained weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-04T00:18:52.056478",
     "start_time": "2018-04-04T00:18:50.879887"
    },
    "code_folding": [],
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zachary_arundel/freightkeras/env/lib/python3.5/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Reshape, Activation, Conv2D, Input, MaxPooling2D, BatchNormalization, Flatten, Dense, Lambda\n",
    "from keras.layers.advanced_activations import LeakyReLU\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint, TensorBoard\n",
    "from keras.optimizers import SGD, Adam, RMSprop\n",
    "from keras.layers.merge import concatenate\n",
    "import matplotlib.pyplot as plt\n",
    "import keras.backend as K\n",
    "import tensorflow as tf\n",
    "import imgaug as ia\n",
    "from tqdm import tqdm\n",
    "from imgaug import augmenters as iaa\n",
    "import numpy as np\n",
    "import pickle\n",
    "import os, cv2\n",
    "from preprocessing import parse_annotation, BatchGenerator\n",
    "from utils import WeightReader, decode_netout, draw_boxes\n",
    "\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-04T00:18:52.075535",
     "start_time": "2018-04-04T00:18:52.057712"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "LABELS = [\"car\", \"truck\", \"pickup\", \"tractor\", \"camping car\", \"boat\",\"motorcycle\", \"van\", \"other\", \"plane\"] \n",
    "\n",
    "IMAGE_H, IMAGE_W = 416, 416\n",
    "GRID_H,  GRID_W  = 13 , 13\n",
    "BOX              = 5\n",
    "CLASS            = len(LABELS)\n",
    "CLASS_WEIGHTS    = np.ones(CLASS, dtype='float32')\n",
    "OBJ_THRESHOLD    = 0.3#0.5\n",
    "NMS_THRESHOLD    = 0.3#0.45\n",
    "ANCHORS          = [0.88,1.69, 1.18,0.7, 1.65,1.77,1.77,0.9, 3.75, 3.57],\n",
    "\n",
    "NO_OBJECT_SCALE  = 1.0\n",
    "OBJECT_SCALE     = 5.0\n",
    "COORD_SCALE      = 1.0\n",
    "CLASS_SCALE      = 1.0\n",
    "\n",
    "BATCH_SIZE       = 16\n",
    "WARM_UP_BATCHES  = 0\n",
    "TRUE_BOX_BUFFER  = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-04T00:18:52.981155",
     "start_time": "2018-04-04T00:18:52.978076"
    }
   },
   "outputs": [],
   "source": [
    "wt_path = 'full_yolo_backend.h5'                      \n",
    "train_image_folder = 'train_image_folder/'\n",
    "train_annot_folder = 'train_annot_folder/'\n",
    "valid_image_folder = 'valid_image_folder/'\n",
    "valid_annot_folder = 'valid_annot_folder/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Construct the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-04T00:18:53.978220",
     "start_time": "2018-04-04T00:18:53.967537"
    }
   },
   "outputs": [],
   "source": [
    "# the function to implement the orgnization layer (thanks to github.com/allanzelener/YAD2K)\n",
    "def space_to_depth_x2(x):\n",
    "    return tf.space_to_depth(x, block_size=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-04T00:18:58.022959",
     "start_time": "2018-04-04T00:18:55.740759"
    },
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "input_image = Input(shape=(IMAGE_H, IMAGE_W, 3))\n",
    "true_boxes  = Input(shape=(1, 1, 1, TRUE_BOX_BUFFER , 4))\n",
    "\n",
    "# Layer 1\n",
    "x = Conv2D(32, (3,3), strides=(1,1), padding='same', name='conv_1', use_bias=False)(input_image)\n",
    "x = BatchNormalization(name='norm_1')(x)\n",
    "x = LeakyReLU(alpha=0.1)(x)\n",
    "x = MaxPooling2D(pool_size=(2, 2))(x)\n",
    "\n",
    "# Layer 2\n",
    "x = Conv2D(64, (3,3), strides=(1,1), padding='same', name='conv_2', use_bias=False)(x)\n",
    "x = BatchNormalization(name='norm_2')(x)\n",
    "x = LeakyReLU(alpha=0.1)(x)\n",
    "x = MaxPooling2D(pool_size=(2, 2))(x)\n",
    "\n",
    "# Layer 3\n",
    "x = Conv2D(128, (3,3), strides=(1,1), padding='same', name='conv_3', use_bias=False)(x)\n",
    "x = BatchNormalization(name='norm_3')(x)\n",
    "x = LeakyReLU(alpha=0.1)(x)\n",
    "\n",
    "# Layer 4\n",
    "x = Conv2D(64, (1,1), strides=(1,1), padding='same', name='conv_4', use_bias=False)(x)\n",
    "x = BatchNormalization(name='norm_4')(x)\n",
    "x = LeakyReLU(alpha=0.1)(x)\n",
    "\n",
    "# Layer 5\n",
    "x = Conv2D(128, (3,3), strides=(1,1), padding='same', name='conv_5', use_bias=False)(x)\n",
    "x = BatchNormalization(name='norm_5')(x)\n",
    "x = LeakyReLU(alpha=0.1)(x)\n",
    "x = MaxPooling2D(pool_size=(2, 2))(x)\n",
    "\n",
    "# Layer 6\n",
    "x = Conv2D(256, (3,3), strides=(1,1), padding='same', name='conv_6', use_bias=False)(x)\n",
    "x = BatchNormalization(name='norm_6')(x)\n",
    "x = LeakyReLU(alpha=0.1)(x)\n",
    "\n",
    "# Layer 7\n",
    "x = Conv2D(128, (1,1), strides=(1,1), padding='same', name='conv_7', use_bias=False)(x)\n",
    "x = BatchNormalization(name='norm_7')(x)\n",
    "x = LeakyReLU(alpha=0.1)(x)\n",
    "\n",
    "# Layer 8\n",
    "x = Conv2D(256, (3,3), strides=(1,1), padding='same', name='conv_8', use_bias=False)(x)\n",
    "x = BatchNormalization(name='norm_8')(x)\n",
    "x = LeakyReLU(alpha=0.1)(x)\n",
    "x = MaxPooling2D(pool_size=(2, 2))(x)\n",
    "\n",
    "# Layer 9\n",
    "x = Conv2D(512, (3,3), strides=(1,1), padding='same', name='conv_9', use_bias=False)(x)\n",
    "x = BatchNormalization(name='norm_9')(x)\n",
    "x = LeakyReLU(alpha=0.1)(x)\n",
    "\n",
    "# Layer 10\n",
    "x = Conv2D(256, (1,1), strides=(1,1), padding='same', name='conv_10', use_bias=False)(x)\n",
    "x = BatchNormalization(name='norm_10')(x)\n",
    "x = LeakyReLU(alpha=0.1)(x)\n",
    "\n",
    "# Layer 11\n",
    "x = Conv2D(512, (3,3), strides=(1,1), padding='same', name='conv_11', use_bias=False)(x)\n",
    "x = BatchNormalization(name='norm_11')(x)\n",
    "x = LeakyReLU(alpha=0.1)(x)\n",
    "\n",
    "# Layer 12\n",
    "x = Conv2D(256, (1,1), strides=(1,1), padding='same', name='conv_12', use_bias=False)(x)\n",
    "x = BatchNormalization(name='norm_12')(x)\n",
    "x = LeakyReLU(alpha=0.1)(x)\n",
    "\n",
    "# Layer 13\n",
    "x = Conv2D(512, (3,3), strides=(1,1), padding='same', name='conv_13', use_bias=False)(x)\n",
    "x = BatchNormalization(name='norm_13')(x)\n",
    "x = LeakyReLU(alpha=0.1)(x)\n",
    "\n",
    "skip_connection = x\n",
    "\n",
    "x = MaxPooling2D(pool_size=(2, 2))(x)\n",
    "\n",
    "# Layer 14\n",
    "x = Conv2D(1024, (3,3), strides=(1,1), padding='same', name='conv_14', use_bias=False)(x)\n",
    "x = BatchNormalization(name='norm_14')(x)\n",
    "x = LeakyReLU(alpha=0.1)(x)\n",
    "\n",
    "# Layer 15\n",
    "x = Conv2D(512, (1,1), strides=(1,1), padding='same', name='conv_15', use_bias=False)(x)\n",
    "x = BatchNormalization(name='norm_15')(x)\n",
    "x = LeakyReLU(alpha=0.1)(x)\n",
    "\n",
    "# Layer 16\n",
    "x = Conv2D(1024, (3,3), strides=(1,1), padding='same', name='conv_16', use_bias=False)(x)\n",
    "x = BatchNormalization(name='norm_16')(x)\n",
    "x = LeakyReLU(alpha=0.1)(x)\n",
    "\n",
    "# Layer 17\n",
    "x = Conv2D(512, (1,1), strides=(1,1), padding='same', name='conv_17', use_bias=False)(x)\n",
    "x = BatchNormalization(name='norm_17')(x)\n",
    "x = LeakyReLU(alpha=0.1)(x)\n",
    "\n",
    "# Layer 18\n",
    "x = Conv2D(1024, (3,3), strides=(1,1), padding='same', name='conv_18', use_bias=False)(x)\n",
    "x = BatchNormalization(name='norm_18')(x)\n",
    "x = LeakyReLU(alpha=0.1)(x)\n",
    "\n",
    "# Layer 19\n",
    "x = Conv2D(1024, (3,3), strides=(1,1), padding='same', name='conv_19', use_bias=False)(x)\n",
    "x = BatchNormalization(name='norm_19')(x)\n",
    "x = LeakyReLU(alpha=0.1)(x)\n",
    "\n",
    "# Layer 20\n",
    "x = Conv2D(1024, (3,3), strides=(1,1), padding='same', name='conv_20', use_bias=False)(x)\n",
    "x = BatchNormalization(name='norm_20')(x)\n",
    "x = LeakyReLU(alpha=0.1)(x)\n",
    "\n",
    "# Layer 21\n",
    "skip_connection = Conv2D(64, (1,1), strides=(1,1), padding='same', name='conv_21', use_bias=False)(skip_connection)\n",
    "skip_connection = BatchNormalization(name='norm_21')(skip_connection)\n",
    "skip_connection = LeakyReLU(alpha=0.1)(skip_connection)\n",
    "skip_connection = Lambda(space_to_depth_x2)(skip_connection)\n",
    "\n",
    "x = concatenate([skip_connection, x])\n",
    "\n",
    "# Layer 22\n",
    "x = Conv2D(1024, (3,3), strides=(1,1), padding='same', name='conv_22', use_bias=False)(x)\n",
    "x = BatchNormalization(name='norm_22')(x)\n",
    "x = LeakyReLU(alpha=0.1)(x)\n",
    "\n",
    "# Layer 23\n",
    "x = Conv2D(BOX * (4 + 1 + CLASS), (1,1), strides=(1,1), padding='same', name='conv_23')(x)\n",
    "output = Reshape((GRID_H, GRID_W, BOX, 4 + 1 + CLASS))(x)\n",
    "\n",
    "# small hack to allow true_boxes to be registered when Keras build the model \n",
    "# for more information: https://github.com/fchollet/keras/issues/2790\n",
    "output = Lambda(lambda args: args[0])([output, true_boxes])\n",
    "\n",
    "model = Model([input_image, true_boxes], output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-26T12:34:03.819802Z",
     "start_time": "2017-11-26T12:34:03.786125Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_1 (InputLayer)             (None, 416, 416, 3)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv_1 (Conv2D)                  (None, 416, 416, 32)  864         input_1[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "norm_1 (BatchNormalization)      (None, 416, 416, 32)  128         conv_1[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_1 (LeakyReLU)        (None, 416, 416, 32)  0           norm_1[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2D)   (None, 208, 208, 32)  0           leaky_re_lu_1[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "conv_2 (Conv2D)                  (None, 208, 208, 64)  18432       max_pooling2d_1[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "norm_2 (BatchNormalization)      (None, 208, 208, 64)  256         conv_2[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_2 (LeakyReLU)        (None, 208, 208, 64)  0           norm_2[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2D)   (None, 104, 104, 64)  0           leaky_re_lu_2[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "conv_3 (Conv2D)                  (None, 104, 104, 128) 73728       max_pooling2d_2[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "norm_3 (BatchNormalization)      (None, 104, 104, 128) 512         conv_3[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_3 (LeakyReLU)        (None, 104, 104, 128) 0           norm_3[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "conv_4 (Conv2D)                  (None, 104, 104, 64)  8192        leaky_re_lu_3[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "norm_4 (BatchNormalization)      (None, 104, 104, 64)  256         conv_4[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_4 (LeakyReLU)        (None, 104, 104, 64)  0           norm_4[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "conv_5 (Conv2D)                  (None, 104, 104, 128) 73728       leaky_re_lu_4[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "norm_5 (BatchNormalization)      (None, 104, 104, 128) 512         conv_5[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_5 (LeakyReLU)        (None, 104, 104, 128) 0           norm_5[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2D)   (None, 52, 52, 128)   0           leaky_re_lu_5[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "conv_6 (Conv2D)                  (None, 52, 52, 256)   294912      max_pooling2d_3[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "norm_6 (BatchNormalization)      (None, 52, 52, 256)   1024        conv_6[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_6 (LeakyReLU)        (None, 52, 52, 256)   0           norm_6[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "conv_7 (Conv2D)                  (None, 52, 52, 128)   32768       leaky_re_lu_6[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "norm_7 (BatchNormalization)      (None, 52, 52, 128)   512         conv_7[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_7 (LeakyReLU)        (None, 52, 52, 128)   0           norm_7[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "conv_8 (Conv2D)                  (None, 52, 52, 256)   294912      leaky_re_lu_7[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "norm_8 (BatchNormalization)      (None, 52, 52, 256)   1024        conv_8[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_8 (LeakyReLU)        (None, 52, 52, 256)   0           norm_8[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling2d_4 (MaxPooling2D)   (None, 26, 26, 256)   0           leaky_re_lu_8[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "conv_9 (Conv2D)                  (None, 26, 26, 512)   1179648     max_pooling2d_4[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "norm_9 (BatchNormalization)      (None, 26, 26, 512)   2048        conv_9[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_9 (LeakyReLU)        (None, 26, 26, 512)   0           norm_9[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "conv_10 (Conv2D)                 (None, 26, 26, 256)   131072      leaky_re_lu_9[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "norm_10 (BatchNormalization)     (None, 26, 26, 256)   1024        conv_10[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_10 (LeakyReLU)       (None, 26, 26, 256)   0           norm_10[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "conv_11 (Conv2D)                 (None, 26, 26, 512)   1179648     leaky_re_lu_10[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "norm_11 (BatchNormalization)     (None, 26, 26, 512)   2048        conv_11[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_11 (LeakyReLU)       (None, 26, 26, 512)   0           norm_11[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "conv_12 (Conv2D)                 (None, 26, 26, 256)   131072      leaky_re_lu_11[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "norm_12 (BatchNormalization)     (None, 26, 26, 256)   1024        conv_12[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_12 (LeakyReLU)       (None, 26, 26, 256)   0           norm_12[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "conv_13 (Conv2D)                 (None, 26, 26, 512)   1179648     leaky_re_lu_12[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "norm_13 (BatchNormalization)     (None, 26, 26, 512)   2048        conv_13[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_13 (LeakyReLU)       (None, 26, 26, 512)   0           norm_13[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling2d_5 (MaxPooling2D)   (None, 13, 13, 512)   0           leaky_re_lu_13[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "conv_14 (Conv2D)                 (None, 13, 13, 1024)  4718592     max_pooling2d_5[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "norm_14 (BatchNormalization)     (None, 13, 13, 1024)  4096        conv_14[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_14 (LeakyReLU)       (None, 13, 13, 1024)  0           norm_14[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "conv_15 (Conv2D)                 (None, 13, 13, 512)   524288      leaky_re_lu_14[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "norm_15 (BatchNormalization)     (None, 13, 13, 512)   2048        conv_15[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_15 (LeakyReLU)       (None, 13, 13, 512)   0           norm_15[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "conv_16 (Conv2D)                 (None, 13, 13, 1024)  4718592     leaky_re_lu_15[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "norm_16 (BatchNormalization)     (None, 13, 13, 1024)  4096        conv_16[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_16 (LeakyReLU)       (None, 13, 13, 1024)  0           norm_16[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "conv_17 (Conv2D)                 (None, 13, 13, 512)   524288      leaky_re_lu_16[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "norm_17 (BatchNormalization)     (None, 13, 13, 512)   2048        conv_17[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_17 (LeakyReLU)       (None, 13, 13, 512)   0           norm_17[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "conv_18 (Conv2D)                 (None, 13, 13, 1024)  4718592     leaky_re_lu_17[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "norm_18 (BatchNormalization)     (None, 13, 13, 1024)  4096        conv_18[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_18 (LeakyReLU)       (None, 13, 13, 1024)  0           norm_18[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "conv_19 (Conv2D)                 (None, 13, 13, 1024)  9437184     leaky_re_lu_18[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "norm_19 (BatchNormalization)     (None, 13, 13, 1024)  4096        conv_19[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "conv_21 (Conv2D)                 (None, 26, 26, 64)    32768       leaky_re_lu_13[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_19 (LeakyReLU)       (None, 13, 13, 1024)  0           norm_19[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "norm_21 (BatchNormalization)     (None, 26, 26, 64)    256         conv_21[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "conv_20 (Conv2D)                 (None, 13, 13, 1024)  9437184     leaky_re_lu_19[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_21 (LeakyReLU)       (None, 26, 26, 64)    0           norm_21[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "norm_20 (BatchNormalization)     (None, 13, 13, 1024)  4096        conv_20[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)                (None, 13, 13, 256)   0           leaky_re_lu_21[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_20 (LeakyReLU)       (None, 13, 13, 1024)  0           norm_20[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)      (None, 13, 13, 1280)  0           lambda_1[0][0]                   \n",
      "                                                                   leaky_re_lu_20[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "conv_22 (Conv2D)                 (None, 13, 13, 1024)  11796480    concatenate_1[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "norm_22 (BatchNormalization)     (None, 13, 13, 1024)  4096        conv_22[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "leaky_re_lu_22 (LeakyReLU)       (None, 13, 13, 1024)  0           norm_22[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "conv_23 (Conv2D)                 (None, 13, 13, 30)    30750       leaky_re_lu_22[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "reshape_1 (Reshape)              (None, 13, 13, 5, 6)  0           conv_23[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "input_2 (InputLayer)             (None, 1, 1, 1, 50, 4 0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lambda_2 (Lambda)                (None, 13, 13, 5, 6)  0           reshape_1[0][0]                  \n",
      "                                                                   input_2[0][0]                    \n",
      "====================================================================================================\n",
      "Total params: 50,578,686\n",
      "Trainable params: 50,558,014\n",
      "Non-trainable params: 20,672\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load pretrained weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Load the weights originally provided by YOLO**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-04T00:18:58.168386",
     "start_time": "2018-04-04T00:18:58.110194"
    }
   },
   "outputs": [],
   "source": [
    "weight_reader = WeightReader(wt_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-04T00:19:04.250579",
     "start_time": "2018-04-04T00:18:58.711706"
    }
   },
   "outputs": [],
   "source": [
    "weight_reader.reset()\n",
    "nb_conv = 23\n",
    "\n",
    "for i in range(1, nb_conv+1):\n",
    "    conv_layer = model.get_layer('conv_' + str(i))\n",
    "    \n",
    "    if i < nb_conv:\n",
    "        norm_layer = model.get_layer('norm_' + str(i))\n",
    "        \n",
    "        size = np.prod(norm_layer.get_weights()[0].shape)\n",
    "\n",
    "        beta  = weight_reader.read_bytes(size)\n",
    "        gamma = weight_reader.read_bytes(size)\n",
    "        mean  = weight_reader.read_bytes(size)\n",
    "        var   = weight_reader.read_bytes(size)\n",
    "\n",
    "        weights = norm_layer.set_weights([gamma, beta, mean, var])       \n",
    "        \n",
    "    if len(conv_layer.get_weights()) > 1:\n",
    "        bias   = weight_reader.read_bytes(np.prod(conv_layer.get_weights()[1].shape))\n",
    "        kernel = weight_reader.read_bytes(np.prod(conv_layer.get_weights()[0].shape))\n",
    "        kernel = kernel.reshape(list(reversed(conv_layer.get_weights()[0].shape)))\n",
    "        kernel = kernel.transpose([2,3,1,0])\n",
    "        conv_layer.set_weights([kernel, bias])\n",
    "    else:\n",
    "        kernel = weight_reader.read_bytes(np.prod(conv_layer.get_weights()[0].shape))\n",
    "        kernel = kernel.reshape(list(reversed(conv_layer.get_weights()[0].shape)))\n",
    "        kernel = kernel.transpose([2,3,1,0])\n",
    "        conv_layer.set_weights([kernel])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Randomize weights of the last layer**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-22T14:08:00.245248Z",
     "start_time": "2017-11-22T14:08:00.215495Z"
    }
   },
   "outputs": [],
   "source": [
    "layer   = model.layers[-4] # the last convolutional layer\n",
    "weights = layer.get_weights()\n",
    "\n",
    "new_kernel = np.random.normal(size=weights[0].shape)/(GRID_H*GRID_W)\n",
    "new_bias   = np.random.normal(size=weights[1].shape)/(GRID_H*GRID_W)\n",
    "\n",
    "layer.set_weights([new_kernel, new_bias])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Perform training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Loss function**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-02-01T20:44:50.211553",
     "start_time": "2017-02-01T20:44:50.206006"
    }
   },
   "source": [
    "$$\\begin{multline}\n",
    "\\lambda_\\textbf{coord}\n",
    "\\sum_{i = 0}^{S^2}\n",
    "    \\sum_{j = 0}^{B}\n",
    "     L_{ij}^{\\text{obj}}\n",
    "            \\left[\n",
    "            \\left(\n",
    "                x_i - \\hat{x}_i\n",
    "            \\right)^2 +\n",
    "            \\left(\n",
    "                y_i - \\hat{y}_i\n",
    "            \\right)^2\n",
    "            \\right]\n",
    "\\\\\n",
    "+ \\lambda_\\textbf{coord} \n",
    "\\sum_{i = 0}^{S^2}\n",
    "    \\sum_{j = 0}^{B}\n",
    "         L_{ij}^{\\text{obj}}\n",
    "         \\left[\n",
    "        \\left(\n",
    "            \\sqrt{w_i} - \\sqrt{\\hat{w}_i}\n",
    "        \\right)^2 +\n",
    "        \\left(\n",
    "            \\sqrt{h_i} - \\sqrt{\\hat{h}_i}\n",
    "        \\right)^2\n",
    "        \\right]\n",
    "\\\\\n",
    "+ \\sum_{i = 0}^{S^2}\n",
    "    \\sum_{j = 0}^{B}\n",
    "        L_{ij}^{\\text{obj}}\n",
    "        \\left(\n",
    "            C_i - \\hat{C}_i\n",
    "        \\right)^2\n",
    "\\\\\n",
    "+ \\lambda_\\textrm{noobj}\n",
    "\\sum_{i = 0}^{S^2}\n",
    "    \\sum_{j = 0}^{B}\n",
    "    L_{ij}^{\\text{noobj}}\n",
    "        \\left(\n",
    "            C_i - \\hat{C}_i\n",
    "        \\right)^2\n",
    "\\\\\n",
    "+ \\sum_{i = 0}^{S^2}\n",
    "L_i^{\\text{obj}}\n",
    "    \\sum_{c \\in \\textrm{classes}}\n",
    "        \\left(\n",
    "            p_i(c) - \\hat{p}_i(c)\n",
    "        \\right)^2\n",
    "\\end{multline}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-26T12:34:28.064549Z",
     "start_time": "2017-11-26T12:34:27.800510Z"
    },
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "def custom_loss(y_true, y_pred):\n",
    "    mask_shape = tf.shape(y_true)[:4]\n",
    "    \n",
    "    cell_x = tf.to_float(tf.reshape(tf.tile(tf.range(GRID_W), [GRID_H]), (1, GRID_H, GRID_W, 1, 1)))\n",
    "    cell_y = tf.transpose(cell_x, (0,2,1,3,4))\n",
    "\n",
    "    cell_grid = tf.tile(tf.concat([cell_x,cell_y], -1), [BATCH_SIZE, 1, 1, 5, 1])\n",
    "    \n",
    "    coord_mask = tf.zeros(mask_shape)\n",
    "    conf_mask  = tf.zeros(mask_shape)\n",
    "    class_mask = tf.zeros(mask_shape)\n",
    "    \n",
    "    seen = tf.Variable(0.)\n",
    "    total_recall = tf.Variable(0.)\n",
    "    \n",
    "    \"\"\"\n",
    "    Adjust prediction\n",
    "    \"\"\"\n",
    "    ### adjust x and y      \n",
    "    pred_box_xy = tf.sigmoid(y_pred[..., :2]) + cell_grid\n",
    "    \n",
    "    ### adjust w and h\n",
    "    pred_box_wh = tf.exp(y_pred[..., 2:4]) * np.reshape(ANCHORS, [1,1,1,BOX,2])\n",
    "    \n",
    "    ### adjust confidence\n",
    "    pred_box_conf = tf.sigmoid(y_pred[..., 4])\n",
    "    \n",
    "    ### adjust class probabilities\n",
    "    pred_box_class = y_pred[..., 5:]\n",
    "    \n",
    "    \"\"\"\n",
    "    Adjust ground truth\n",
    "    \"\"\"\n",
    "    ### adjust x and y\n",
    "    true_box_xy = y_true[..., 0:2] # relative position to the containing cell\n",
    "    \n",
    "    ### adjust w and h\n",
    "    true_box_wh = y_true[..., 2:4] # number of cells accross, horizontally and vertically\n",
    "    \n",
    "    ### adjust confidence\n",
    "    true_wh_half = true_box_wh / 2.\n",
    "    true_mins    = true_box_xy - true_wh_half\n",
    "    true_maxes   = true_box_xy + true_wh_half\n",
    "    \n",
    "    pred_wh_half = pred_box_wh / 2.\n",
    "    pred_mins    = pred_box_xy - pred_wh_half\n",
    "    pred_maxes   = pred_box_xy + pred_wh_half       \n",
    "    \n",
    "    intersect_mins  = tf.maximum(pred_mins,  true_mins)\n",
    "    intersect_maxes = tf.minimum(pred_maxes, true_maxes)\n",
    "    intersect_wh    = tf.maximum(intersect_maxes - intersect_mins, 0.)\n",
    "    intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1]\n",
    "    \n",
    "    true_areas = true_box_wh[..., 0] * true_box_wh[..., 1]\n",
    "    pred_areas = pred_box_wh[..., 0] * pred_box_wh[..., 1]\n",
    "\n",
    "    union_areas = pred_areas + true_areas - intersect_areas\n",
    "    iou_scores  = tf.truediv(intersect_areas, union_areas)\n",
    "    \n",
    "    true_box_conf = iou_scores * y_true[..., 4]\n",
    "    \n",
    "    ### adjust class probabilities\n",
    "    true_box_class = tf.argmax(y_true[..., 5:], -1)\n",
    "    \n",
    "    \"\"\"\n",
    "    Determine the masks\n",
    "    \"\"\"\n",
    "    ### coordinate mask: simply the position of the ground truth boxes (the predictors)\n",
    "    coord_mask = tf.expand_dims(y_true[..., 4], axis=-1) * COORD_SCALE\n",
    "    \n",
    "    ### confidence mask: penelize predictors + penalize boxes with low IOU\n",
    "    # penalize the confidence of the boxes, which have IOU with some ground truth box < 0.6\n",
    "    true_xy = true_boxes[..., 0:2]\n",
    "    true_wh = true_boxes[..., 2:4]\n",
    "    \n",
    "    true_wh_half = true_wh / 2.\n",
    "    true_mins    = true_xy - true_wh_half\n",
    "    true_maxes   = true_xy + true_wh_half\n",
    "    \n",
    "    pred_xy = tf.expand_dims(pred_box_xy, 4)\n",
    "    pred_wh = tf.expand_dims(pred_box_wh, 4)\n",
    "    \n",
    "    pred_wh_half = pred_wh / 2.\n",
    "    pred_mins    = pred_xy - pred_wh_half\n",
    "    pred_maxes   = pred_xy + pred_wh_half    \n",
    "    \n",
    "    intersect_mins  = tf.maximum(pred_mins,  true_mins)\n",
    "    intersect_maxes = tf.minimum(pred_maxes, true_maxes)\n",
    "    intersect_wh    = tf.maximum(intersect_maxes - intersect_mins, 0.)\n",
    "    intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1]\n",
    "    \n",
    "    true_areas = true_wh[..., 0] * true_wh[..., 1]\n",
    "    pred_areas = pred_wh[..., 0] * pred_wh[..., 1]\n",
    "\n",
    "    union_areas = pred_areas + true_areas - intersect_areas\n",
    "    iou_scores  = tf.truediv(intersect_areas, union_areas)\n",
    "\n",
    "    best_ious = tf.reduce_max(iou_scores, axis=4)\n",
    "    conf_mask = conf_mask + tf.to_float(best_ious < 0.6) * (1 - y_true[..., 4]) * NO_OBJECT_SCALE\n",
    "    \n",
    "    # penalize the confidence of the boxes, which are reponsible for corresponding ground truth box\n",
    "    conf_mask = conf_mask + y_true[..., 4] * OBJECT_SCALE\n",
    "    \n",
    "    ### class mask: simply the position of the ground truth boxes (the predictors)\n",
    "    class_mask = y_true[..., 4] * tf.gather(CLASS_WEIGHTS, true_box_class) * CLASS_SCALE       \n",
    "    \n",
    "    \"\"\"\n",
    "    Warm-up training\n",
    "    \"\"\"\n",
    "    no_boxes_mask = tf.to_float(coord_mask < COORD_SCALE/2.)\n",
    "    seen = tf.assign_add(seen, 1.)\n",
    "    \n",
    "    true_box_xy, true_box_wh, coord_mask = tf.cond(tf.less(seen, WARM_UP_BATCHES), \n",
    "                          lambda: [true_box_xy + (0.5 + cell_grid) * no_boxes_mask, \n",
    "                                   true_box_wh + tf.ones_like(true_box_wh) * np.reshape(ANCHORS, [1,1,1,BOX,2]) * no_boxes_mask, \n",
    "                                   tf.ones_like(coord_mask)],\n",
    "                          lambda: [true_box_xy, \n",
    "                                   true_box_wh,\n",
    "                                   coord_mask])\n",
    "    \n",
    "    \"\"\"\n",
    "    Finalize the loss\n",
    "    \"\"\"\n",
    "    nb_coord_box = tf.reduce_sum(tf.to_float(coord_mask > 0.0))\n",
    "    nb_conf_box  = tf.reduce_sum(tf.to_float(conf_mask  > 0.0))\n",
    "    nb_class_box = tf.reduce_sum(tf.to_float(class_mask > 0.0))\n",
    "    \n",
    "    loss_xy    = tf.reduce_sum(tf.square(true_box_xy-pred_box_xy)     * coord_mask) / (nb_coord_box + 1e-6) / 2.\n",
    "    loss_wh    = tf.reduce_sum(tf.square(true_box_wh-pred_box_wh)     * coord_mask) / (nb_coord_box + 1e-6) / 2.\n",
    "    loss_conf  = tf.reduce_sum(tf.square(true_box_conf-pred_box_conf) * conf_mask)  / (nb_conf_box  + 1e-6) / 2.\n",
    "    loss_class = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=true_box_class, logits=pred_box_class)\n",
    "    loss_class = tf.reduce_sum(loss_class * class_mask) / (nb_class_box + 1e-6)\n",
    "    \n",
    "    loss = loss_xy + loss_wh + loss_conf + loss_class\n",
    "    \n",
    "    nb_true_box = tf.reduce_sum(y_true[..., 4])\n",
    "    nb_pred_box = tf.reduce_sum(tf.to_float(true_box_conf > 0.5) * tf.to_float(pred_box_conf > 0.3))\n",
    "\n",
    "    \"\"\"\n",
    "    Debugging code\n",
    "    \"\"\"    \n",
    "    current_recall = nb_pred_box/(nb_true_box + 1e-6)\n",
    "    total_recall = tf.assign_add(total_recall, current_recall) \n",
    "\n",
    "    loss = tf.Print(loss, [tf.zeros((1))], message='Dummy Line \\t', summarize=1000)\n",
    "    loss = tf.Print(loss, [loss_xy], message='Loss XY \\t', summarize=1000)\n",
    "    loss = tf.Print(loss, [loss_wh], message='Loss WH \\t', summarize=1000)\n",
    "    loss = tf.Print(loss, [loss_conf], message='Loss Conf \\t', summarize=1000)\n",
    "    loss = tf.Print(loss, [loss_class], message='Loss Class \\t', summarize=1000)\n",
    "    loss = tf.Print(loss, [loss], message='Total Loss \\t', summarize=1000)\n",
    "    loss = tf.Print(loss, [current_recall], message='Current Recall \\t', summarize=1000)\n",
    "    loss = tf.Print(loss, [total_recall/seen], message='Average Recall \\t', summarize=1000)\n",
    "    \n",
    "    return loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Parse the annotations to construct train generator and validation generator**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-26T12:38:44.283547Z",
     "start_time": "2017-11-26T12:38:44.277155Z"
    }
   },
   "outputs": [],
   "source": [
    "generator_config = {\n",
    "    'IMAGE_H'         : IMAGE_H, \n",
    "    'IMAGE_W'         : IMAGE_W,\n",
    "    'GRID_H'          : GRID_H,  \n",
    "    'GRID_W'          : GRID_W,\n",
    "    'BOX'             : BOX,\n",
    "    'LABELS'          : LABELS,\n",
    "    'CLASS'           : len(LABELS),\n",
    "    'ANCHORS'         : ANCHORS,\n",
    "    'BATCH_SIZE'      : BATCH_SIZE,\n",
    "    'TRUE_BOX_BUFFER' : 50,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize(image):\n",
    "    return image / 255."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-26T12:38:51.836129Z",
     "start_time": "2017-11-26T12:38:51.766843Z"
    }
   },
   "outputs": [],
   "source": [
    "train_imgs, seen_train_labels = parse_annotation(train_annot_folder, train_image_folder, labels=LABELS)\n",
    "### write parsed annotations to pickle for fast retrieval next time\n",
    "#with open('train_imgs', 'wb') as fp:\n",
    "#    pickle.dump(train_imgs, fp)\n",
    "\n",
    "### read saved pickle of parsed annotations\n",
    "#with open ('train_imgs', 'rb') as fp:\n",
    "#    train_imgs = pickle.load(fp)\n",
    "train_batch = BatchGenerator(train_imgs, generator_config, norm=normalize)\n",
    "\n",
    "valid_imgs, seen_valid_labels = parse_annotation(valid_annot_folder, valid_image_folder, labels=LABELS)\n",
    "### write parsed annotations to pickle for fast retrieval next time\n",
    "#with open('valid_imgs', 'wb') as fp:\n",
    "#    pickle.dump(valid_imgs, fp)\n",
    "\n",
    "### read saved pickle of parsed annotations\n",
    "#with open ('valid_imgs', 'rb') as fp:\n",
    "#    valid_imgs = pickle.load(fp)\n",
    "valid_batch = BatchGenerator(valid_imgs, generator_config, norm=normalize, jitter=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Setup a few callbacks and start the training**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-26T12:38:15.714460Z",
     "start_time": "2017-11-26T12:38:15.708674Z"
    },
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "early_stop = EarlyStopping(monitor='val_loss', \n",
    "                           min_delta=0.001, \n",
    "                           patience=10, \n",
    "                           mode='min', \n",
    "                           verbose=1)\n",
    "\n",
    "checkpoint = ModelCheckpoint('weights_truck2.h5', \n",
    "                             monitor='val_loss', \n",
    "                             verbose=1, \n",
    "                             save_best_only=True, \n",
    "                             mode='min', \n",
    "                             period=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2017-11-26T20:38:54.037Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "12/13 [==========================>...] - ETA: 4s - loss: 0.5833Epoch 00000: val_loss improved from inf to 0.47258, saving model to weights_truck2.h5\n",
      "13/13 [==============================] - 70s - loss: 0.5712 - val_loss: 0.4726\n",
      "Epoch 2/100\n",
      "12/13 [==========================>...] - ETA: 3s - loss: 0.5854Epoch 00001: val_loss improved from 0.47258 to 0.44968, saving model to weights_truck2.h5\n",
      "13/13 [==============================] - 68s - loss: 0.5872 - val_loss: 0.4497\n",
      "Epoch 3/100\n",
      "12/13 [==========================>...] - ETA: 3s - loss: 0.5750Epoch 00002: val_loss improved from 0.44968 to 0.43230, saving model to weights_truck2.h5\n",
      "13/13 [==============================] - 61s - loss: 0.5659 - val_loss: 0.4323\n",
      "Epoch 4/100\n",
      "12/13 [==========================>...] - ETA: 3s - loss: 0.5410Epoch 00003: val_loss did not improve\n",
      "13/13 [==============================] - 52s - loss: 0.5439 - val_loss: 0.4491\n",
      "Epoch 5/100\n",
      "12/13 [==========================>...] - ETA: 3s - loss: 0.5806Epoch 00004: val_loss did not improve\n",
      "13/13 [==============================] - 48s - loss: 0.5815 - val_loss: 0.4697\n",
      "Epoch 6/100\n",
      "12/13 [==========================>...] - ETA: 3s - loss: 0.6094Epoch 00005: val_loss did not improve\n",
      "13/13 [==============================] - 51s - loss: 0.6043 - val_loss: 0.4687\n",
      "Epoch 7/100\n",
      "12/13 [==========================>...] - ETA: 3s - loss: 0.5725Epoch 00006: val_loss did not improve\n",
      "13/13 [==============================] - 52s - loss: 0.5736 - val_loss: 0.4330\n",
      "Epoch 8/100\n",
      "12/13 [==========================>...] - ETA: 3s - loss: 0.5668Epoch 00007: val_loss did not improve\n",
      "13/13 [==============================] - 50s - loss: 0.5612 - val_loss: 0.4426\n",
      "Epoch 9/100\n",
      "12/13 [==========================>...] - ETA: 3s - loss: 0.5820Epoch 00008: val_loss did not improve\n",
      "13/13 [==============================] - 49s - loss: 0.5785 - val_loss: 0.4446\n",
      "Epoch 10/100\n",
      "11/13 [========================>.....] - ETA: 6s - loss: 0.5646"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-98657455a4f5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     22\u001b[0m                     \u001b[0mvalidation_steps\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalid_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m                     \u001b[0mcallbacks\u001b[0m        \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mearly_stop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcheckpoint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensorboard\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m                     max_queue_size   = 3)\n\u001b[0m\u001b[1;32m     25\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/freightkeras/env/lib/python3.5/site-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     85\u001b[0m                 warnings.warn('Update your `' + object_name +\n\u001b[1;32m     86\u001b[0m                               '` call to the Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 87\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     88\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/freightkeras/env/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m   2009\u001b[0m                 \u001b[0mbatch_index\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2010\u001b[0m                 \u001b[0;32mwhile\u001b[0m \u001b[0msteps_done\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2011\u001b[0;31m                     \u001b[0mgenerator_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_generator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2012\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2013\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgenerator_output\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'__len__'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/freightkeras/env/lib/python3.5/site-packages/keras/utils/data_utils.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    503\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    504\u001b[0m             \u001b[0;32mwhile\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_running\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 505\u001b[0;31m                 \u001b[0minputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mqueue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mblock\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    506\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0minputs\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    507\u001b[0m                     \u001b[0;32myield\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/multiprocessing/pool.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    600\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    601\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 602\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    603\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mready\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    604\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/multiprocessing/pool.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    597\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    598\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 599\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_event\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    600\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    601\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    547\u001b[0m             \u001b[0msignaled\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_flag\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    548\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0msignaled\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 549\u001b[0;31m                 \u001b[0msignaled\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cond\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    550\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0msignaled\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    551\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    291\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m    \u001b[0;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    292\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 293\u001b[0;31m                 \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    294\u001b[0m                 \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    295\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "tb_counter  = len([log for log in os.listdir(os.path.expanduser('~/logs/')) if 'truck_' in log]) + 1\n",
    "tensorboard = TensorBoard(log_dir=os.path.expanduser('~/logs/') + 'truck_' + '_' + str(tb_counter), \n",
    "                          histogram_freq=0, \n",
    "                          write_graph=True, \n",
    "                          write_images=False)\n",
    "\n",
    "optimizer = Adam(lr=0.1e-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)\n",
    "#optimizer = SGD(lr=1e-4, decay=0.0005, momentum=0.9)\n",
    "#optimizer = RMSprop(lr=1e-4, rho=0.9, epsilon=1e-08, decay=0.0)\n",
    "\n",
    "model.load_weights(\"wednesday2.h5\")\n",
    "\n",
    "model.compile(loss=custom_loss, optimizer=optimizer)\n",
    "\n",
    "\n",
    "\n",
    "#history = model.fit_generator(generator        = train_batch, \n",
    "                    steps_per_epoch  = len(train_batch), \n",
    "                    epochs           = 100, \n",
    "                    verbose          = 1,\n",
    "                    validation_data  = valid_batch,\n",
    "                    validation_steps = len(valid_batch),\n",
    "                    callbacks        = [early_stop, checkpoint, tensorboard], \n",
    "                    max_queue_size   = 3)\n",
    "\n",
    "#print(history.history.keys())\n",
    "# summarize history for accuracy\n",
    "#plt.plot(history.history['loss'])\n",
    "#plt.plot(history.history['val_loss'])\n",
    "#plt.title('model loss')\n",
    "#plt.ylabel('loss')\n",
    "#plt.xlabel('epoch')\n",
    "#plt.legend(['train', 'test'], loc='upper left')\n",
    "#plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Perform detection on image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-22T14:07:49.271978Z",
     "start_time": "2017-11-22T14:07:49.268999Z"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'model' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-3100b1404c64>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"best_weights.h5\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'model' is not defined"
     ]
    }
   ],
   "source": [
    "model.load_weights(\"best_weights.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-04T00:19:07.263359",
     "start_time": "2018-04-04T00:19:05.658285"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "image = cv2.imread('train_image_folder/00001018.jpg')\n",
    "dummy_array = np.zeros((1,1,1,1,TRUE_BOX_BUFFER,4))\n",
    "\n",
    "plt.figure(figsize=(10,10))\n",
    "\n",
    "input_image = cv2.resize(image, (416, 416))\n",
    "input_image = input_image / 255.\n",
    "input_image = input_image[:,:,::-1]\n",
    "input_image = np.expand_dims(input_image, 0)\n",
    "\n",
    "netout = model.predict([input_image, dummy_array])\n",
    "\n",
    "boxes = decode_netout(netout[0], \n",
    "                      obj_threshold=0.3,\n",
    "                      nms_threshold=NMS_THRESHOLD,\n",
    "                      anchors=ANCHORS, \n",
    "                      nb_class=CLASS)\n",
    "            \n",
    "image = draw_boxes(image, boxes, labels=LABELS)\n",
    "\n",
    "plt.imshow(image[:,:,::-1]); plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Perform detection on video"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-06T13:28:28.029334Z",
     "start_time": "2017-10-06T13:28:28.024662Z"
    }
   },
   "outputs": [],
   "source": [
    "model.load_weights(\"weights_coco.h5\")\n",
    "\n",
    "dummy_array = np.zeros((1,1,1,1,TRUE_BOX_BUFFER,4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-06T13:39:09.640646Z",
     "start_time": "2017-10-06T13:31:44.627609Z"
    }
   },
   "outputs": [],
   "source": [
    "video_inp = '../basic-yolo-keras/images/phnom_penh.mp4'\n",
    "video_out = '../basic-yolo-keras/images/phnom_penh_bbox.mp4'\n",
    "\n",
    "video_reader = cv2.VideoCapture(video_inp)\n",
    "\n",
    "nb_frames = int(video_reader.get(cv2.CAP_PROP_FRAME_COUNT))\n",
    "frame_h = int(video_reader.get(cv2.CAP_PROP_FRAME_HEIGHT))\n",
    "frame_w = int(video_reader.get(cv2.CAP_PROP_FRAME_WIDTH))\n",
    "\n",
    "video_writer = cv2.VideoWriter(video_out,\n",
    "                               cv2.VideoWriter_fourcc(*'XVID'), \n",
    "                               50.0, \n",
    "                               (frame_w, frame_h))\n",
    "\n",
    "for i in tqdm(range(nb_frames)):\n",
    "    ret, image = video_reader.read()\n",
    "    \n",
    "    input_image = cv2.resize(image, (416, 416))\n",
    "    input_image = input_image / 255.\n",
    "    input_image = input_image[:,:,::-1]\n",
    "    input_image = np.expand_dims(input_image, 0)\n",
    "\n",
    "    netout = model.predict([input_image, dummy_array])\n",
    "\n",
    "    boxes = decode_netout(netout[0], \n",
    "                          obj_threshold=0.3,\n",
    "                          nms_threshold=NMS_THRESHOLD,\n",
    "                          anchors=ANCHORS, \n",
    "                          nb_class=CLASS)\n",
    "    image = draw_boxes(image, boxes, labels=LABELS)\n",
    "\n",
    "    video_writer.write(np.uint8(image))\n",
    "    \n",
    "video_reader.release()\n",
    "video_writer.release()  "
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "toc": {
   "nav_menu": {
    "height": "122px",
    "width": "252px"
   },
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {
    "height": "758px",
    "left": "0px",
    "right": "1096px",
    "top": "73px",
    "width": "253px"
   },
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}