11. training_deep_neural_networks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Training Deep Neural Networks**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook is inspired from the handson-ml2 GitHub repository by Aurélien Geron\n",
    "\n",
    "https://github.com/ageron/handson-ml2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, let's import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures. We also check that Python 3.5 or later is installed (although Python 2.x may work, it is deprecated so we strongly recommend you use Python 3 instead), as well as Scikit-Learn ≥0.20 and TensorFlow ≥2.0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"HDF5_USE_FILE_LOCKING\"] = \"FALSE\"\n",
    "\n",
    "# Python ≥3.5 is required\n",
    "import sys\n",
    "assert sys.version_info >= (3, 5)\n",
    "\n",
    "# Scikit-Learn ≥0.20 is required\n",
    "import sklearn\n",
    "assert sklearn.__version__ >= \"0.20\"\n",
    "\n",
    "try:\n",
    "    # %tensorflow_version only exists in Colab.\n",
    "    %tensorflow_version 2.x\n",
    "except Exception:\n",
    "    pass\n",
    "\n",
    "# TensorFlow ≥2.0 is required\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "assert tf.__version__ >= \"2.0\"\n",
    "\n",
    "%load_ext tensorboard\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "np.random.seed(42)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "mpl.rc('axes', labelsize=14)\n",
    "mpl.rc('xtick', labelsize=12)\n",
    "mpl.rc('ytick', labelsize=12)\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"deep\"\n",
    "IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID)\n",
    "os.makedirs(IMAGES_PATH, exist_ok=True)\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True, fig_extension=\"png\", resolution=300):\n",
    "    path = os.path.join(IMAGES_PATH, fig_id + \".\" + fig_extension)\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format=fig_extension, dpi=resolution)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vanishing/Exploding Gradients Problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# defining the Logit function\n",
    "def logit(z):\n",
    "    return 1 / (1 + np.exp(-z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAEMCAYAAAAidwoiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdd3wURf/A8c+kV4FACIYqUoMQQEA6oSlIiXTpiIqA6MMPrCBIUVAeqg1FxShIkaYUQfIAkV4CEpAWSoCEhBJCCOnl5vfHHiGXXELKJXdJ5v167Su53bmd720u+92dnZ0VUkoURVGU0snK3AEoiqIo5qOSgKIoSimmkoCiKEopppKAoihKKaaSgKIoSimmkoCiKEopppJAKSOECBBCfGXuOCB3sQgh/hVCzCiikDLW6yeE2FoE9fgIIaQQokIR1DVGCHFdCKEzxzbNFMsoIUSsOWNQNELdJ1ByCCHcgZnAi8CTQDTwL/CZlNJfX8YNSJFSPjBboHq5iUUI8S+wXko5o5Bi8AH2AO5SysgM88ug/X9Em7Cuq8BXUsr5GebZAW7ALVmI/4xCiHLAbWASsB54IKUskp2wEEICA6SU6zPMcwRcpZS3iyIGJXs25g5AMakNgBPwKnAJqAh0AMo/LCCljDJPaFlZUiyZSSnvF1E9ycDNIqiqOtr/+1YpZUQR1JcjKWUCkGDuOBRASqmmEjABZQEJdHlMuQC0o9GHrz2AzWj/kNeAV9DOHmZkKCOBccAfQDwQDHQEqgB/AXHASaBpprr6AqeBJCAUmIr+7DObWCrq63gYy+jMsRj5PE/r33NTH8cJoGemMnbAHP06k4ArwNtADf1nyzj56d/jh7bDBBgD3AKsM613FbA5N3HoP6tBXfr5PvrXFfKw3a4CHwHfATFAGPBuDttolJHPWQOYAfxrpGxshtcz9H+Dl4HLwAPg94zx6suNzBDzLeDnDLFmrPeqsXr0895AO3hJ1v98PdNyqf9brNNv4yvAMHP/7xX3SV0TKDli9VNvIYRDHt73M9pRYifAFximf53ZR8AawBsI1P/+I/AN0AQIR9txAiCEeBbtn3Uj0BD4APgQmJBDLH5ALaAL8BIwAm1nlRMXYDvQVR/bBmCjEKJeps84Aq0ppD7amVI02g62n75MA7QmtP8YqWMdUEZfx8PP54K2vVbmMo6+aDvrWfp6njT2YfKw3f4PbafbFPgcmCeEaGVsncBaoJv+9xb6ukOzKWtMDWAQ0Ad4Hu3v/WmGmN9AS0g/AY3QmiP/1S9urv/5ur7eh68NCCH6AF8Bi4FngCXAN0KIXpmKTkdLtt76z7VcCFEtD59FyczcWUhNppvQdmhRQCJwCJgPPJepTAD6o2+gLtrRVcsMy6sCaWQ9E5ib4fUz+nmTMszzIcMRLfArsDtT3TOAsGxiqaN/f5sMy6tnjiWX2+Ew8JH+99r69XbLpqxB3Bnm+6E/E9C/3gisyPB6GHAfcMhNHPrXV4F3cqo/l9vtKrA6U5mLGesyEkszfT01Mq03N2cCiUCZDPOmApcyvA5Du+6UXd0S6P+Yeg4Ay438Dfbn8D20QTszVWcDBZjUmUAJIqXcAHgCvdCOSlsDh4UQU7J5Sz1Ah3Zk/3AdoWhH9ZmdyvD7Lf3P00bmVdT/rI/2j53RfqCyEOIJI+uvr4/laIZYrmUTSzohhLMQYp4Q4qwQ4p6+x0kz4OHRYRP9evfktJ5cWAm8JIRw0r8eCmyQUibmMo7cyu12O5WpTDiPtr2pXZOG10jS6xJCVAQqA7sKWEd2n9sr07z0zy2lTAXuUHifu1RQSaCEkVImSin9pZSzpJSt0ZpsZuh7oRRESsZqcpiXm+9UTr1g8tpDZj4wAJiGdhG8MVoiKejnzWwbkAr46nd8XXjUFFRUcWTcNilGluX1/1kHiEzzbI2UM0Vd+ZX5+2DOWEoktfFKvrNop83GrhOcR/sOPPtwhhCiCtrZREGdA9pkmtcWrVnDWJfQh7G0yBBLtVzE0hb4RUq5QUp5Cq1p4ukMy0/q19sxm/cn639a51SJlDIJra1+KFr7+E205qzcxvGwrhzrIe/brSDuAB5CiIyJoHFeViC1Lp43gM45FEsh/5/7bF7iUfJOJYESQghRXgixWwgxTAjRSAjxlBBiAPAesEtKGZP5PVLKC2i9e74VQrQUQjRGu7gXT96PyDNbAHQQQswQQtQRQgwFJgPzjBXWx7ID+E4I0Uofix+P70YYDPQRQjQVQjREOzpPT3hSymDgN+AHIUQ//XZpJ4QYri9yDe2z9hBCuOsv+GZnJfACMBatTV6X2zj0rgLthBCVc7g5LE/brYAC0O5RmCKEeFoI8SrQPx/r+RSYKIT4P33MjYUQkzMsvwp0FkJU0t+vYMx/geFCiDeFELWFEG+hJdzC+NxKBioJlByxaBci/wP8DZxB6xa5Cu3INTuj0I5aA9C6iv6KdlNRYkGCkVKeQGse6Yf+hjX9lNMdwqOAEGA3sEUf+9XHVDVJH+8+tOsgh/W/ZzRCv64v0M44/NB6+yClvAF8jLYju/WY+PahHfV6YdgUlNs4pqNdeL+MdhSeRT63W75IKc+hdf0dg9bW3hXtO5PX9SwF3kTrAfQvWjJvkKHIZLQzsVDgn2zW8TvwFlqvp7No3+PxUsoteY1HyRt1x7BiQH+EGg4M1l9oVhSlBFN3DJdyQohOgCtaT5+KaEfEkWhHc4qilHAmaw4SQkwQQgQKIZKEEH45lBsphDguhIgRQoTpu9WpZGQ+tsAnaElgC9r1gPZSyjizRqUoSpEwWXOQEKIvWpezFwBHKeWobMqNQ2s3PAK4o7VDr5NSfmaSQBRFUZRcM9kRuJRyI4AQohnamDLZlVua4eUNIcSvZN99T1EURSlEltAM0x6tJ4tRQogxaL0XcHR0fLZq1apFFVe2dDodVlaqYxWobQEQGhqKlJJq1dQQNlA03wmd1JGkS8LR2rFQ6ykoS/n/CA4OjpRSuhtbZtYkIIQYjXZr/WvZlZFSLgOWATRr1kwGBgZmV7TIBAQE4OPjY+4wLILaFuDj40N0dDQnT540dygWobC/E8lpyXT/tTtHbxzln//8Q3mn8o9/k5lYyv+HEOJadsvMlgSEEC8Bc9GGPo58XHlFURQpJW9sfYPdIbvx8/Wz6ARQXJglCQghugHfAz2klKcfV15RFAVgzr45+J30Y3r76YxsPNLc4ZQIJksC+m6eNmhjhFjrx7RP1Y/0l7FcJ7S7UvtIKY9mXZOiKEpWe6/t5aM9HzGs0TBm+MwwdzglhimvWHyENs7LB2hjrScAHwkhqgkhYjM8+GEa2i37f+rnxwohtpswDkVRSqC21drybY9v+aHXDxiOeacUhCm7iM5AewCFMS4ZyqnuoIqi5NqlqEvYWtlSvWx13mj2hrnDKXEsoYuooiiKUZHxkXT/tTsONg4EjQ3CSpi/u2VJo5KAoigWKTE1kZfWvETo/VB2j9ytEkAhUUlAURSLo5M6XvnjFQ6EHuC3/r/Rumprc4dUYqnUqiiKxfn66Nes+XcNn3X+jAENBpg7nBJNnQkoimJxRjUehbWVNeOajTN3KCWeOhNQFMViHA8/TlxyHK72roxvPl51BS0CKgkoimIRzt45S+dfOjN221hzh1KqqCSgKIrZ3Yy9yYu/voijrSOfdvrU3OGUKuqagKIoZhWfEk/v1b25E3+HvaP2Uq2MGpK7KKkkoCiKWb29/W0CwwP5/eXfedbzWXOHU+qoJKAoillNaTeFDtU70Ltub3OHUiqpJKAoilkcCTtCi8otqFmuJjXL1TR3OKWWujCsKEqR2xa8jdbLW7PkyBJzh1LqqSSgKEqR+ifiHwatH0TjSo15rWm2T5ZViohKAoqiFJmwmDB6ru6Jm6MbWwZvwcXO5fFvUgqVuiagKEqR0Ekdfdf25UHSAw6MPoCnq6e5Q1JQSUBRlCJiJaz4vMvnpOhSaOjR0NzhKHoqCSiKUqiklByPOE4zz2Z0fEo9WNDSqGsCiqIUqgWHFtD8++YEXA0wdyiKESoJKIpSaDac3cC7/u8ysMFA2ldvb+5wFCNUElAUpVAcCTvCsE3DaFWlFX6+furxkBbKpH8VIcQEIUSgECJJCOH3mLL/J4S4KYSIEUIsF0LYmzIWRVHMJyohit5reuPp6skfL/+Bo62juUNSsmHq1BwOfAIsz6mQEOIF4AOgM1AdqAnMNHEsiqKYiZujG7M7zubPIX/i7uxu7nCUHJi0d5CUciOAEKIZUCWHoiOBH6WUZ/TlZwO/oiWGbF24cAEfHx+DeQMHDmT8+PHEx8fz4osvZnnPqFGjGDVqFJGRkfTv3z/L8nHjxjFo0CBCQ0MZPnx4luWTJ0+mV69eXLhwgTfeeAOA6OhoypYtC8BHH31Ely5dOHnyJBMnTszy/jlz5tC6dWsOHjzIlClTsixfvHgxjRs35n//+x+ffPJJluXfffcddevWZcuWLSxYsCDL8hUrVlC1alXWrl3L0qVLsyxfv349FSpUwM/PDz8/vyzL//zzT5ycnPjmm2/47bffsiwPCAgAYP78+WzdutVgmaOjI++//z4As2fPZteuXQbLy5cvz4YNGwD48MMPOXTokMHyKlWqsHLlSgAmTpzIyZMnDZbXqVOHZcuWATBmzBiCg4MNljdu3JjFixcDMGzYMMLCwgyWt2rVirlz5wLQr18/7t69a7C8c+fOTJs2DYDu3buTkJBgsLxnz5688847AFm+d/Dou6fT6bh06VKWMoXx3cvIEr97OqHjduptKllXKvTv3vbt24HS/d3L734vI3N1EW0A/JHhdRDgIYQoL6U02FpCiDHAGABbW1uio6MNVhQcHExAQACJiYlZlgGcP3+egIAA7t+/b3T5mTNnCAgI4Pbt20aXnz59GldXV65fv56+PC0tLf33oKAgbGxsuHTpktH3nzhxguTkZP7991+jywMDA4mOjiYoKMjo8iNHjhAREcHp06eNLj906BCXL1/mzJkzRpcfOHCAMmXKcP78eaPL9+7di4ODA8HBwUaXP/xHvHz5cpblCQkJxMbGEhAQQEhISJblOp0u/f0Zt99Dtra26cvDwsKyLA8PD09fHh4enmV5WFhY+vJbt25lWX79+vX05Xfu3CEmJsZgeUhISPryqKgokpKSDJZfvnw5fbmxbfPwuxcdHY2UMkuZwvjuZWRp3z2JJLRpKPc972Ptb13o372Hyy35uxcbG2vS755OZ4NO50xg4B1++eUoMTGp3LhRA53OAZ3OHp3OASkdWLWqLEeOXOb+/WTOnRsO/E12hJQy24X5JYT4BKgipRyVzfLLwJtSyh3617ZAMvCUlPJqdutt1qyZDAwMNHm8eRUQEGA0O5dGaltoR2rR0dFZjiZLm0/2fsK0PdMYVX0UP436ydzhWISM/x+pqRAdDXfvQlTUo+nuXbh3Dx48gJgY7Wd2vycn5zcScVxK2czYEnOdCcQCT2R4/fD3B2aIRVGUAlp1ehXT9kxjhPcIRpQZYe5wioSU2o755s2s061b2s/Ll5uSkqLt6O/fL3idNjbwxBPg7AxOTtrk6KhND3/P/NPREaZOzWGdBQ8rX84A3sDDhkBv4FbmpiBFUSzfPxH/8Mofr+BTw4fve33PwX0HzR2SSSQnQ1gYXL/+aLp2zfB1fPzj1vLoWFcIKFcO3Ny0qXz5R7+XK6ft3F1ds/7M+Lu9vbae3MWfzKpVqxg5cmTRJQEhhI1+ndaAtRDCAUiVUqZmKvoL4CeE+BWtR9FHgJ8pY1EUpWg8U/EZPmz7IW8/9zZ21nbmDidPUlPh6lW4eBGCg7Xp4e/Xr2tH+zlxdoZKlQwnD49Hv4eGHuf555/FzQ3KlgWrIrpVIjY2lm7dunHgwAG6d++eY1lTnwl8BHyc4fUwYKYQYjlwFvCSUl6XUu4QQswD9gCOwIZM71MUxcJFxkeSpkvDw8WDGT4zzB1OjqTUjupPnzaczp/Pvp3dygqqVIFq1R5N1asbvi5TJud6AwIeUKuW6T9PTqKiovDx8eHixYu4uroSHh6eY3lTdxGdAczIZrHBwOFSyoXAQlPWryhK0UhMTcR3jS/3Eu5xatwpbKwsZyxKKSE0FI4efTQFBWkXZY2pWhXq1IHatR/9rF0bnnoK7IrXiQ3h4eG0bduWsLAwUlJSsLOz48aNGzm+x3L+coqiFAs6qWPU76M4GHqQdQPWmT0BJCbC4cNw4IC2wz9yRLswm1mFCtCwoeHUoAG4lJDn2ly6dIm2bdsSGRlJWloaACkpKUV7JqAoSsn30e6PWHtmLZ93+Zz+XjnfiFQYkpO1Hf2ePdp06BBk6maPmxu0aPFoatpUa6PP7UXV4iYoKAgfHx/u379Pxm7/CQkJhIaG5vhelQQURcm11adXM3f/XMY0HcO7rd8tsnqDg2HrVti+XTviz3SDLY0aQYcO0LKlttN/+umSu8PP7MCBA3Tr1o3Y2Fijyy9dupTj+1USUBQl155/+nk+aPMBszrOQhTiXjYlBfbu1Xb8W7dC5v1YgwbQsSP4+Gg7/woVCi0Ui7Zt2zYGDhxIfA59Va9du5bjOlQSUBTlsa5GX+VJlycp71SeuV3mFkodqala885vv8HGjdrdtA+5uUH37tCjB3TuDBUrFkoIxcrKlSsZM2ZMlnGHMouIiMhxuUoCiqLkKOJBBB38OtC6amtW91tt0nVLCfv3w8qV2o4/MvLRsvr1wdcXevaE557T7pZVNN999x0TJ04kMTHxsWUjM25UI9RmVRQlW3HJcfRa3Yu78XdNeg0gLAx++QV++smwqaduXRg0CAYO1Jp8FONiYmKwtbXF1taWBw9yHm1Hf6aQbdudetSPoihGpenSGLpxKP/c/Ic1/dfQ9MmmBVtfGvz+u9asU726Np7NpUvg6QkffACnTsG5czBzpkoAj/Puu+8SGRnJihUraNWqFQ4ODtmW1S+zzW65OhNQFMWoaXum8ceFP/ii2xf0rNMz3+u5dw+WL4evvtKGaADtJixfX3jlFXj+ebC2Nk3MpYmdnR2+vr4EBQVx4sSJbMvZaO1o2d72ppKAoihGDW04FGdbZ9567q18vf/CBVi8WGv2edh5pWZNmDABRozQBlBTCkZKydKlSw2eR2Bvb4+XlxcXL15ECPGwOUidCSiKkjtX7l3hqbJP0aBiAxpUzHu7zOXLznz7rdbL5+F9S127wttva01B6qjfdA4cOJDl/gAhBJs2baJSpUps3bqV7777Dn9//8yDeKZT1wQURUl3IuIEjZY2YuGhvA/rdfSo1sTz2mvNWbtW683z2mtw5gzs3Kn18lEJwLSWLl1KXFycwbxGjRpRvXp17O3t6devHzt37oQcntWizgQURQEg9H4oPVf1pLxTeYY2Gprr9506BR9+CH/+qb22s0tj7Fhr3n1XG4VTKRxxcXFs2rTJYJgIFxcXJkyYkKf1qCSgKAoxSTH0XN2TuJQ4Dgw/QCWXSo99z/XrMH261uYvpTYQ25tvQosWh+nbt00RRF26bdiwAetMp1ZpaWn069cvT+tRzUGKUspJKRmyYQhn75xl/YD1PFPxmRzLR0XBu+9qwy7//LPW7PP223DlCnz2Gbi5pRRR5KXbl19+aXA9QAhB3759cXJyytN61JmAopRyQgheb/o6/er3o+vTXbMtp9NpN3e9/772zFyAwYNh9mxtwDal6Fy9epV///3XYJ6zszPjxo3L87pUElCUUuz6/etUK1MN33q+OZY7cQLGj9eGcAZt0LYFC+DZZ4sgSCWL5cuXG1wLAHB1daV169Z5XpdqDlKUUmr92fXU+qIW/pf9sy1z757Wr795cy0BPPkkrFqlDfSmEoB56HQ6vvvuO4N7AxwcHBg3bly+RnZVZwKKUgodDjvM8E3DaV65Oe2qtzNaZssWGDMGbt7UunZOmgQffwxPPFHEwSoG9u3bZ3To6FdeeSVf61NJQFFKmSv3rtB7dW88XT35fdDvONgYjjsTFQX/+Y82sidAmzbw7bfwTM7Xi5Ui8s0332S5N6BJkyZUyWd/XNUcpCilSGxyLD1W9SBVl8qfQ/7E3dndYPmWLdrgbStXgqOjNuzD3r0qAViK2NhYNm/enOXegLfffjvf6zRpEhBCuAkhNgkh4oQQ14QQQ7IpZy+E+FYIcUsIESWE2CKEqGzKWBRFycrZ1pnhjYazadAm6laomz7/wQMYNQp699aaf9q2haAg7YzASh0qWox169ZluTdAp9Ph65vzhf2cmPrP+zWQDHgAQ4GlQghjg4/8B2gFNAI8gXvAlyaORVEUPSklN2JuIIRgSrspdKjRIX3ZiRPaRd6ff9aO/hctgr//htq1zRiwYpSfn5/B9QArKysGDBiAo6NjvtdpsiQghHAG+gHTpJSxUsr9wGZguJHiTwF/SSlvSSkTgbWAGkFcUQrJrL9n0XBpQ65GX02fJyUsWaI9nP3iRe1h7cePw8SJ6ujfUn311VeMGjUKJycnXFxcsLe3z9e9ARmZ8sJwHSBVShmcYV4Q0MFI2R+BJUIITyAa7axhu7GVCiHGAGMAPDw8CAgIMGHI+RMbG2sRcVgCtS0gOjqatLQ0i90O/rf8mXN+Di94vEDIPyFcFVe5f9+Wzz+vy6FD2hPafX1vMG7cZW7d0nHrVsHqU9+JRwpjW4wYMYKXX36Z/fv3c/78eeLj4wtWh5TSJBPQDriZad7rQICRsmWANYAEUoF/ALfH1fHss89KS7Bnzx5zh2Ax1LaQskOHDtLb29vcYRgVEBIgbWfZyo5+HWVSapKUUsqjR6WsUkVKkLJsWSk3bjRtneo78YilbAsgUGazXzXlSV8skLkH8RMYH8L0a8AeKA84AxvJ5kxAUZT8uRx1mT5r+/C029NsGLgBO2s7/PygXTvtGb8tW8LJk9Cnj7kjVczJlEkgGLARQmS8nOQNnDFStjHgJ6WMklImoV0UbiGEqGDCeBSlVPN09WRgg4FsG7INF5tyvP229jjHpCR44w3t4m/16uaOUjE3k10TkFLGCSE2ArOEEK+h7eh9AWODWRwDRgghAoB4YDwQLqWMNFU8ilJaJaQkkJyWTBmHMnzb81tu34YuXbT+/ra28PXX8Prr5o5SsRSm7gMwHnAEbgOrgXFSyjNCiHZCiIzPQHsHSAQuAneAFwF1UqooBaSTOkb+PpK2P7UlKTWJf//Vxv3Zu1cb9+fvv1UCUAyZdNgIKWUU8JKR+fsAlwyv76L1CFIUxYSm7prKurPr+G/X/7IvwJ5+/SAmRmv/37hRSwRK0Zk4cSJt27blq6++Mnco2VK9gRWlhPj++Pd8duAzxj47lnLnJtO9u5YABgyA3buLTwK4c+cO48ePp0aNGtjb2+Ph4UHnzp3x989+tNOMAgICEEIQGVl0rct+fn64uLhkmT9r1izmzp1bZHHkhxpATlFKgF1XdjFu2zi6Pd0dt8Nf89qn2pDC770Hc+cWr5u/+vXrR3x8PD/++CO1atXi9u3b/P3339x9+CSbIpScnIydnV2+3//EE0/g6upqwogKQXZ9Ry1xUvcJWB61LSzjPoGw+2FyyG8j5aDByRKktLKSculS88RSkO/EvXv3JCD9/f2zLbNixQrZrFkz6eLiIt3d3WX//v1lWFiYlFLKkJAQiXb/Ufo0cuRIKaX2d3rzzTcN1jVy5EjZo0eP9NcdOnSQY8eOlZMnT5YVKlSQzZo1k1JKuWDBAtmwYUPp5OQkPT095auvvirv3buX/nkz1/nxxx9LKaX09vY2qLN69epy9uzZcsyYMdLV1VVWrlxZzps3zyCmCxcuyPbt20t7e3tZp04duW3bNuns7Cx/+umnfG1TKYvuPgFFUYpYVEIUabo0ytlUJtrPj7WrbXFxga1bYexYc0eXdy4uLri4uLB582YSExONlklOTmbmzJkEBQWxdetWIiMjGTx4MABVq1Zlw4YNAJw5c4aIiAiWLFmSpxhWrlyJlJJ9+/bxyy+/ANoYPYsXL+bMmTOsWrWKo0eP8tZbbwHQunVrFi9ejJOTExEREURERPDOO+9ku/5FixbRsGFDTpw4wfvvv897773HoUOHAG0wuD59+mBjY8Phw4fx8/Nj5syZBg+QMTXVHKQoxVRcchxdV3SlpmNTbi77nv37oXx52LEDmjUzd3T5Y2Njg5+fH6+//jrLli2jSZMmtGnThgEDBvDcc88BMHr06PTyNWvWZOnSpdSvX5+wsDCqVKmCm5sbABUrVqRChbzfevTUU0+xYMECg3kTJ05M/71GjRrMmzcPX19ffv75Z+zs7ChTpgxCCCpVqvTY9T///PNMmDABgLfeeosvvviCXbt20apVK/z9/blw4QI7d+6kcmVtYOVFixbRpk2bPH+O3FJnAopSDKXp0hi8YTD/XArnxNz57N8PVarAvn3FNwE81K9fP8LDw9myZQvdu3fn4MGDtGzZkjlz5gBw4sQJfH19qV69Oq6urjTTf+Dr16+bpP5njTw3c/fu3XTt2pUqVarg6upK3759SU5O5ubNm3lef6NGjQxee3p6cvv2bQDOnz+Pp6dnegIAaN68OVaFeFFHJQFFKYYm75zMlqNBVFhzjivnylC7NuzfD/Xrmzsy03BwcKBr165Mnz6dgwcP8uqrrzJjxgzu37/PCy+8gJOTEytWrODYsWPs2LED0JqJcmJlZZXl4ewpKSlZyjk7Oxu8vnbtGj169KB+/fqsW7eO48ePs3z58lzVaYytra3BayEEOp0uz+sxFZUEFKWY+fro1yz5cysuK09yJ7QsjRtrZwAleQgILy8vUlNTOXnyJJGRkcyZM4f27dtTr1699KPohx725klLSzOY7+7uTkREhMG8oKCgx9YdGBhIcnIyixYtolWrVtSpU4fw8PAsdWauLz/q1atHeHi4wfoDAwMLNUmoJKAoxYx7Yhucfj1GbGQ52rSBPXvAw8PcUZnG3bt36dSpEytXruTUqVOEhISwbt065s2bR+fOnfHy8sLe3p6vvvqKK1eusG3bNqZNm2awjurVqyOEYNu2bdy5c4fYWG2wgk6dOrF9+3Y2b97MhQsXmDRpEqGhoY+NqXbt2uh0OhYvXkxISAirV69m8eLFBmVq1G0H6yoAACAASURBVKhBYmIi/v7+REZGGn0QfG507dqVunXrMnLkSIKCgjh8+DCTJk3CxsYGIUS+1vk4KgkoSjERlRDFhQsw8eXGxEeVo3177SJw2bLmjsx0XFxcaNmyJUuWLKFDhw40aNCAKVOmMGTIENauXYu7uzs///wzv//+O15eXsycOZOFCxcarKNy5crMnDmTqVOn4uHhkX4RdvTo0elTmzZtcHV1pU8uhlBt1KgRS5YsYeHChXh5efHDDz8wf/58gzKtW7dm7NixDB48GHd3d+bNm5evz29lZcWmTZtISkqiRYsWjBw5kqlTpyKEwMHBIV/rfKzs+o5a4qTuE7A8alsUzX0C16KvSff32knX8g8kSOnjI2VsbKFWmW/qO/GIKbbFyZMnJSADAwPzvQ5yuE9AdRFVFAsXkxRD5wVvEfnNOmSsC506wZYt4ORk7siUwrBp0yacnZ2pXbs2V69eZdKkSXh7e9O0adNCqU8lAUWxYClpKby4ZBKXFi6DOA+6dIE//lAJoCR78OAB77//PqGhoZQrVw4fHx8WLVpUaNcEVBJQFAs2YvkMDnwyE+I86NpVSwCOjuaOSilMI0aMYMSIEUVWn0oCimKhQkPhfzM+gAeutG8Pv/+uEoBieqp3kKJYoIvXYujSBSLDXWnRQhsLSDUBKYVBJQFFsTA7Th2j3nNhBAeDt7fWDdTSRyNWii+VBBTFggRdC6FXDxt0t7yoXTeVnTuhXDlzR6WUZCoJKIqFCL8XReuut0gNa0LVGsns2WVDxYrmjkop6VQSUBQLkJCcjPfzp4m/2BI392QCdtmRYSBJRSk0KgkoiplJCRPfsiEysAOOLsns9rejZk1zR6WUFiZNAkIINyHEJiFEnBDimhBiSA5lmwoh9gohYoUQt4QQ/zFlLIpSXEyZlsyyZVbY28OObXZ4e5s7IqU0MfV9Al8DyYAH0BjYJoQIklKeyVhICFEB2AH8H7AesAOqmDgWRbF4wz88wsrPnsPKSrJ2raB9e3NHpJQ2JjsTEEI4A/2AaVLKWCnlfmAzMNxI8UnAX1LKX6WUSVLKB1LKc6aKRVGKg+lfnmXl580B+ObbNHx9zRyQUiqZsjmoDpAqpQzOMC8IaGCkbEsgSghxUAhxWwixRQhRzYSxKIpF+3H9dWb/Xy2QVkyflcAbr6ub9xXzMOU3zwWIyTTvPmDsNpcqQFOgK3AamAesBrI8TVkIMQYYA+Dh4UFAQIDpIs6n2NhYi4jDEqhtAdHR0aSlpeV6O5w4K3nn/5pDmh0v9jmPT9ublKRNqL4TjxSHbWHKJBALPJFp3hPAAyNlE4BNUspjAEKImUCkEKKMlPJ+xoJSymXAMoBmzZpJHx8fE4acPwEBAVhCHJZAbQsoW7Ys0dHRudoO16/D4CE6ZLIVz/veZcv6elhZ1Sv8IIuQ+k48Uhy2hSmbg4IBGyFE7QzzvIEzRsqeAjI+8VkaKaMoJUrUPR3du+u4GWFFhw6weW15rFQnbcXMTPYVlFLGARuBWUIIZyFEG8AXWGGk+E9AHyFEYyGELTAN2J/5LEBRSorkZGjS8Qpnz1pRt56OTZvA3t7cUSmK6W8WGw84ArfR2vjHSSnPCCHaCSFiHxaSUu4GpgDb9GVrAdneU6AoxZmU0P6li1wPqoVTufvs2C7UeECKxTBplwQpZRTwkpH5+9AuHGectxRYasr6FcUSDZ1wmSPba2Ntn8Duv5ypUaNwnhClKPmhWiQVpRDNWBjG6m+eBpHGmrWS55qrrqCKZVFJQFEKyY4d8Ml72ihwny2Kob+veiqMYnlUElCUQnAkMIkBAyRpaYIPP4T3/6MuAiiWSSUBRTGxkKtpdOgaS2ysYMgQySefmDsiRcmeSgKKYkLR0dDM5xZJ0eWp3fQGy5cLdS+AYtHU11NRTCQ5GVp0DSXqmidu1W5y5H+V1b0AisVTSUBRTEBK6D7oOhcDq2Jf5h7HAiqqewGUYkH1V1MUE7h1awynTlXDyi6B/+1wpOZT6vhKKR5UElCUAoq41ZVbt8ZjZQWb1jnQtqW6GUwpPtThiqIUwJa/YgkOfheAJUugd2+VAJTiRSUBRcmnoNMp9O0L6OwoU+1HJkwwd0SKkncqCShKPty8KWnbJZrUeBdcKu+ketmvzR2SouSLSgKKkkfx8dC8UwSxt915sm4YjZ9agBA6gzIbNmzA29ub8ePH8+uvvxIcHIxOp8tmjYpiPurCsKLkQVoaDB0KYec8ca54hxMBlXn55aQs5erVq8fp06c5deoUK1asQEqJTqejQYMG+Pj40KpVK5o1a0bVqlURQl1HUMxHJQFFyYNJk9L4/XdrypaFA3sqUKmS8R14gwYN6NSpE7t27SI2Nv1RGgQGBnLixAlcXFxISUnB1taWRo0a0bFjR1q2bEn79u1xcXExuk5FKQyqOUhRcmna3Dt88YU1Nrbak8G8vHI+gv/kk09wcso6cqhOpyMmJoaEhARiYmLYv38/c+bMoXfv3syfP7+wwlcUo1QSUJRcWPFbDJ9MLQ/A3CW3yc2zw1u2bEm9erl7iLxOp6NatWq8++67BYhSUfJOJQFFeYyDh5MZNdwWpBWvTrrGO+Mq5fq9n376aa6ad5ydndm5cyfOzs4FCVVR8kwlAUXJwdWrks7d49ElO9Le9wrfz6+ep/e/8MILVKqUc9Kwt7enV69ePP300wUJVVHyRSUBRclGVBS8+CIkRpelZtMQ/H+rSV478gghmD17do5nA0lJSWzevJnu3bsTExNTwKgVJW9UElAUIxISoFdvyblzggYNJIH/q4GdXf7W1b9//8c2CcXFxREQEECDBg04d+5c/ipSlHwwaRIQQrgJITYJIeKEENeEEEMeU95OCHFOCBFmyjgUpSDS0qDrS3c4eEBQyTOVHTsE5crlvy+/jY0NH3/8cZb2fkdHR4PXSUlJ3Lhxg+bNm7N+/fp816coeWHqM4GvgWTAAxgKLBVCNMih/LvAHRPHoCj5JiUMe+0eB3a6Y+V4n41b4qlSpeDrHTVqFLa2tumvnZycGDhwYJZEIKUkLi6OESNG8H//93+kpaUVvHJFyYHJkoAQwhnoB0yTUsZKKfcDm4Hh2ZR/ChgGzDVVDIpSUFNmxLLGrxzYJLJ6fTytmj5hkvU6ODjw3nvv4eDggJOTE99//z1+fn78/PPPRu8lSEhIYNmyZbRr147IyEiTxKAoxpjyTKAOkCqlDM4wLwjI7kzgS2AKkGDCGBQl3777IZnPZrmA0PHZN6EMfPFJk67/zTffxMrKilGjRjFkiNZSOmDAAI4dO0aVKlWwz/Qsyvj4eI4fP46XlxfHjx83aSyK8pCQUppmRUK0A9ZJKStlmPc6MFRK6ZOpbB9gjJSyuxDCB1gppTR60i2EGAOMAfDw8Hh2zZo1Jom3IGJjY9Wt/XolZVscPuzG1KnPoNNZ0ePV7bwzzPHxb9KbOHEiaWlpfPnll48tGxERQcWKFbG2tjaYHxcXx4wZMzh9+jRJSVnHIrK3t+ftt9/mxRdfzHVc5lJSvhOmYCnbomPHjsellM2MLpRSmmQCmgDxmeZNBrZkmucMXARq61/7AGG5qePZZ5+VlmDPnj3mDsFilIRtceSIlE5OOglSfvCBLs/v79Chg/T29i5wHGlpaXLmzJnS0dFRAlkmJycn+corr8ikpKQC11WYSsJ3wlQsZVsAgTKb/aopm4OCARshRO0M87yBM5nK1QZqAPuEEDeBjcCTQoibQogaJoxHUR7r33+h0/OJxMcLhgxLYc4c843oaWVlxfTp09m4cSOurq5YWRn+e8bHx7NmzRqaNWvGjRs3zBSlUtKYLAlIKePQduizhBDOQog2gC+wIlPRf4GqQGP99BpwS/97qKniUZTHuXwZOnROJO6+AxWbHuGHH0SebwYrDN26dSMoKIinn34aBwcHg2UJCQmcO3eOZ555hr1795opQqUkMXUX0fGAI3AbWA2Mk1KeEUK0E0LEAkgpU6WUNx9OQBSg079W/eGUIhEeDh06JRF12wGXukf5d7cXjvaWM7L6U089RVBQED179szSeyg1NZXo6Gi6devGggULHjazKkq+mDQJSCmjpJQvSSmdpZTVpJSr9PP3SSmNXh2RUgbIbC4KK8b5+PgwQT3QNt/u3oWOnVO4cd0e26onCdxVBfcyruYOKwtHR0d+++035s6dm+V+AtDOCqZPn07fvn2Ji4szQ4RKSVBqho24c+cO48ePp0aNGtjb2+Ph4UHnzp3x9/fP1fsDAgIQQhRpn20/Pz+jPQs2btzI3Lnq9or8iImBbt0g+Lwt9k9exH+HDXUre5o7rGwJIXj77bfx9/enXLly2NgYnq3Ex8ezY8cOvL29uXz5spmiVIqzUpME+vXrx9GjR/nxxx8JDg5m69atdO/enbt37xZ5LMnJyQV6v5ubG66ulnfkaukSEqB3b0lgINSsCReP1qSD1zPmDitX2rRpw5kzZ3jmmWeynBUkJiYSEhJC48aN2bZtm5kiVIqt7LoNWeKU3y6i9+7dk4D09/fPtsyKFStks2bNpIuLi3R3d5f9+/eXYWFhUkopQ0JCsnTXGzlypJRS6x745ptvGqxr5MiRskePHumvO3ToIMeOHSsnT54sK1SoIJs1ayallHLBggWyYcOG0snJSXp6espXX31V3rt3T0qpdS3LXOfHH39stM7q1avL2bNnyzFjxkhXV1dZuXJlOW/ePIOYLly4INu3by/t7e1lnTp15LZt26Szs7P86aef8rVNH7KULnCPk5wsZc+eWjdQZ7doeelS3ruCZsdUXURzIykpSb7++uvSycnJaDdSR0dH+dFHH8m0tLQiiceY4vKdKAqWsi0ooi6iFsvFxQUXFxc2b95MYmKi0TLJycnMnDmToKAgtm7dSmRkJIMHDwagatWqbNiwAYAzZ86wYcMGlixZkqcYVq5ciZSSffv28csvvwBal8DFixdz5swZVq1axdGjR3nrrbcAaN26NYsXL8bJyYmIiAgiIiJ45513sl3/okWLaNiwISdOnOD999/nvffe49ChQ4D21Ko+ffpgY2PD4cOH8fPzY+bMmUZvSiqJUlLg5Zdh61YBjnfpP3cZTz9tAd2A8sHOzo5ly5axdOnSbIebWLhwIV27diU6OtoMESrFTnbZwRKngtwstn79elmuXDlpb28vW7ZsKSdPniwPHz6cbflz585JQIaGhkopHx2Z37lzxyC75/ZMoGHDho+Ncfv27dLOzi79KO6nn36Szs7OWcoZOxN4+eWXDcrUqlVLzp49W0op5Y4dO6S1tXX6mY2UUh44cEACJf5MIDlZyv79pQQpcYiSHee8K9N0pj1KLsozgYz++ecf6eHhIe3s7LKcEdjZ2UlPT095+vTpIo/L0r8TRclStgWl/UwAtGsC4eHhbNmyhe7du3Pw4EFatmzJnDlzADhx4gS+vr5Ur14dV1dXmjXT7rC+fv26Sep/9tlns8zbvXs3Xbt2pUqVKri6utK3b1+Sk5O5efNmntffqFEjg9eenp7cvn0bgPPnz+Pp6UnlypXTlzdv3jzLzUglTWoqDBsG69cDDtF4Tfo/tr4zAytRMj5348aNOXv2LC1atMhyVpCcnEx4eDjPPfccq1evNlOESnFQMv4bcsnBwYGuXbsyffp0Dh48yKuvvsqMGTO4f/8+L7zwAk5OTqxYsYJjx46xY8cO4PEXca2srLL0005JSclSLvNY8teuXaNHjx7Ur1+fdevWcfz4cZYvX56rOo3JOEwxaL1KdDpdntdTUqSmwvDh8Ntv4OSSwlNvjWX3h5/jZJu1CaU4c3NzIyAggAkTJhjtRhofH89rr73GBx98YIbolOKgVCWBzLy8vEhNTeXkyZNERkYyZ84c2rdvT7169dKPoh+y0z9WKvP47u7u7kRERBjMCwoKemzdgYGBJCcns2jRIlq1akWdOnUIDw/PUqcpxpOvV68e4eHhBusPDAwssUkiNRVGjoQ1a8DVFXb52xL82Uo8XDzMHVqhsLa25vPPP2fVqlU4OzsjMt32LKUkNjbWTNEplq5UJIG7d+/SqVMnVq5cyalTpwgJCWHdunXMmzePzp074+Xlhb29PV999RVXrlxh27ZtTJs2zWAd1atXRwjBtm3biI6OTv+n6tSpE9u3b2fz5s1cuHCBSZMmERr6+NEvateujU6nY/HixYSEhLB69WoWL15sUKZGjRokJibi7+9PZGQk8fHx+fr8Xbt2pW7duowcOZKgoCAOHz7MpEmTsLGxybLDKO6Sk2HwYFi1Cqwd4nnri620bAk2VpZzN3Bheemllzh+/DjVqlVLH5baxsaG+vXrs2jRIjNHp1iqUpEEXFxcaNmyJUuWLKFDhw40aNCAKVOmMGTIENauXYu7uzs///wzv//+O15eXsycOZOFCxcarKNy5crMnDmTqVOn0rdv3/Q7dkePHp0+tWnTBldXV/r06fPYmBo1asSSJUtYuHAhXl5e/PDDD8yfP9+gTOvWrRk7diyDBw/G3d2defPm5evzW1lZsWnTJpKSkmjRogUjR45k6tSpCCGyjE1TnCUkQJ8+2jUAO+d40oZ0oXbj0vVAlrp163L69Gk6d+6Mg4MDTzzxBNu2bcvSXKgo6bK7YmyJkxpK2nROnjwpARkYGFig9VjKtoiJkbJjR60XkFOZOMmYJvKjXR8VSd3m6h2UE51OJ7/++usC/33zw1K+E5bAUrYFOfQOKvnnyAoAmzZtwtnZmdq1a3P16lUmTZqEt7c3TZs2NXdoBXbvHrz4Ihw+DGXdE4ge0IwhnZswq+Msc4dmNkIIxo8fb+4wlGJAJYFS4sGDB7z//vuEhoZSrlw5fHx8WLRoUbG/JnDrljYW0MmTUL069J27nMDECizvvbzYfzZFKQoqCZQSI0aMYMSIEeYOw6SCg7UEEBICdepI/vc/QdWqb5KSNgZba9UGrii5USouDCslz+HD0Lq1lgAaN03libE9CeUggEoAipIHKgkoxc6WLdCpk/ZcgG7d03B6rQen43ZluWlPKbgaNWpk6bWmlCyqOUgpVr77DsaPB50ORo+WxHcbxY6zO1ndbzVtqrUxd3jF0qhRo4iMjGTr1q1Zlh07dizL3e5KyVLizgQWLFjAmjVruH//vrlDUUwoLQ3eew/GjtUSwMcfw5NDprPm7Eo+7fQpLz/zsrlDLJHc3d2NjlZa1Ar6DA4leyUqCYSHhzNlyhTGjBlDxYoVadmyZfqwzUrxdf8+9OoF//0v2NjA99/DtOlpnLt7llebvMqHbT80d4glVubmICEEy5YtY8CAATg7O1OzZk1Wrlxp8J47d+7w8ssvU65cOcqVK0ePHj24ePFi+vLLly/j6+tLpUqVcHZ2pmnTplnOQmrUqMGMGTMYPXo0ZcuWZejQoYX7QUuxEpUEtmzZgo2NDQ8ePCA5OZkjR44wffp0c4elFMDFi9CyJWzfDuXLw//+B6++KrG2smbdgHUs7bFUdQUtYrNmzcLX15egoCAGDRrE6NGj00fbjY+PZ9KkSTg4OPD3339z6NAhnnzySbp06ZI+7ElsbCzdu3fH39+foKAg+vXrR9++fTl//rxBPQsXLqRevXoEBgamj/armF6JSgIrV640GF/H2tqa/v37mzEipSD8/aFFCzh/Hp55Bo4dA3evs3Tw60Do/VCshJXqCWQGw4cPZ9iwYdSqVYvZs2djY2PD3r17AVizZg1SSn766ScaNWpEvXr1+O6774iNjU0/2vf29mbs2LE0bNiQWrVqMXXqVJo2bcr69esN6unQoQPvvfcetWrVonbt2kX+OUuLEnNhODY2lmPHjhnMc3R0pF+/fmaKSMkvnU5r+pk6VbsW4OsLK1ZAvLhFpx97kJCSgE6WzBFQi4OMz66wsbHB3d09fdTd48ePExERkeUZ2PHx8Vy+fBmAuLg4Zs6cydatW4mIiCAlJYXExMQsz8R4+EwPpXCZNAkIIdyAH4HngUjgQynlKiPl3gVGAtX15b6RUv63IHX/9ddf2NnZGTwy0crKiueee64gq1WK2N272jDQD5+XPnUqzJoFiWnx9PLrxa3YW/w96m+ql61u3kBLsZyeXaHT6ahVq5bRB967ubkB8M4777Bjxw7mz59P7dq1cXJyYsSIEVku/qpeSUXD1GcCXwPJgAfQGNgmhAiSUp7JVE4AI4BTwNPATiFEqJRyTX4rXrNmDQ8ePDCY16tXrxL/9KyS5PBhGDQIrl+HcuXgl1+gZ0/QSR3DNg4jMDyQTYM20bxyc3OHqmSjadOmrFixggoVKlC2bFmjZfbv38+IESPSz9ITExO5fPkyderUKcpQFT2T7SGFEM5AP2CalDJWSrkf2AwMz1xWSjlPSnlCSpkqpbwA/AHku5N3amoq27dvN5jn6urKyy+rboPFgZSweDG0a6clgOeeg3/+0RIAwL2Ee1yKusTCFxbiW8/XvMGWUDExMZw8edJgunr1ap7XM3ToUNzc3PD19eXvv/8mJCSEvXv3Mnny5PQeQnXq1GHTpk2cOHGC06dPM2zYMBITE038iZTcMuWZQB0gVUoZnGFeENAhpzcJrWtHO+C7bJaPAcYAeHh4EBAQkKXMyZMns9wtmpiYiK2trdHyBRUbG1so6y2OCrot7t6147//rcuRI+UB6N8/lDFjrhASIgkJeVRuft352CYUzt+zoKKjo0lLS7PI2HLj5s2b7Nu3jyZNmhjMb9++ffpResbPdubMGSpUqJD+OnOZTz/9lFWrVvHSSy8RFxdH+fLl05+HfOPGDQYMGMB///tf2rRpg4uLC/3798fLy4ubN2+mr8NYvcVRsdhXZDfGdF4ntB35zUzzXgcCHvO+mWjJwv5xdWT3PIEJEyZIKysrCaRPnTt3zufI249nKWOEW4KCbIu1a6V0c9OeAVCunJQbNhgu/zP4TzngtwEyLjmuYEEWMkt8noA5qf+PRyxlW1BEzxOIBZ7INO8J4IGRsgAIISagXRtoJ6VMyq5cTqSUrFu3zuB5uc7OzurmEgsWFQUTJsDq1drrbt3gxx/B0/NRmaCbQQxcP5BabrVUTyBFKUSmvGoaDNgIITJ26PUGMl8UBkAIMRr4AOgspQzLb6Vnz57NckE4JSWFng8blBWLsnUrNGyoJQAnJ1i6FP780zAB3Ii5QY9VPShjX4atg7fiYudivoAVpYQz2ZmAlDJOCLERmCWEeA2td5Av0DpzWSHEUGAO0FFKeaUg9W7cuJHU1FSDefXq1cPd3b0gq1VM7MYN+M9/YMMG7XWrVlrvn1q1DMs9SHpAz9U9uZ90n/2v7KfyE5WLPlhFKUVM3X9yPOAI3AZWA+OklGeEEO2EELEZyn0ClAeOCSFi9dO3+alw9erVBv2LHRwcVFOQBUlLgy+/hPr1tQTg4gKLFsHevVkTAEBIdAgRDyJYN2Ad3pW8iz5gRSllTHqfgJQyCnjJyPx9gEuG10+Zor6IiAiuXDE8kRBC0KdPH1OsXimgo0fhzTchMFB7/dJL8MUXULVq9u9p5NGIy29fxtlO3SikKEWhWN9JtXnzZmxsDPOYm5ubGmfEzK5fh6FDtf7+gYFQpQr8/jts2pR9Alh0aBEf7/kYKaVKAIpShIp1Evj111+Ji4tLf21tbc2gQYPMGFHpFhMDU6ZA3bqwahXY28P778PZs9r4P9nZdG4Tk3dO5sydM0jU08EUpSgV2wHkYmNjOXr0qME8JycnNWqoGSQmamP8f/IJ6McRY/BgmDMHatTI+b1Hbxxl6MahtKjcghV9VmAlivVxiaIUO8U2CezcuTPLgHFCCDVgXBFKTIRNmyozdCiEh2vzWrWChQu1ZwA8ztXoq/Ra3QsPFw82D96Mo61j4QasKEoWxTYJrF69Osv9AT179lQDxhWBpCTt5q45c+DGDe36i7e39sjHl16C3D7j5Xj4caSU/DnkTyo6VyzEiBVFyU6xSAJCCCfguYdjm6gB48wjMhK+/Ra++gpu3dLm1awZy/z5Lvj6Ql7zbz+vfjz/9PO42rs+vrCiKIWiWCQBoAmw6+TJk3Tr1o3mzZtnOeJPTk6mS5cu5omuhLtwQevb//PPWhMQQOPGMG0alC0bSKdOPrlel5SSCX9OoNNTnejn1U8lAEUxs+KSBC4AKVJKu7/++osDBw6QkJBgUKBt27Y4Oqo2ZVNJSYEtW2DZMvjrr0fze/SASZOgY0et2SevAyTO2TeHbwK/oYJTBfp5qae+KYq5FYsGdCllJJB+BTg2Npa0tLT05dbW1pw6dYr33nuPo0ePGgwmp+TN5cvw4Ydaf/5+/bQEYG8Pr7+udfXcuhU6dcp9u39Gq0+v5qM9HzGs0TBm+MwweeyKouRdcTkTALgINDW2IC0tjTt37rBo0SK++eYbnJycCA4OzvbJRoqhu3dh/XptULe//34038sLxoyB4cNB/2TAfNt/fT+j/hhF++rt+aHXD4j8ZBFFUUyuOCWBE2STBB5KTU3Fzs6O3r17U6ZMmSIKq3iKjYXNm7Wbuv76Cx6OwefoCAMHajv/Vq3yd8RvzF+X/qJG2RpsGrQJext706xUUZQCK1ZJQAiR5QliGTk5OdGrVy+WLVumjjSNiIjQdvx//AG7dsHDcfesreGFF2DIEK2L5xOZnwphArM7zead1u9QxkElZ0WxJMUpCZzJKQk4OjrSuXNnfv31V3WvgF5qqjZ2j7+/1paf8QZrIaBNG+3O3gEDoGIhdNNPTE3klT9e4YM2H+BdyVslAEWxQMUpCZzNLgE4ODjQunVrNmzYgLW1dRGHZTmkhOBg2LMHdu6E3bvh/v1Hyx0coGtXbRyfnj3Bw6PwYtFJHaP/GM2af9fQp14fNSy0olioYpMEpJSR1tbWWc4E7O3tadKkCVu3bsXW1tZM0ZlHUhKcOAH798OBA3DwINy5Y1imVi1tx//CC9ClCzgX0QCd0/dMZ/W/RU8iygAAC1VJREFUq5nTaQ4DGwwsmkoVRcmzYpMEQDvij4+PT39tZ2dH/fr18ff3x8HBwYyRFb7UVO2mrX/+0Xb8x45pU1KmJzN7eEC7dtqOv2tXeMokT27Im5/++YlP933Kq01e5YO2HxR9AIqi5FqxSgKOjo7pScDW1paaNWsSEBCAc1Ed3haRqCg4d07rl//PP9oUFASZ7o8DtG6cbdpA27baz5o1TdejJz+klPx29je61uzK0h5L1QV6RbFwxSoJODs7Ex8fT0pKClWqVGH//v3FtitoUhJcuwYhIdoR/rlzj6aHwzFnVqMGNGkCTZtqU8uWBe+/b2pCCP54+Q+SUpOwtS5dzXOKUhwVqyTg4OBAcnIylSpV4uDBg5QvX97cIWUrMVHrkhkWpu3oQ0LgypVHv9+4oV3INcbZGerV057L6+2t7fAbN7a8HX5GN2NvMnHHRL7s/iXuzu7YWduZOyRFUXKhWCUBJycn2rZtyy+//EKlSpWKvP60NK2p5to1J/bt0y7C3rqljaV/44b28+HvUVE5r8vKCqpV05pvatXSmnXq19emKlXyPiKnOcWnxNN7dW/O3DnDu63fxd3Z3dwhKYqSS8UqCVhbWxOQ1xHLMklK0rpNxsQY/sz8+7172tDJkZHazj4yUpunHb23eGw9Njbw5JPg6aldnM041aypjc1TEjozpck0hm0cRmB4IJsGbeJZz2fNHZKiKHlg0iQghHADfgSeByKBD6WUq4yUE8BnwGv6WT8AH8icbgcGoqNh5UqIj8/79P/t3X9sVfUZx/H30x9jlloQ6hBkqGMaGSMY1/hjRm0WkU0kmiwEI/sR54RpFIzK9I+ZoIsmahTNUidEGZvK2FioTFEnbKkRiTLcikLETmFiTRYFaWmpUNo+++Pc0nK5/UHv6T3n3vN5JTe359zvvX367bnf537Puec5ra3BIJ/+bZoT+/tg7FgoK2tj0qQyKivh1FPh9NODwb77fsKEYH0+fZofqmW7llHbWMvjMx/nmnP7uZCwiMRS2DOBGqAdGAecB6w3s23uviOt3XzgWmA64MAGYDfwVH8v/tFHQTGzbJSUwKhRwa2ioufn9OXRo4OBvLKSo4P9KacEJRbq6rZQXV2dXSAFoPlQM2/ufZPbLriNRRctijocERkCG+DD9+BfyGwksB/4trs3pNY9C3zq7vektd0MrHT35anlG4Gb3L3fK9OWlJzrY8f+hqKiwxQXH+r3vqjoEMXFx96XlBykqOhw1l+hbGpqUoXSlL0H9zJ25FiM5H4VtL6+no6ODqqqqqIOJRb0/ugRl754/fXX33H3jBtomDOBc4CO7gSQsg24PEPbqanHerebmulFzWw+wcyB0tJSxo9fPOiAurqCW3eFzLB0dnbS1NQU7ovmkS9Hfcm+M/cx4d0JWIfRfKR54CcVsI6ODtw90dtEb0l/f/SWD30RZhIoBw6krWsGMl0/sDz1WO925WZm6ccFUrOF5QBVVVW+devW8CIeorq6usTuDmo80MiFT1/ISXYSr9W8xs6tOxPbF92qq6tpamqivr4+6lBiIcnvj3Rx6Yv+TtoM89BlK5BehLgCaBlE2wqgdaADwxKtlsMtXL3qaloOt/Dy9S9zWnnuv6YrIuEKMwk0ACVmdnavddOB9IPCpNZNH0Q7iYmOrg7m/mUu2z/bzpo5a5g2blrUIYlICEJLAu5+EFgL3G9mI83sEuAa4NkMzf8A3GFmp5vZBOBOYGVYsUj43v/8fTbt2cSTs55k5jdnRh2OiIQk7K+I3gKsAD4D9gE3u/sOM7sUeMXdy1PtlgHfAN5LLT+dWicxNW3cNBpua9AuIJECE2oScPcvCL7/n77+DYKDwd3LDvwydZMYq32/lj3Ne1h00SIlAJEClIBzWmWotny6hXlr57F6x2raO9ujDkdEhoGSgGS0e/9uZv9xNqeVn8a669apKqhIgcqrAnKSG/u/3M+sVbNo72yn7qd1fG3kMFyFXkRiQUlAjrNx10Z27d/Fqz96lSmnTok6HBEZRkoCcpw5U+dw8dcvZmLFxKhDEZFhpmMCctSjmx9lw0cbAJQARBJCSUAAWPXeKu7acBerth93+QcRKWBKAsIbH7/BDetu4LIzLuOpWf1e0kFECoySQMI17Gvg2j9dy1mjz6J2bi0jSkZEHZKI5JCSQMKtrF9JkRWx/vr1jDlpTNThiEiOKQkk3APfe4B35r/D5DGTow5FRCKgJJBAXd7F3Rvu5sMvPsTMmDRqUtQhiUhElAQS6N5/3MvDmx/mxQ9ejDoUEYmYkkDCrPj3Ch7c9CA3nX8Tt190e9ThiEjElAQSZOOujSx4aQFXTr6Smqtq+r3uqIgkg5JAQrg7j2x+hCmVU1gzZw2lxaVRhyQiMaDaQQlhZtTOraX5UDMVIyqiDkdEYkIzgQLXdqSNO/92JwcOH6CstIzxJ4+POiQRiRElgQLW2dXJvLXzWPrWUt5ufDvqcEQkhrQ7qIAt3rCYF3a+wBPff4IZk2dEHY6IxJBmAgWqZksNS99aysILFrLwwoVRhyMiMRVKEjCzMWZWa2YHzexjM7u+n7aLzWy7mbWY2W4zWxxGDNKj7UgbD735ELPPmc1jMx+LOhwRibGwdgfVAO3AOOA8YL2ZbXP3HRnaGvAT4F1gMvCamX3i7qtDiiXxykrL2HzjZkZ/dTTFRcVRhyMiMZb1TMDMRgI/BO5191Z33wT8Ffhxpvbu/rC7/8vdO9z9A2AdcEm2cQg0Hmjkvrr76OzqZGLFRMq/Uh51SCISc2HMBM4BOty9ode6bcDlAz3RglNWLwWW9dNmPjA/tdhqZh9kEWtYKoG9UQfRlyUsyeWvi3Vf5FClmakfAtomesSlL87o64EwkkA5cCBtXTNw8iCeu4RgNvK7vhq4+3Jg+VCDGw5mttXdq6KOIw7UFwH1Qw/1RY986IsBdweZWZ2ZeR+3TUArkH4KagXQMsDr3kpwbGCWux8e6h8gIiJDN+BMwN2r+3s8dUygxMzOdvf/pFZPBzIdFO5+zs+Ae4DL3L1x8OGKiEiYsj4w7O4HgbXA/WY20swuAa4Bns3U3szmAQ8CM9x9V7a/PyKx2j0VMfVFQP3QQ33RI/Z9Ye6e/YuYjQFWADOAfcA97r4q9dilwCvuXp5a3g1MBHrvAnrO3X+RdSAiInJCQkkCIiKSn1Q2QkQkwZQEREQSTEkgS2Z2tpkdMrPnoo4lCmY2wsyeSdWMajGzejP7QdRx5cqJ1M0qZEnfDvqSD+ODkkD2aoB/Rh1EhEqATwjOEB8F/Ar4s5mdGWFMudS7btY84LdmNjXakCKR9O2gL7EfH5QEsmBm1wFNwN+jjiUq7n7Q3Ze4+3/dvcvdXwJ2A9+JOrbhdqJ1swpZkreDvuTL+KAkMERmVgHcD9wRdSxxYmbjCOpJ9XmyYAHpq25WEmcCx0jYdnCcfBoflASG7tfAMzrjuYeZlQLPA793951Rx5MD2dTNKlgJ3A4yyZvxQUkgg4HqJZnZecAVwNKoYx1ug6gd1d2uiOAs8Xbg1sgCzq0h1c0qZAndDo6Rb+ODrjGcwSDqJd0OnAnsCaphUw4Um9m33P38YQ8whwbqCzhaEvwZgoOjV7n7keGOKyYaOMG6WYUswdtBumryaHzQGcNDYGZlHPsJ8C6Cf/rN7v55JEFFyMyeIrii3BXu3hp1PLlkZqsBB35O0AcvA9/t46p6BS3J20Fv+TY+aCYwBO7eBrR1L5tZK3Aojv/g4WZmZwALCGpB/S/1yQdggbs/H1lguXMLQd2szwjqZt2c0ASQ9O3gqHwbHzQTEBFJMB0YFhFJMCUBEZEEUxIQEUkwJQERkQRTEhARSTAlARGRBFMSEBFJMCUBEZEE+z9DTfR0i/Vl5wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plotting the Logit function\n",
    "z = np.linspace(-5, 5, 200)\n",
    "\n",
    "plt.plot([-5, 5], [0, 0], 'k-')\n",
    "plt.plot([-5, 5], [1, 1], 'k--')\n",
    "plt.plot([0, 0], [-0.2, 1.2], 'k-')\n",
    "plt.plot([-5, 5], [-3/4, 7/4], 'g--')\n",
    "plt.plot(z, logit(z), \"b-\", linewidth=2) # calling the Logit function we created\n",
    "props = dict(facecolor='black', shrink=0.1)\n",
    "plt.annotate('Saturating', xytext=(3.5, 0.7), xy=(5, 1), arrowprops=props, fontsize=14, ha=\"center\")\n",
    "plt.annotate('Saturating', xytext=(-3.5, 0.3), xy=(-5, 0), arrowprops=props, fontsize=14, ha=\"center\")\n",
    "plt.annotate('Linear', xytext=(2, 0.2), xy=(0, 0.5), arrowprops=props, fontsize=14, ha=\"center\")\n",
    "plt.grid(True)\n",
    "plt.title(\"Sigmoid activation function\", fontsize=14)\n",
    "plt.axis([-5, 5, -0.2, 1.2])\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Xavier and He Initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Constant',\n",
       " 'GlorotNormal',\n",
       " 'GlorotUniform',\n",
       " 'HeNormal',\n",
       " 'HeUniform',\n",
       " 'Identity',\n",
       " 'Initializer',\n",
       " 'LecunNormal',\n",
       " 'LecunUniform',\n",
       " 'Ones',\n",
       " 'Orthogonal',\n",
       " 'RandomNormal',\n",
       " 'RandomUniform',\n",
       " 'TruncatedNormal',\n",
       " 'VarianceScaling',\n",
       " 'Zeros',\n",
       " 'constant',\n",
       " 'deserialize',\n",
       " 'get',\n",
       " 'glorot_normal',\n",
       " 'glorot_uniform',\n",
       " 'he_normal',\n",
       " 'he_uniform',\n",
       " 'identity',\n",
       " 'lecun_normal',\n",
       " 'lecun_uniform',\n",
       " 'ones',\n",
       " 'orthogonal',\n",
       " 'random_normal',\n",
       " 'random_uniform',\n",
       " 'serialize',\n",
       " 'truncated_normal',\n",
       " 'variance_scaling',\n",
       " 'zeros']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[name for name in dir(keras.initializers) if not name.startswith(\"_\")] # sneak peek at the Keras initializers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<keras.layers.core.Dense at 0x7ff237ae7eb8>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keras.layers.Dense(10, activation=\"relu\", kernel_initializer=\"he_normal\") # He Initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<keras.layers.core.Dense at 0x7ff237269358>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# He initialization with a uniform distribution but based on fan(avg) rather than fan(in)\n",
    "init = keras.initializers.VarianceScaling(scale=2., mode='fan_avg',\n",
    "                                          distribution='uniform')\n",
    "keras.layers.Dense(10, activation=\"relu\", kernel_initializer=init)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on package keras.api._v2.keras in keras.api._v2:\n",
      "\n",
      "NAME\n",
      "    keras.api._v2.keras - Public API for tf.keras namespace.\n",
      "\n",
      "PACKAGE CONTENTS\n",
      "    __internal__ (package)\n",
      "    activations (package)\n",
      "    applications (package)\n",
      "    backend (package)\n",
      "    callbacks (package)\n",
      "    constraints (package)\n",
      "    datasets (package)\n",
      "    estimator (package)\n",
      "    experimental (package)\n",
      "    initializers (package)\n",
      "    layers (package)\n",
      "    losses (package)\n",
      "    metrics (package)\n",
      "    mixed_precision (package)\n",
      "    models (package)\n",
      "    optimizers (package)\n",
      "    premade (package)\n",
      "    preprocessing (package)\n",
      "    regularizers (package)\n",
      "    utils (package)\n",
      "    wrappers (package)\n",
      "\n",
      "VERSION\n",
      "    2.6.0\n",
      "\n",
      "FILE\n",
      "    /usr/local/anaconda/envs/tensorflow2/lib/python3.6/site-packages/keras/api/_v2/keras/__init__.py\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(keras)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Nonsaturating Activation Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Leaky ReLU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# defining the Leaky ReLU activation function\n",
    "def leaky_relu(z, alpha=0.01):\n",
    "    return np.maximum(alpha*z, z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWsAAAEMCAYAAAACt5eaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de3xU1b338c+PAJJwRyRF6gFFxGtBjVcqBqp4q7fH1gcVW7QIWqnWg1gei4WCHLVeKlVUQBRFUKytVkU4VTAoLccKiodSFUWhoiAoBAghCUzW88eawBBymZBM1ly+79crr+zZs7P3d3Z2flmzZu29zTmHiIgktyahA4iISO1UrEVEUoCKtYhIClCxFhFJASrWIiIpQMVaRCQFqFinCDMrMLOHQ+dIB2aWb2bOzDo2wrZWm9mtjbCdI81ssZmVmNnqRG8vjjzOzH4UOkc6UbFuAGY23cxeDZ2jrqL/AFz0q8zMVpnZXWZ2QB3XM9jMimrZzj7/aGr7uYZQTbH8O9AZ+LYBtzPWzP5ZxVMnAY801HZqcCdQDBwZ3WajqOHY7wy80lg5MkHT0AEkuCeB24Hm+D/yJ6Pz/1+wRAnmnCsD1jfStjY2xnaAw4G/OOdWN9L2auSca5T9m0nUsm4EZtbWzKaY2QYz22ZmC80sL+b5A83sWTNba2Y7zGyFmV1Tyzp/YGaFZna9mfU1s51m9p1Ky0wws/+tJV6xc269c+7fzrk/Aa8DAyqtp4uZPWdmm6Nfc8ysRx13w34xs7vN7OPoflltZr8zsxaVljnfzN6JLvOtmb1iZi3MrADoCtxb8Q4iuvzubhAzaxP9uQsrrXNAdJ92qi2HmQ0GxgDHxLxTGRx9bq+WvZn9h5m9GD0OtpnZn83suzHPjzWzf5rZwOg7nW1m9lJNXTbR19UL+E1022PNrFt0Oq/yshXdEzHLXGZmr5tZsZn9y8zOrvQzR5rZy2a2xcyKot0tx5nZWOCnwAUxrzu/8naij48zszei+29TtEXeNub56Wb2qpndbGZfRo+zJ80sp7rXnWlUrBPMzAyYA3QBfggcD7wFLDCzztHFWgDvRZ8/BpgITDazH1Szzh8BLwJDnXOPOefeAlYBP4lZpkn08bQ6ZO0F9AF2xszLAd4ESoAzgdOAdcAbjfSHtB24FjgK+DkwEPh1TL5zgZfx/2ROBPoBC/HH9v8B1gLj8G/LO1OJc24r/u36VZWeugp43Tm3IY4cs4H7gY9jtjO78raiv5O/ALnRnP2Ag4GXosdJhW7A/wUuxf/jPB6YUM3+Ibq9j6MZOgP31bBsVSYAf8AX/HeB58ysVTTzwcAiwAFnAycAk4Cs6HaeB96Ied1/r+J1twT+GygCTo6+rtOBJyotegZwLHAWe17/zXV8LenLOaeven4B04FXq3muP/4gza40fxlwWw3rfA54POZxAfAwMBTYAgyotPytwIcxj88DSoEDa9hGAVAWzVeK/4OMAJfFLHMt8AlgMfOy8P29l0cfDwaKatnOw1XMr/HnqlnX9cCnMY//BjxXw/KrgVsrzcuPvtaO0ccX4ft7W0cfZwNbgSvrkGMs8M+ato8vdhGgW8zzhwHlwFkx6ykB2sYs8+vYbVWT55/A2JjH3aKvMa/Scg74UaVlhsU83yU67/vRxxOANUDzuhz7lbZzXfSYbV3F7+DwmPV8AWTFLDMVeGN//ibT8Ust68Q7EcgBNkbfQhaZ/1DtWKA7gJllmdmvzex/o2/ji/Ctwv+otK5L8K2ac51zf6303FPAYWZ2evTxtcBLzrnaPkSbDfTGt5ifB6Y63x0Sm/9QYFtM9i1A+4r8iWRmPzKzRWa2Prrt37P3fjkemF/PzczFF+tLo48vAgx4qQ454nEU8JWL6Vd2zn0GfAUcHbPcGufclpjHXwGd6rituojtKvsq+r1ie8cDi5zv599fRwH/65zbFjPv7/h/UrGv+1/OuUilLIl83SlFHzAmXhPga/xbvMq2Rr/fCozAv+Vbjm/p/hf7HqgfAMcBPzOz/3HR5gf4D7LM7GXgWjP7GF9wLqR2W5xznwKY2SBghZkNds5Nj8m/DP+2v7JNcawf/OtsW8X8dvjCXyUzOxX/DuO3wC1AIf511fVtfo2cczvN7Hl818fT0e8vOueKGzFH7OUvd1bxXF0bVuXR77u7V8ysWTXL7t6ec85Fe2QaqyHX0K87balYJ957+D7K8mgrqirfB15xzs2A3f3cR+CLQqzPgV/guxWmmNnQ2IKNf9v4AvAZfrTDG3UJGi1a/wXcZWbPR4vVe8AVwDfOucp54vUxcL6ZWaW8J0Sfq04f4Evn3PiKGWbWtdIy7wM/wL/2qpThu21q8wzwlpkdDZyL//ygLjni2c6HwMFm1q2idW1mh+H7rf8VR8a6qBiFEttP33s/1vM+MMjMmlfTuo73dV9rZq1jWten4wvxh/uRKSPpv1bDaWNmvSt9dcMXzL8BfzGz88zsUDM7zcx+a2YVre2VwA/M7PtmdiS+b/rQqjYSLfj98AVlcqUPpl7H9yWPAaY758qrWEVtZuFbNMOjj2fi3xn8xczOjObva2b3294jQppU8fqPjT73KL5v9iEz62VmPc3sFvw/gXtryLIS6GJmV5nZYWZ2Q/RnYk0Afmxmd5rZ0WZ2jJndEvPh52rgDPMjWqodUeGc+zu+b3YW8A17d63Ek2M10NXMTjA/yqSqsepv4LscZppZnvmRGjPx/xAX1LAf6sw5twP4H+BX0X1yOvv3TuARoBXwvJmdZGaHm9kVZlZR+FcDx0Z/px2rab3PxHczPW1+VEhfYDLw54p3dVI7FeuGcwa+FRL7dV+0JXk+/o9xKr4l+TzQkz39g3cC/8D3nb6FH3kws7oNOedW4T+gOY+Ygh3d1pNAM/aMl66TaOvpYeC2aEuoGOiLb63/EfgI3z/eHtgc86PZVbz+gug6P4uuowfw1+hrHQj82Dk3t4Ysr+CL+YP4Inc28JtKy7yG72s+L7rNhfh/ZhX/qH4DHIIfLVPbmOeZ+BERz8X2ncaTA/gT8Bq+yG9k32Je8fu5OPr8m9Gv9cAlld5xNJRro9/fxRfH0XVdgXPuS/zvrjk+7/v4d3e7ootMxbeOl+BfV58q1lEMnAO0wf/u/wIsjskncbDEHCMSipk9iv+E/exaFxaRlKE+6zRh/gSDo/Fjqy8PHEdEGpiKdfr4C/6Eg2nOuTmhw4hIw1I3iIhICtAHjCIiKSBh3SAdO3Z03bp1S9Tq47J9+3ZatmwZNEOy0L7wPv74YyKRCEcffXTtC2cAHRd7VLUvvvoK1q2DZs3g6KOhaSN0HC9duvQb59xBlecnbNPdunVjyZIliVp9XAoKCsjPzw+aIVloX3j5+fkUFhYGPzaThY6LPSrvi7ffhvx8MIN586B//8bJYWZrqpqvbhARkUo2b4arroLycvjVrxqvUNdExVpEJIZzcN118MUXcPLJMG5c6ESeirWISIzHH4c//Qlat4Znn/X91clAxVpEJOrDD+Hm6O0OHn0UDjssbJ5YdSrWZtbD/N2Tn0lUIBGREMrKmnDFFbBjB1x9te+zTiZ1bVlPwl8URkQkrUyZchgffACHHw6TJoVOs6+4i7WZDcRfX7m+d+UQEUkqc+bAn/70XZo2hVmzfH91somrWJtZG/xNR/8zsXFERBrXunUweLCfnjABTjopaJxqxXtSzHj8BYLW7n2t+72Z2VD8DV3Jzc2loKCg3gHro6ioKHiGZKF94RUWFhKJRLQvojL9uCgvh9tu+x7ffNOB3r03kpe3gmTdHbUW6+gdIc7C3zizRs65KcAUgLy8PBf6zCidnbWH9oXXrl07CgsLtS+iMv24uPdeWLoUOnaE0aM/oX///MCJqhdPyzoff8v6f0db1a2ALDM72jl3QuKiiYgkzrvvwu23++np06Fly/rcwD3x4umzngJ0x99sszfwGDAHf5seEZGUs20bXHEF7NoFN90EF1wQOlHtam1ZR++fVlzx2MyKgBLnXG33sxMRSUrDh8OqVdCrF9xzT+g08anzVfecc2MTkENEpFHMnAlPPw3Z2f508hYtQieKj043F5GM8dlncMMNfnriRDjqqLB56kLFWkQyws6dvp962za47DIYMiR0orpRsRaRjDBmDPzjH3DIITB1qr+pQCpRsRaRtLdgAdx9NzRp4vus27cPnajuVKxFJK198w0MGuRvKnDHHXDGGaET7R8VaxFJW87Btdf663/06QOjR4dOtP9UrEUkbT3yCLzyCrRr57s/GuPu5ImiYi0iaWn5chgxwk9PnQpdu4bNU18q1iKSdoqLYeBAKC31Q/R+9KPQiepPxVpE0s6IEfCvf8GRR8KDD4ZO0zBUrEUkrbz4Ijz2GDRv7k8nb9kydKKGoWItImnjiy/gZz/z07/7HfTuHTZPQ1KxFpG0EIn4u5Jv3gznn+8vfZpOVKxFJC3cdRcsXAi5ufDkk6l3OnltVKxFJOX9/e8wdqyfnjEDOnUKGichVKxFJKUVFsKVV/pukJEj4eyzQydKDBVrEUlZzsH118OaNZCXB3feGTpR4qhYi0jKmj4dZs/2w/NmzfLD9dKVirWIpKSPP4Zf/MJPP/II9OgRNk+iqViLSMopLfV3fdm+3fdXX3116ESJp2ItIinn9tvh/ffh0EPh0UfTb5heVVSsRSSlzJsHDzwAWVm+n7pNm9CJGoeKtYikjK+/hp/+1E+PHw+nnho2T2NSsRaRlFBe7gv1hg3Qrx/cdlvoRI1LxVpEUsKDD8J//zcceKA/SzErK3SixqViLSJJ7733YNQoPz1tGnTpEjZPCCrWIpLUior8ML2dO+HGG+Hii0MnCkPFWkSS2k03wcqVcOyxcO+9odOEo2ItIklr9mx/udMWLeC55yA7O3SicFSsRSQprV4NQ4f66d//Ho45Jmic4FSsRSTp7NrlTyPfuhUuuQSGDQudKDwVaxFJOr/9LSxe7Ed9PP54ZpxOXhsVaxFJKgsXwoQJvkA/84wfVy0q1iKSRDZtgkGD/E0Ffv1ryM8PnSh5qFiLSFJwDoYMgbVr4bTTYMyY0ImSi4q1iCSFyZPhxRf9VfRmzYKmTUMnSi4q1iIS3IoVcMstfnryZOjWLWicpBRXsTazZ8xsnZltNbOVZjYk0cFEJDPs2OFPJy8pgWuugYEDQydKTvG2rO8Cujnn2gAXAXea2YmJiyUimWLkSFi+HI44Av7wh9Bpkldcxdo5t8I5V1rxMPrVPWGpRCQjvPwyTJoEzZrBs89Cq1ahEyWvuLvwzewRYDCQDbwPvFbFMkOBoQC5ubkUFBQ0SMj9VVRUFDxDstC+8AoLC4lEItoXUSGPi40bmzNkyElAM4YM+ZStW9cS8teS7H8j5pyLf2GzLOA0IB+4xzm3s7pl8/Ly3JIlS+odsD4KCgrI10BNQPuiQn5+PoWFhSxbtix0lKQQ6riIRODss+HNN+Gcc+C116BJ4OEOyfI3YmZLnXN5lefXafc45yLOuUXAd4EbGiqciGSW3/3OF+pOneCpp8IX6lSwv7uoKeqzFpH98M47cMcdfvqppyA3N2yeVFFrsTazTmY20MxamVmWmZ0DXAHMT3w8EUknW7f6YXqRiB9Xfe65oROljng+YHT4Lo/H8MV9DfBL59zLiQwmIunFObjhBvj8czj+eLjrrtCJUkutxdo5txE4sxGyiEgamzHDn0aek+OH6R1wQOhEqUXd+iKScJ9+6m92C/DQQ9CzZ9g8qUjFWkQSqqzM91MXFcHll/tTyqXuVKxFJKHuuAOWLIGuXf1FmnTXl/2jYi0iCfP6635MdVaW769u1y50otSlYi0iCbFxI/zkJ356zBg4/fSweVKdirWINDjnfN/0+vXQty/cfnvoRKlPxVpEGtxDD8GcOdC+vb/pbVZW6ESpT8VaRBrUsmX+GtUA06bBIYeEzZMuVKxFpMFs3+6H6ZWVwbBhcOmloROlDxVrEWkwt9wCH30ERx8NDzwQOk16UbEWkQbxwgswdao/jfy55/xp5dJwVKxFpN7+/W+47jo/fd99cNxxYfOkIxVrEamXXbvgqqugsBAuvHDPNUCkYalYi0i9TJgAixZB587wxBM6nTxRVKxFZL8tWgTjxvkC/cwz0LFj6ETpS8VaRPbL5s1w5ZVQXg6/+hX07x86UXpTsRaROnMOhg6FL76Ak0/2rWtJLBVrEamzadP8UL3Wrf3V9Jo1C50o/alYi0idfPgh3HSTn370UejePWyeTKFiLSJxKynxp5Pv2AFXX+2H7EnjULEWkbiNGgUffOBb05MmhU6TWVSsRSQuc+bAxInQtKm/O3nr1qETZRYVaxGp1bp1MHiwn54wAU46KWicjKRiLSI1Ki/3t+f65hs46yy49dbQiTKTirWI1Oj+++GNN/zZiU8/DU1UNYLQbheRar377p77J06f7q//IWGoWItIlbZt88P0du3y46ovuCB0osymYi0iVRo+HFatgl694J57QqcRFWsR2cesWb5/OjvbD9Nr0SJ0IlGxFpG9fPYZXH+9n544EY46Kmwe8VSsRWS3nTt9P/W2bXDZZTBkSOhEUkHFWkR2GzMG/vEPOOQQf/Nb3fUleahYiwgACxbA3Xf7cdQzZ0L79qETSSwVaxHhm2/8VfScgzvugDPOCJ1IKlOxFslwzsG118JXX0GfPjB6dOhEUhUVa5EM98gj8Mor0Lat7/5o2jR0IqlKrcXazA4ws2lmtsbMtpnZMjM7rzHCiUhiffZZS0aM8NNTp0LXrmHzSPXiaVk3Bb4AzgTaAqOB582sW+JiiUiiFRfD+PFHU1rqh+j9+MehE0lNan3D45zbDoyNmfWqmX0OnAisTkwsEUm0ESNg9eqWHHkkPPhg6DRSmzr3TplZLnAEsKKK54YCQwFyc3MpKCiob756KSoqCp4hWWhfeIWFhUQikYzfF2+/3ZHHHjuWpk0jjBjxPu++WxQ6UnDJ/jdizrn4FzZrBswFVjnnhtW0bF5enluyZEk949VPQUEB+fn5QTMkC+0LLz8/n8LCQpYtWxY6SjBr1/qLM23aBDfe+AkPP9wjdKSkkCx/I2a21DmXV3l+3KNBzKwJMAMoA4Y3YDYRaSSRCAwa5Av1+efDZZd9GTqSxCmuYm1mBkwDcoHLnHM7E5pKRBLirrtg4ULIzYUnn9Tp5Kkk3pb1o8BRwIXOuR0JzCMiCbJ4MYwd66effho6dQoaR+oonnHWXYFhQG9gvZkVRb+uSng6EWkQhYX+anqRCIwcCQMGhE4kdRXP0L01gN4siaQo5/z1qdesgbw8uPPO0Ilkf+h0c5E0N306zJ4NLVv6O8A0bx46kewPFWuRNLZyJfziF3560iTooVF6KUvFWiRNlZbCwIGwfTtceSX85CehE0l9qFiLpKnbb4f334dDD4VHH9UwvVSnYi2ShubNgwcegKws30/dpk3oRFJfKtYiaebrr+GnP/XT48fDqaeGzSMNQ8VaJI2Ul8PgwbBhA/TrB7fdFjqRNBQVa5E08uCDvgvkwANhxgzfDSLpQcVaJE289x6MGuWnp02DLl3C5pGGpWItkgaKivzp5Dt3wo03wsUXh04kDU3FWiQN3HyzPwHm2GPh3ntDp5FEULEWSXGzZ8MTT0CLFvDcc5CdHTqRJIKKtUgKW70ahg710w88AMccEzSOJJCKtUiK2rXLn0a+dStccom/sp6kLxVrkRQ1bpy/oUCXLvD44zqdPN2pWIukoIUL/XWpzeCZZ/y4aklvKtYiKWbTJn/TW+f8xZqS4Ibc0ghUrEVSiHMwZAisXQunnQZjxoROJI1FxVokhUyZAi++6K+iN2sWNGsWOpE0FhVrkRSxYgX88pd+evJk6NYtaBxpZCrWIimgpMSfTl5S4q+qN3Bg6ETS2FSsRVLAyJGwfLm/h+JDD4VOIyGoWIskuVdegYcf9v3Tzz0HrVqFTiQhqFiLJLEvv4RrrvHTd90FJ5wQNo+Eo2ItkqQiEX9H8m+/hQED4JZbQieSkFSsRZLUvffCggXQqRM89RQ00V9rRtOvXyQJvfMOjB7tp596Cr7znbB5JDwVa5Eks3WrH6YXifiuj3PPDZ1IkoGKtUiS+fnP4fPP4fjj/YeKIqBiLZJUZsyAmTMhJweefRYOOCB0IkkWKtYiSeLTT32rGvyJLz17hs0jyUXFWiQJlJX5fuqiIrj88j1jq0UqqFiLJIE77oAlS6BrV3+RJt31RSpTsRYJ7PXX4Xe/g6wsf9nTdu1CJ5JkpGItEtDGjf4sRfA3Ejj99LB5JHmpWIsE4pzvm16/Hvr29bfoEqlOXMXazIab2RIzKzWz6QnOJJIRHnoI5syB9u39TW+zskInkmTWNM7lvgLuBM4BshMXRyQzfPCBv0Y1wLRpcMghYfNI8ourWDvn/gxgZnnAdxOaSCTNbd/u7/RSVgbDhsGll4ZOJKkg3pZ1XMxsKDAUIDc3l4KCgoZcfZ0VFRUFz5AstC+8wsJCIpFI0H1x331H8NFHB9O163YuuWQpBQXlwbLouNgj2fdFgxZr59wUYApAXl6ey8/Pb8jV11lBQQGhMyQL7QuvXbt2FBYWBtsXL7zg+6kPOABefrkl3/te3yA5Kui42CPZ94VGg4g0kn//G667zk/fdx9873th80hqUbEWaQS7dsFVV0FhIVx4Idx4Y+hEkmri6gYxs6bRZbOALDNrAexyzu1KZDiRdDFhAixaBJ07wxNP6HRyqbt4W9ajgR3AKGBQdHp0okKJpJNFi2DcOF+gZ8yAjh1DJ5JUFO/QvbHA2IQmEUlDmzf77o/ychg1Cn7wg9CJJFWpz1okQZyDoUP9B4snn+xb1yL7S8VaJEGmTfND9Vq39lfTa9YsdCJJZSrWIgnw0Udw881++tFHoXv3sHkk9alYizSwkhJ/OnlxMVx9te+zFqkvFWuRBjZqlL9QU/fuMGlS6DSSLlSsRRrQa6/BxInQtKm/O3nr1qETSbpQsW4A+fn5DB8+PHQMCWzdOhg82E9PmAAnnRQ0jqSZjCjWgwcP5oc//GHoGJLGysv97bk2boSzzoJbbw2dSNJNRhRrkUS7/3544w1/duLTT0MT/WVJA8v4Q2rLli0MHTqUTp060bp1a84880yWLFmy+/lvv/2WK664gu9+97tkZ2dzzDHH8OSTT9a4zvnz59OuXTsee+yxRMeXJLBkyZ77J06f7q//IdLQMrpYO+e44IIL+PLLL3n11Vd5//336du3L/3792fdunUAlJSUcMIJJ/Dqq6+yYsUKbr75ZoYNG8b8+fOrXOcLL7zApZdeypQpU7j++usb8+VIANu2wRVX+Kvq3XQTXHBB6ESSrhr05gOp5s0332TZsmVs3LiR7Gx/a8nx48fzyiuvMGPGDG677Ta6dOnCyIqb5QFDhw5lwYIFPPvss/yg0oUepkyZwsiRI3nhhRcYMGBAo74WCWP4cPj0U+jVC+65J3QaSWcZXayXLl1KcXExBx100F7zS0pKWLVqFQCRSIS7776b2bNn8+WXX1JaWkpZWdk+d5R46aWXmDx5Mm+99RannXZaY70ECWjWLN8/nZ3th+m1aBE6kaSzjC7W5eXl5Obm8vbbb+/zXJs2bQC47777uP/++5k4cSLHHXccrVq14vbbb2fDhg17Ld+rVy+WL1/OtGnTOPXUUzFdsDitffYZVPRyTZwIRx0VNo+kv4wu1ieccAJff/01TZo04bDDDqtymUWLFnHhhRdy9dVXA76fe+XKlbRr126v5Q499FAeeugh8vPzGTp0KFOmTFHBTlM7d8KVV/r+6ssugyFDQieSTJAxHzBu3bqVZcuW7fV1+OGH06dPHy6++GLmzp3L559/zuLFixkzZszu1vYRRxzB/PnzWbRoER999BHDhw/n888/r3Ibhx12GG+++Sbz5s1j2LBhOOca8yVKIxkzBt55Bw45BKZO1V1fpHFkTLF+++23Of744/f6GjlyJK+99hr9+/fnuuuuo2fPnlx++eV8/PHHHHzwwQCMHj2ak08+mfPOO4++ffvSsmVLrqrhyjzdu3enoKCAuXPnqmCnoQUL4O67/TjqmTOhffvQiSRTZEQ3yPTp05k+fXq1z0+cOJGJEydW+Vz79u3585//XOP6CwoK9nrcvXt3vvjii7rGlCT3zTf+KnrOwW9+A2ecETqRZJKMaVmL1Idz8LOfwVdfQZ8+MFp3IJVGpmItEodHHoGXX4a2bX33R9OMeE8qyUTFWqQWy5fDiBF+eupU6No1bB7JTCrWIjXYscOfTl5a6ofo/fjHoRNJpkrpYl1cXMygQYOYM2dO6CiSpkaMgBUr4Mgj4cEHQ6eRTJayxXrDhg2ccsop/PGPf+Tyyy/f60p5Ig3hxRf9zW6bN/enk7dsGTqRZLKULNYrV66kd+/efPTRR5SVlVFcXMzZZ5/N6tWrQ0eTNLF27Z4zE++5B3r3DptHJOWK9d/+9jdOOukk1q9fz65du3bP37JlCxdddFHAZJIuIhEYNAg2bYLzz4ebbw6dSCTFivXs2bMZMGAAW7du3efMwOzsbG6vuAK8SD3cfTcsXAi5ufDkkzqdXJJDShRr5xx33XUX11xzDcXFxXs9Z2a0bt2aefPmMXDgwEAJJV0sXuyv/QH+8qedOoXNI1Ih6Yf2RyIRhg0bxrPPPsuOHTv2eq5p06Z07NiRgoICevbsGSihpIstW/zV9CIRGDkSdP8ISSZJXay3b9/OxRdfzOLFi/dpUbdo0YLu3buzYMECOqn5I/XkHAwbBqtXQ14e3Hln6EQie0vaYr1+/Xr69+/PZ599Rmlp6V7P5eTk0KdPH1566SVycnICJZR0Mn06zJ7th+fNmuWH64kkk6Tss/7www/p1asXn3zySZWFetCgQcydO1eFWhrEypXwi1/46UmToEePsHlEqpJ0xfqtt97ilFNOYcOGDXsNzQM/4uO3v/0tkydPJisrK1BCSSelpf508u3bfX/1T34SOpFI1YIU6yVLlnDKKadQWFi41/yZM2dy7rnnsm3btn1+Jicnh6eeeopbb721sWJKBvj1r+G99+DQQ/3ZihqmJ8kqSLGeMGECS5Ys4ZxzzqGsrAznHOPHj+e6667bZ4YTrSEAAAfzSURBVMSHmdGmTRveeOMNfqyr6EgDmjcP7r8fsrJ8P3X0HskiSanRP2DcuHEjc+fOpby8nOXLl3PllVfSqlUr/vjHP+5TqJs1a8ZBBx1EQUEBPdSRKA3o66/hpz/10+PGwamnhs0jUptGL9aPP/747rt+79ixg7lz5wJUOTSvR48ezJ8/n4MOOqixY0qaGzwYNmyAfv3gV78KnUakdnF1g5hZBzN70cy2m9kaM7tyfzZWXl7OxIkTKSkp2T2vuLh4n0Kdk5NDv379eOedd1SopcFt2HAA8+bBgQfCjBm+G0Qk2cXbsp4ElAG5QG9gjpl94JxbUZeN/fWvf2X79u01LpOTk8M111zDH/7wB5o0SbrBKpJidu2CwkJ/UabNm/0wvXXrsgGYNg26dAkcUCROVvmCSPssYNYS2Awc65xbGZ03A/jSOTequp9r3bq1O/HEE/ea98EHH+wzAiRWkyZN6Ny5M4cffnj8r6AGhYWFtGvXrkHWlepSfV9EIr7w7toFO3dW/b2qeZFI5TUtA6BHj94cfHCjv4ykk+rHRUNKln2xcOHCpc65vMrz42lZHwHsqijUUR8AZ1Ze0MyGAkPBfzgYW5jLysrYsmVLjRsqLy9n/fr1tGnThuYNcApZJBKp8Z9DJkmGfeEcRCIW89WEXbv2PK5uOhJpQi1tihplZbndX2VljmbNIuTkFKJDIzmOi2SR7PsinmLdCthaad4WoHXlBZ1zU4ApAHl5eS727i2jRo1i1apVlJWV1brB0tJSFi9eTNu2beOIV72CggLy8/PrtY500VD7wjl/X8KKboVNm+KfruV/dY2ys6FDB2jf3n+Pd7pNG4jtTcvPz6ewsJBly5bVe1+kA/2N7JEs+8KqGewfT7EuAiqPQG0D7HvmSjV27tzJY489FlehLi8vZ82aNYwbN477778/3k1IHUUivnjWpdhWPK50BYC4mUG7dnUrthWPW7Ro2NcvkmriKdYrgaZm1sM590l0Xi8g7g8XX3rpJSL7dh7u1rp1a0pLS+ncuTPnn38+5513Hv369Yt39RmtqlZuVQV31arv4dye+Vu2sN9dCwcc4EdS1LWV27bt3q1cEYlfrcXaObfdzP4MjDOzIfjRIBcDp8e7kXvuuYeioqLdj1u2bEkkEqFt27YMGDCACy64gH79+mXspU4rWrl17VbYvBliRkHWosNej+rTys3ObvBdICK1iHfo3s+BJ4ANwLfADfEO2/vkk09YunQp2dnZNG/enP79+3PRRRfRr18/unbtup+xk1NJSd2L7aZNfmjZ/rZymzevuZVb8fiLLz6gX79eu59r21bji0VSSVzF2jm3CbhkfzZw4IEHMnXqVL7//e/Ts2fPajvPk0V5efyt3MqP42/l7qtt25qLbXXT2dnxXXyooGAzJ520//lEJKyEn27eoUMHhgwZkujN7KOkBL79tjkrVtTenxs7vXlz/Vq5dS22HTr47gi1ckWkJkl7pxjwrdytW/dvmJi/JlTc3ep7adOmbsW2YjonR5fYFJHEaJRiXVq6fx+ebd7sC/b+aNYMWrUq4zvfaV6nUQvt2kHTpP4XJiKZKGFl6V//gkMO8YW30nWa6qRNm7oPEevQwbdyFy78e1IMchcRqa+EFesdO2Dt2uhGmu7fELF27XwLWUQk0yWsWB91lL8TR4cO/o7R6ssVEdl/CSvWOTnwH/+RqLWLiGQWnfwrIpICVKxFRFKAirWISApQsRYRSQEq1iIiKUDFWkQkBahYi4ikABVrEZEUoGItIpICzO3vxZtrW7HZRmBNQlYev47AN4EzJAvtiz20L/bQvtgjWfZFV+fcQZVnJqxYJwMzW+KcywudIxloX+yhfbGH9sUeyb4v1A0iIpICVKxFRFJAuhfrKaEDJBHtiz20L/bQvtgjqfdFWvdZi4iki3RvWYuIpAUVaxGRFKBiLSKSAjKqWJtZDzMrMbNnQmcJwcwOMLNpZrbGzLaZ2TIzOy90rsZiZh3M7EUz2x7dB1eGzhRCph8H1Un2+pBRxRqYBLwbOkRATYEvgDOBtsBo4Hkz6xYwU2OaBJQBucBVwKNmdkzYSEFk+nFQnaSuDxlTrM1sIFAIzA+dJRTn3Hbn3Fjn3GrnXLlz7lXgc+DE0NkSzcxaApcBdzjnipxzi4CXgavDJmt8mXwcVCcV6kNGFGszawOMA/4zdJZkYma5wBHAitBZGsERwC7n3MqYeR8Amdiy3kuGHQf7SJX6kBHFGhgPTHPOrQ0dJFmYWTNgJvCUc+6j0HkaQStga6V5W4DWAbIkjQw8DqqSEvUh5Yu1mRWYmavma5GZ9QbOAn4fOmui1bYvYpZrAszA998ODxa4cRUBbSrNawNsC5AlKWTocbCXVKoPTUMHqC/nXH5Nz5vZL4FuwL/NDHwLK8vMjnbOnZDwgI2otn0BYH4nTMN/yHa+c25nonMliZVAUzPr4Zz7JDqvF5n71j9Tj4PK8kmR+pD2p5ubWQ57t6huxf9ybnDObQwSKiAzewzoDZzlnCsKnacxmdlzgAOG4PfBa8DpzrmMK9iZfBzESqX6kPIt69o454qB4orHZlYElCTbL6IxmFlXYBhQCqyPtiQAhjnnZgYL1nh+DjwBbAC+xf9BZmKhzvTjYLdUqg9p37IWEUkHKf8Bo4hIJlCxFhFJASrWIiIpQMVaRCQFqFiLiKQAFWsRkRSgYi0ikgJUrEVEUsD/B/fVMkeuiJ/dAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plotting the Leaky ReLU activation function\n",
    "plt.plot(z, leaky_relu(z, 0.05), \"b-\", linewidth=2) # calling the Leaky ReLU we created\n",
    "plt.plot([-5, 5], [0, 0], 'k-')\n",
    "plt.plot([0, 0], [-0.5, 4.2], 'k-')\n",
    "plt.grid(True)\n",
    "props = dict(facecolor='black', shrink=0.1)\n",
    "plt.annotate('Leak', xytext=(-3.5, 0.5), xy=(-5, -0.2), arrowprops=props, fontsize=14, ha=\"center\")\n",
    "plt.title(\"Leaky ReLU activation function\", fontsize=14)\n",
    "plt.axis([-5, 5, -0.5, 4.2])\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['deserialize',\n",
       " 'elu',\n",
       " 'exponential',\n",
       " 'gelu',\n",
       " 'get',\n",
       " 'hard_sigmoid',\n",
       " 'linear',\n",
       " 'relu',\n",
       " 'selu',\n",
       " 'serialize',\n",
       " 'sigmoid',\n",
       " 'softmax',\n",
       " 'softplus',\n",
       " 'softsign',\n",
       " 'swish',\n",
       " 'tanh']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = [] \n",
    "for m in dir(keras.activations):\n",
    "    if not m.startswith(\"_\"): # sneak peek at Keras activation functions\n",
    "        result.append(m)\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['deserialize',\n",
       " 'elu',\n",
       " 'exponential',\n",
       " 'gelu',\n",
       " 'get',\n",
       " 'hard_sigmoid',\n",
       " 'linear',\n",
       " 'relu',\n",
       " 'selu',\n",
       " 'serialize',\n",
       " 'sigmoid',\n",
       " 'softmax',\n",
       " 'softplus',\n",
       " 'softsign',\n",
       " 'swish',\n",
       " 'tanh']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[m for m in dir(keras.activations) if not m.startswith(\"_\")] # sneak peek at Keras activation functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['LeakyReLU', 'PReLU', 'ReLU', 'ThresholdedReLU']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[m for m in dir(keras.layers) if \"relu\" in m.lower()] # sneak peek at all Keras ReLU activation functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's train a neural network on Fashion MNIST using the Leaky ReLU:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loading the Fashion-MNIST dataset\n",
    "(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.fashion_mnist.load_data()\n",
    "\n",
    "# trimming the data since it takes lot of time\n",
    "X_train_full = X_train_full[:30000]\n",
    "y_train_full = y_train_full[:30000]\n",
    "\n",
    "X_test = X_test[:5000]\n",
    "y_test = y_test[:5000]\n",
    "\n",
    "# scaling the dataset\n",
    "X_train_full = X_train_full / 255.0\n",
    "X_test = X_test / 255.0\n",
    "\n",
    "# dividing the dataset into traingin and validation set\n",
    "X_valid, X_train = X_train_full[:5000], X_train_full[5000:]\n",
    "y_valid, y_train = y_train_full[:5000], y_train_full[5000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000, 28, 28)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.shape # checking shape of test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.random.set_seed(42) # setting random\n",
    "np.random.seed(42)\n",
    "\n",
    "# defining the model with Keras Sequential API\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]), # flatten the input\n",
    "    keras.layers.Dense(300, kernel_initializer=\"he_normal\"),\n",
    "    keras.layers.LeakyReLU(),\n",
    "    keras.layers.Dense(100, kernel_initializer=\"he_normal\"),\n",
    "    keras.layers.LeakyReLU(),\n",
    "    keras.layers.Dense(10, activation=\"softmax\")\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/anaconda/envs/tensorflow2/lib/python3.6/site-packages/keras/optimizer_v2/optimizer_v2.py:356: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  \"The `lr` argument is deprecated, use `learning_rate` instead.\")\n"
     ]
    }
   ],
   "source": [
    "# compiling the model\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
    "              optimizer=keras.optimizers.SGD(lr=1e-3),\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "782/782 [==============================] - 3s 3ms/step - loss: 1.6047 - accuracy: 0.5390 - val_loss: 1.1910 - val_accuracy: 0.6736\n",
      "Epoch 2/5\n",
      "782/782 [==============================] - 2s 3ms/step - loss: 1.0457 - accuracy: 0.6811 - val_loss: 0.9182 - val_accuracy: 0.7080\n",
      "Epoch 3/5\n",
      "782/782 [==============================] - 2s 3ms/step - loss: 0.8689 - accuracy: 0.7145 - val_loss: 0.8006 - val_accuracy: 0.7426\n",
      "Epoch 4/5\n",
      "782/782 [==============================] - 2s 3ms/step - loss: 0.7793 - accuracy: 0.7402 - val_loss: 0.7345 - val_accuracy: 0.7594\n",
      "Epoch 5/5\n",
      "782/782 [==============================] - 2s 3ms/step - loss: 0.7225 - accuracy: 0.7579 - val_loss: 0.6867 - val_accuracy: 0.7724\n"
     ]
    }
   ],
   "source": [
    "# training the model\n",
    "history = model.fit(X_train, y_train, epochs=5,\n",
    "                    validation_data=(X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's try PReLU:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating a new model with Parametric ReLU (PReLU)\n",
    "tf.random.set_seed(42)\n",
    "np.random.seed(42)\n",
    "\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.Dense(300, kernel_initializer=\"he_normal\"),\n",
    "    keras.layers.PReLU(),\n",
    "    keras.layers.Dense(100, kernel_initializer=\"he_normal\"),\n",
    "    keras.layers.PReLU(),\n",
    "    keras.layers.Dense(10, activation=\"softmax\")\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/anaconda/envs/tensorflow2/lib/python3.6/site-packages/keras/optimizer_v2/optimizer_v2.py:356: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  \"The `lr` argument is deprecated, use `learning_rate` instead.\")\n"
     ]
    }
   ],
   "source": [
    "# compiling the model\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
    "              optimizer=keras.optimizers.SGD(lr=1e-3),\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "782/782 [==============================] - 3s 3ms/step - loss: 1.6803 - accuracy: 0.5328 - val_loss: 1.2757 - val_accuracy: 0.6780\n",
      "Epoch 2/20\n",
      "782/782 [==============================] - 2s 3ms/step - loss: 1.1068 - accuracy: 0.6842 - val_loss: 0.9619 - val_accuracy: 0.7088\n",
      "Epoch 3/20\n",
      "782/782 [==============================] - 2s 3ms/step - loss: 0.9026 - accuracy: 0.7150 - val_loss: 0.8268 - val_accuracy: 0.7454\n",
      "Epoch 4/20\n",
      "782/782 [==============================] - 2s 3ms/step - loss: 0.8016 - accuracy: 0.7396 - val_loss: 0.7536 - val_accuracy: 0.7572\n",
      "Epoch 5/20\n",
      "782/782 [==============================] - 2s 3ms/step - loss: 0.7397 - accuracy: 0.7582 - val_loss: 0.7019 - val_accuracy: 0.7720\n",
      "Epoch 6/20\n",
      "782/782 [==============================] - 2s 3ms/step - loss: 0.6967 - accuracy: 0.7697 - val_loss: 0.6672 - val_accuracy: 0.7814\n",
      "Epoch 7/20\n",
      "782/782 [==============================] - 3s 3ms/step - loss: 0.6637 - accuracy: 0.7795 - val_loss: 0.6370 - val_accuracy: 0.7956\n",
      "Epoch 8/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.6369 - accuracy: 0.7901 - val_loss: 0.6143 - val_accuracy: 0.8024\n",
      "Epoch 9/20\n",
      "782/782 [==============================] - 2s 3ms/step - loss: 0.6150 - accuracy: 0.7972 - val_loss: 0.5943 - val_accuracy: 0.8052\n",
      "Epoch 10/20\n",
      "782/782 [==============================] - 2s 3ms/step - loss: 0.5965 - accuracy: 0.8031 - val_loss: 0.5789 - val_accuracy: 0.8122\n",
      "Epoch 11/20\n",
      "782/782 [==============================] - 3s 3ms/step - loss: 0.5807 - accuracy: 0.8094 - val_loss: 0.5708 - val_accuracy: 0.8128\n",
      "Epoch 12/20\n",
      "782/782 [==============================] - 2s 3ms/step - loss: 0.5668 - accuracy: 0.8132 - val_loss: 0.5523 - val_accuracy: 0.8174\n",
      "Epoch 13/20\n",
      "782/782 [==============================] - 2s 3ms/step - loss: 0.5547 - accuracy: 0.8148 - val_loss: 0.5439 - val_accuracy: 0.8234\n",
      "Epoch 14/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.5440 - accuracy: 0.8197 - val_loss: 0.5314 - val_accuracy: 0.8282\n",
      "Epoch 15/20\n",
      "782/782 [==============================] - 3s 3ms/step - loss: 0.5339 - accuracy: 0.8214 - val_loss: 0.5243 - val_accuracy: 0.8298\n",
      "Epoch 16/20\n",
      "782/782 [==============================] - 2s 3ms/step - loss: 0.5250 - accuracy: 0.8233 - val_loss: 0.5182 - val_accuracy: 0.8276\n",
      "Epoch 17/20\n",
      "782/782 [==============================] - 2s 3ms/step - loss: 0.5169 - accuracy: 0.8260 - val_loss: 0.5106 - val_accuracy: 0.8300\n",
      "Epoch 18/20\n",
      "782/782 [==============================] - 2s 3ms/step - loss: 0.5100 - accuracy: 0.8280 - val_loss: 0.5009 - val_accuracy: 0.8336\n",
      "Epoch 19/20\n",
      "782/782 [==============================] - 2s 3ms/step - loss: 0.5031 - accuracy: 0.8295 - val_loss: 0.4960 - val_accuracy: 0.8342\n",
      "Epoch 20/20\n",
      "782/782 [==============================] - 2s 3ms/step - loss: 0.4971 - accuracy: 0.8307 - val_loss: 0.4927 - val_accuracy: 0.8384\n"
     ]
    }
   ],
   "source": [
    "# training the model\n",
    "history = model.fit(X_train, y_train, epochs=20,\n",
    "                    validation_data=(X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ELU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# defining the ELU activation function\n",
    "def elu(z, alpha=1):\n",
    "    return np.where(z < 0, alpha * (np.exp(z) - 1), z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAEOCAYAAAB2GIfKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deZgU1b3G8e9vFkX21XFBJa4RjaIQczUqEyWKxF2jcSFBE1HARLhqEg3mmkjwcbuSGCWSaIgoKooxQFxuXFpcMaCojAoBAdlktcEZhhnoOfeP08MMMz17zVR39ft5nnroqVNd9eua6pea06erzDmHiIhEQ07YBYiISHAU6iIiEaJQFxGJEIW6iEiEKNRFRCJEoS4iEiEKdRGRCFGoi4hEiEJdAmNmk81sVoS2k2NmD5rZRjNzZlbY2tusp5Y2ec3JbXUzs7VmdlBbbK+pzOwpM7s+7DrSlekbpeEws8nAj1I0zXHO/Veyvadz7sw6nh8DFjjnrq0xfxjwR+dcx0ALbty2u+CPqXgmbaee7Z8JPAMUAp8Bm5xz5a25zeR2Y9R43W31mpPbugt/7F3R2ttKse2TgRuA/sA+wBXOuck1lvkG8BrwNefc5rauMd3lhV1AlnsJGFpjXquHRmtpqzdYG76RDwbWOOfeaqPt1amtXrOZtQd+ApzVFttLoSOwAHgkOdXinPvIzD4DLgfub8PaMoK6X8JV5pz7osa0qbU3amaDzex1M/vSzDaZ2Ytmdni1djOz683sP2ZWZmYrzez2ZNtkYCAwKtkl4cysT2Wbmc0ys+HJP99za2x3qpnNaEwdjdlOtfXsbmYTktvcZmbvmNmJ1dpjZvaAmY03sw1mts7M7jazOo//5PbvBfZPbntZtXX9seaylfU0ZlvN2b9Nfc3Nfd3AEMABb6bYJ/3N7GUzKzWzxWZ2spldZGa1lm0u59xzzrmbnXNPAxX1LDoDuCSo7UaJQj07dQAmAMfhuxY2AzPNbLdk+3jgFuB24Ajg+8CKZNt1wNvAX4G9k1NlW6WngC7AdytnmFlH4Bzg0UbW0ZjtVLoTuBi4EjgG+Ah4wcz2rrbMZcAO4ATgWmB08jl1uQ74LbAyue1v1rNsTQ1tq6X7Fxr3mhtTS00nAfNcjX5ZM/sm8DrwKnAU8A7wG+BXyddCjeVvNrPiBqaT6qmjIe8Cx5nZHi1YRzQ55zSFMAGT8W+24hrTHdXaZ9Xz/Bi+77zm/GFAcRNr6QAkgBPxf/5uA65pxrZ31ozvi55Sre1yfGi3a0wdTdhOB3yX1Q+rtecCS4Bx1dbzdo11/Av4SwP75QZgWUOvvUY99W6rufu3qa+5ua8beBb4W4r5s4Enq/08JPm7erWO9XTHd1/VN+3RwP4vBobV0XYU/i+Kg5pyrGfDpD71cM0GhteY1xYfhB0E3AZ8C+iF/4stB9gfHxa7Ay+3cDOPAn8zs/bOua34M8bpzrltjayjsQ4C8qnWXeCcS5jZ20Dfast9WON5q4E9m7CdpqhvW31p+f5t7GtuqJZU9gDWVp9hZnvhz+C/U212Of53VessPVnPJqA1uxJLk//qTL0GhXq4tjrnFjfzuVvwXRw1dcWfEddnFr5b4WpgFf4vho+B3ep7UhP9M7nec8zsZWAQcHob11G9C2F7irbmdD9WAFZjXn6Nn4PaVnPUHM7W1Fo2AN1qzKv8vGVutXmHAQudc2+kWomZ3QzcXH+pnOGce72BZerSPfnv+mY+P7IU6plrITDEzMwl/x5NOjbZlpKZ9QC+Dox0zr2anHcsVcfCJ0AZcCrwnzpWU47/c79OzrkyM3sKf4beE/gC3x3Q2DoatR18l0M58O3kY5If0B4PTG3guc2xHt/PXd3RwLJGPj+I/duar/l9fBdedV3x/xkkktvqhO9L/6Ke9fwJmNbAtlY1r0QAjgRWOefWNrhkllGoh2v35J+21SWcc5VnH53NrF+N9rhzbhkwEf/B131m9md8P+0Q/IiAs+vZ5pf4s7GrzGwFsC9wF/4sGefcV2b2e+B2MyvDdxH1APo75yYm17EM/yFVH3y/5ybnXKqRCo/iuxm+BjxeY5l662jsdpxzJWY2EbjDzDYAS4ExQAHwQD37obleASaY2dn4/zyvBvajkaHe3P1bYx2t+ZpfTK63h3NuY3LefPxfJzeZ2WP439Ma4GAzO8Q5V+s/p+Z2vyQ/UD84+WMOfvRRP/zv/vNqi56UrFVqCrtTP1sn/AdfLsW0soH2p6ut45v4A3stvstlDnBuI7Z9Cn4s8Lbkv6dT7UMp/Jvpl/gv3JTjR1/8rtrzD8WP0NiarKlPtZpnVVvO8AHlgKOaUUdjt7M7fhTNWvxZ8DskP2xNtseo54PHevZTqg9K8/Fjozckp99Q+4PSerfVnP3b1Nfcwtf9NjCqxryb8X+lbAMew3fRvAmsD/h9UUjq435ytWXa4Y/3/wr7fZyOk75RKiK7MLPBwO+Bvs65RNj11GRmo4BznHOnhV1LOtI4dRHZhXPuBfxfI73DrqUO24Gfhl1EutKZuohIhOhMXUQkQhTqIiIREvqQxp49e7o+ffqEWkNJSQkdOnQItYZ0oX3hLVy4kEQiQd++Nb+gmZ3S9bgoK4NPPoFEAgoKoHcbfAqQLvti3rx5G5xzvWrODz3U+/Tpw9y5cxtesBXFYjEKCwtDrSFdaF94hYWFxOPx0I/NdJGOx8XmzXD88T7Qv/c9+Mc/ILehr6oFIF32hZktTzVf3S8iknESCbjkEn+WfsQRMHVq2wR6JlCoi0jGufFGeP556NEDZsyAzp3Drih9KNRFJKM89BDcey/k58Mzz8CBB4ZdUXoJNNTN7FEzW2NmW8xskZn9JMj1i0h2mz0bRozwjydOhJNPDreedBT0mfrt+OtzdMZfVGqcmfUPeBsikoWWLoXzz4ft22HMGPjxj8OuKD0FGurOuSLnXFnlj8npoCC3ISLZZ8sWOOss2LgRBg+GO+8Mu6L0FfiQRjN7AH895j3w12Z+LsUyw0ne8aegoIBYLBZ0GU1SXFwceg3pQvvCi8fjJBIJ7YukMI+LRALGjv0GRUU9OOCAEkaNeo833gjvOmNp/x5pjUs/4i/wfyIwFsivb9n+/fu7sL366qthl5A2tC+8gQMHuqOPPjrsMtJGmMfFDTc4B8517+7c4sWhlbFTurxHgLkuRaa2yugX51zC+dtc9QZGtMY2RCT6Jk+Gu++GvDyYPh0OUmdug1p7SGMe6lMXkWZ44w0Ynrwt+/33Qxp8iTMjBBbqZranmf3AzDqaWa6ZnY6/tVpL70ovIllm2TI47zw/0uVnP6sKd2lYkB+UOnxXy5/w/1ksB0Y752YEuA0RibivvoKzz4YNG+C00+Cee8KuKLMEFurO3yx5YFDrE5HsU1EBl18OH30Ehx0GTz7p+9Ol8XSZABFJGzff7K/l0q0bzJwJXbuGXVHmUaiLSFp45BG44w5/tcWnn4ZDDgm7osykUBeR0L31Flx1lX98331wyinh1pPJFOoiEqrly/1Il/JyGDWq6oJd0jwKdREJTXGxH+mybh0MGgQTJoRdUeZTqItIKCoqYOhQ+PBD338+bZpGugRBoS4iobjlFnj2WT/CZeZMP+JFWk6hLiJt7rHHYPx4P9Jl2jQ/Jl2CoVAXkTY1Z07VDS4mTIDvfjfceqJGoS4ibWbFCjjnHCgrg2uu8aNdJFgKdRFpEyUlfqTL2rV+HPof/gBmYVcVPQp1EWl1FRXwwx/C/Plw8MHw1FOQnx92VdGkUBeRVnfrrfDMM9Clix/p0r172BVFl0JdRFrVE0/AbbdBTo5//PWvh11RtCnURaTVvPsuXHGFf/y//wuDB4dbTzZQqItIq1i1Cs49F7Zt8xfr+tnPwq4oOyjURSRwW7f6oYtr1sDAgfDHP2qkS1tRqItIoCoqYNgwmDcPDjwQpk+H3XYLu6rsoVAXkUDddpsfsti5sx/p0qNH2BVlF4W6iATmqaf88MXKkS59+4ZdUfZRqItIIObNgx/9yD++6y4444xw68lWCnURabHVq/0lAEpL4corYcyYsCvKXgp1EWmR0lI/dHH1ajjpJJg4USNdwqRQF5Fmc86fmf/739Cnj0a6pAOFuog027hx/gPRjh39SJdevcKuSBTqItIs06fDr3/tu1oefxyOPDLsigQU6iLSDO+/7y+lC3DnnXDmmeHWI1UU6iLSJGvW+JEuW7f6IYzXXx92RVKdQl1EGm3bNjjvPFi5Er79bXjwQY10STcKdRFpFOf8DaPnzIEDDvA3vdh997CrkpoCC3Uz293MHjKz5Wb2lZnNNzN9p0wkIm6/HaZOhQ4dYMYM2HPPsCuSVII8U88DVgADgS7AWGCamfUJcBsiEoLXX+/Jr37lu1qmToWjjgq7IqlLXlArcs6VALdWmzXLzJYC/YFlQW1HRNrW/PkwfvzhgD9bP/vskAuSerVan7qZFQCHAkWttQ0RaV1r1/oQ37Ytl6FD4ec/D7siaUhgZ+rVmVk+8BjwN+fcpynahwPDAQoKCojFYq1RRqMVFxeHXkO60L7w4vE4iUQiq/dFeXkO//3fR7NiRRcOO+xLLr/8I157rSLsskKX7u+RwEPdzHKAKUA5cG2qZZxzk4BJAAMGDHCFhYVBl9EksViMsGtIF9oXXteuXYnH41m7L5zzY9CLimC//WD8+I857bSTwy4rLaT7eyTQUDczAx4CCoAhzrntQa5fRNrGnXfClCnQvr0f6RKP662cKYLuU58IHA6c5ZwrDXjdItIGZsyAm27yjx99FPr1C7ceaZogx6kfAFwN9AO+MLPi5HRZUNsQkdb14Ydw6aW+++V3v/PfHpXMEuSQxuWAvjAskqHWrfMjXUpKfLBXnq1LZtFlAkSEsjI4/3xYvhyOOw7+8hdd0yVTKdRFspxzcM018Oab0Ls3PPss7LFH2FVJcynURbLcPffA5Mk+yP/xD9h777ArkpZQqItksVmzqr4lOmUKHHtsuPVIyynURbLUggVwySW+++W3v4ULLgi7IgmCQl0kC61fD2edBcXF8IMfwNixYVckQVGoi2SZ8nJ/Vr5sGQwYAA8/rJEuUaJQF8kizsHIkfD667DPPv6DUY10iRaFukgWmTABHnqoaqTLPvuEXZEETaEukiWefx5uuME/njzZd71I9CjURbLAxx/7D0QrKuB//gcuuijsiqS1KNRFIm7DBj/SZcsW+P734de/DrsiaU0KdZEIKy+HCy+Ezz6D/v19t0uO3vWRpl+vSEQ5Bz/9Kbz2mv/q/z/+4W96IdGmUBeJqPvug0mToF07f5GuffcNuyJpCwp1kQh68UUYM8Y/fvhhfzldyQ4KdZGI+fRTuPhiP9Jl7Fh/fRfJHgp1kQjZtMmPdNm82V8K4De/CbsiaWsKdZGI2L7dD1lcvBiOOQb+9jeNdMlG+pWLRMR118Err0BBgR/p0qFD2BVJGBTqIhFw//0wcSLsvrsP9P32C7siCYtCXSTD/etf/iwd/MW6vvWtcOuRcCnURTLYokX+Oi6JBNx0E1x2WdgVSdgU6iIZ6ssv/UiXeBzOPRfGjQu7IkkHCnWRDLR9uz9DX7QIjj7a3zRaI10EFOoiGWnMGHjpJdhzT5gxAzp2DLsiSRcKdZEMM3GiH+2y227+mi777x92RZJOFOoiGeSVV/yVFwH+/Gc4/vhw65H0o1AXyRD/+Y+/NnoiAT//Ofzwh2FXJOlIoS6SAeJxP9KlcsTL+PFhVyTpKtBQN7NrzWyumZWZ2eQg1y2SrXbs8FddXLgQvvENeOwxyM0NuypJV3kBr281MA44Hdgj4HWLZKXrr4f/+z/o1cuPdOnUKeyKJJ0FGurOuWcAzGwA0DvIdYtko0mT4A9/gPx8eOYZ6NMn7Iok3alPXSRNxWIwapR/PGkSnHhiqOVIhgi6+6VRzGw4MBygoKCAWCwWRhk7FRcXh15DutC+8OLxOIlEIrR9sWpVO0aO7M+OHflcdNEK+vRZQpi/Fh0XVdJ9X4QS6s65ScAkgAEDBrjCwsIwytgpFosRdg3pQvvC69q1K/F4PJR9sXkzjBwJW7bA974HU6fuR25uuNfS1XFRJd33hbpfRNJIIuHvKfrJJ3DEETB1qka6SNMEeqZuZnnJdeYCuWbWDtjhnNsR5HZEourGG+H556FHDz/SpXPnsCuSTBP0mfpYoBT4JXB58vHYgLchEkkPPQT33ls10uXAA8OuSDJR0EMabwVuDXKdItlg9mwYMcI/njgRTj453Hokc6lPXSRkS5fC+ef7a6SPGQM//nHYFUkmU6iLhGjLFn8tl40bYfBguPPOsCuSTKdQFwlJIgGXXgpFRXD44fDEE5AXyiBjiRKFukhIfvlL+Oc/oXt3mDkTunQJuyKJAoW6SAgmT4a77/Zn5tOnw0EHhV2RRIVCXaSNvfEGDB/uH99/P6TxlxMlAynURdrQsmVVI11+9rOqcBcJikJdpI189ZUf6bJ+PZx2GtxzT9gVSRQp1EXaQCIBl10GCxbAYYfBk09qpIu0DoW6SBv41a/8CJdu3fy/XbuGXZFElUJdpJU98gjccYe/2uLTT8Mhh4RdkUSZQl2kFb31Flx1lX98331wyinh1iPRp1AXaSXLl8N550F5ub8tXeUFu0Rak0JdpBUUF8PZZ8O6dTBoEEyYEHZFki0U6iIBq6iAoUPhww/h0ENh2jSNdJG2o1AXCdjYsfDss36ES+WIF5G2olAXCdCjj8Ltt/uRLk895c/URdqSQl0kIO+8Az/5iX/8+9/7vnSRtqZQFwnA55/DuedCWZkf5TJqVNgVSbZSqIu0UEkJnHMOrF0Lp57qz9JFwqJQF2mBypEu8+fDwQf7kS75+WFXJdlMoS7SAr/+Nfz97/6uRTNn+rsYiYRJoS7STFOnwu9+50e6TJsGX/962BWJKNRFmmXOHLjySv/43nv99dFF0oFCXaSJVqyoGuly9dVw7bVhVyRSRaEu0gSVI12++MLfW/S++8As7KpEqijURRqpogJ+9CN4/3046CB/bXSNdJF0o1AXaaRbb4Xp06FzZz/SpUePsCsSqU2hLtIITzwBt90GOTn+/qKHHx52RSKpKdRFGvDuu3DFFf7xPffA4MHh1iNSH4W6SD1WrfIjXbZt8xfruu66sCsSqV+goW5m3c3s72ZWYmbLzezSINcv0pYqKoxzzoE1a2DgQLj/fo10kfQX9P1Y7gfKgQKgH/BPM/vAOVcU8HZEWt3nn7dn82Y48EA/0mW33cKuSKRh5pwLZkVmHYAvgSOdc4uS86YAq5xzv6zreZ06dXL9+/cPpIbmisfjdO3aNdQa0oX2hffOO/MpK4Pc3H4ccwx06BB2ReHScVElXfbFa6+9Ns85N6Dm/CDP1A8FdlQGetIHwMCaC5rZcGA4QH5+PvF4PMAymi6RSIReQ7rQvoB4PJ+yMv94//1L2L59O1m+S3RcVJPu+yLIUO8IbKkxbzPQqeaCzrlJwCSAAQMGuLlz5wZYRtPFYjEKCwtDrSFdZPu+ePXVytEtheyzTymffTYn7JLSQrYfF9Wly76wOj7gCTLUi4HONeZ1Br4KcBsirebDD/1Il/Jy2Hdf6NmzLOySRJosyNEvi4A8Mzuk2ryjAX1IKmlv+XJ/hr5lC1x4ob8MgEgmCizUnXMlwDPAb82sg5l9GzgHmBLUNkRawxdfwOmnVw1dnDJFQxclcwX95aORwB7AOuBxYISGM0o6W7sWTjkFFi6Eo46CZ5+Fdu3Crkqk+QIdp+6c2wScG+Q6RVrLunX+RtGffAJHHgkvvQRpMFJNpEV0mQDJSpWBXlQEffvCyy9Dr15hVyXScgp1yTpLl8K3vw0LFvirLb7yCuy5Z9hViQRDoS5Z5cMP4YQTYPFiOOYYPy69oCDsqkSCo1CXrPHaa3DyyX60y3e+A7GYAl2iR6EuWeEvf4Hvfhc2b4YLLoDnnvN3MBKJGoW6RNqOHTB6NFx1FWzfDmPG+DsXadiiRFXQl94VSRvr18Nll8G//uVvEP2nP8GVV4ZdlUjrUqhLJM2eDZdcAqtX+6GKf/+7H/EiEnXqfpFISSRg3Dj/Qejq1XDiifDeewp0yR4KdYmMxYv9tVtuuQUqKuCmm/yQxd69w65MpO2o+0UyXkWFv3/oL34BpaWw114webK/SJdItlGoS0YrKoIRI+D11/3Pl14K990H3buHW5dIWNT9IhmpuBh+/nPo188Heq9eMH06PPaYAl2ym0JdMkpFhb/e+eGHw113+Q9Gr7nGXzr3/PPDrk4kfOp+kYzx8stw443w/vv+52OPhYkT4bjjwq1LJJ3oTF3S3htv+K/4DxrkA33fff0Hoe++q0AXqUln6pKWnPNfIBo3zt+8Avy1Wn7xC/+1//btw61PJF0p1CWtlJf7a7NMmOC/NAQ+zEeP9lO3buHWJ5LuFOqSFjZsgAcf9OPN16zx83r1gpEj4brrFOYijaVQl9Ds2AEvvgh//SvMmOGvogj+fqGjR/uLcelqiiJNo1CXNuUcfPwxPPKIH5pYeVaekwPf+54P81NPBbNw6xTJVAp1aXXOwfz5/stB06fDp59WtR16KFxxBQwd6ke1iEjLKNSlVZSV+W96vvCCv+ztZ59VtXXv7r8odMUVcPzxOisXCZJCXQLhHCxa5G9I8cIL/uqIW7dWte+5J5x3Hlx4ob+SYn5+eLWKRJlCXZolkYAFC/xY8tmz/Vn52rW7LnPUUTB4MAwZ4q9rnpsbTq0i2UShLg1yDpYsgXnzYO5cP82bB199tetye+7pb04xeDCcdhrss0849YpkM4W67KKkJJd33vEjVIqK4IMPfIDH47WX3X9/35Vy8sl+OuQQ9Y+LhE2hnoXKymDZMli61H+AuXhxVYivXHlSyucUFMA3vwkDBkD//n7ae++2rVtEGqZQjxjnYPNmf3/O1ath1SpYvrwqwD/7zM9zLvXz8/Mr6Ns3hyOOgL59/ReBBgzwXSk6CxdJfwr1DJBIwJdf+q/S15zWr/df4Fm1qirIq486SSU313edHHhg1dS3r5+WL5/NqacWtsnrEpHgBRLqZnYtMAz4BvC4c25YEOuNgh07/H0zt26FLVv8tHlz/f9u2eL7sDdu9MG9aVPdZ9apdOjgv8izzz5+2m8/OOggH95f+5r/ua4hhStXBvO6RSQcQZ2prwbGAacDewS0zkarqPDhmUiknnbs8Ff/SzVt3w5z53bnyy/rXqZyubKyqoCu/Leux5X/Vl7PpKW6dYOePVNPe+/tw7syyDt1UleJSLYKJNSdc88AmNkAoHdTnvv++wvp2LEQ56rORtu3v4iOHUeyfftWNmwYsrOtcsrNHYbZMHbs2EBFxYUp1joCuBhYAQxN0X49cBawELg6RftYYBAwHxidon08cALwFnBzivYJQD/gJWAcOTm+yyM3F/LyoG/fB9lrr8PYsmUmixbdQ15eVVteHtx44xQOPng/3n33SZ55ZiJ5ebuG9MMPP03Pnj2ZPHkykydPrrX15557jvbt2/PAAw8wbdq0Wu2xWAyAu+++m1mzZu3SVlpaypw5cwC47bbbePnll3dp79GjB9OnTwfgpptu4u23396lvXfv3jz66KMAjB49mvnz5+/SfuihhzJp0iQAhg8fzqJFi3Zp79evHxMmTADg8ssvZ2WNPx2OP/54br/9dgAuuOACNm7cuEv7qaeeyi233ALAGWecQWlp6S7tZ555JjfccAMAhYWF1HTRRRcxcuRIKioqWLx4ca1lhg0bxrBhw9iwYQMXXlj72BsxYgQXX3wxK1asYOjQ2sfe9ddfz1lnncXChQu5+urax97YsWMZNGgQ8+fPZ/To2sfe+PHjOeGEE3jrrbe4+ebax96ECRPo168fL730EuPGjavV/uCDD3LYYYcxc+ZM7rnnnlrtU6ZMYb/99uPJJ59k4sSJO+fH43G6du3K00+33rG3xx578PzzzwPZfext3bqVIUOG1Gpv6NirFEqfupkNB4b7nzpSUrJre2mp73qoS0VFXesFcOTnJ9htt+3AdrZtc4AjJ8e3mzm6dSulW7fN7NixhdWrdwAV5OQYZpCT4zj44I3stddqiovXsmBBGWYu+VzfftJJyznwwG6sXfsZr75aQk6OS05+/VdeOZ/DDy+mqOgDHn+89ljAUaPmsP/+a3jrrY/48sva7R06vE0isYTNm4soKand/uabb9KlSxc+/fRT4inGGs6ePZt27dqxaNGilO2Vb6wlS5bUas/Nzd3ZvnTp0lrtFRUVO9s///zzWu35+fk721euXFmrffXq1TvbV69eXat95cqVO9vXrl1bq/3zzz/f2b5+/Xq2bNmyS/vSpUt3tm/atImysrJd2pcsWbKzPdW+WbRoEbFYjHg8jnOu1jKffvopsViMzZs3p3x+UVERsViMdevWpWz/6KOP6NSpU8p9B/DBBx+Ql5fH4sWLU7a/9957lJeXs2DBgpTtc+fOJR6P88EHH6RsnzNnDmvWrOGjjz5K2f7222+zZMkSioqKdmlPJBLE4/FWPfZKS0sz4tgrLi5u1WNv27ZtKdsbOvYqmWtKZ20DzGwc0Lspfep9+w5wU6fO3Xkmm2qqPJOta8pp4U35YrFYyv85s5H2hVdYWEg8Hq91tpetdFxUSZd9YWbznHMDas5v8EzdzGLAwDqa33TOndiSwtq3h379WrIGERGp1GCoO+cK26AOEREJQFBDGvOS68oFcs2sHbDDObcjiPWLiEjjtLA3eqexQCnwS+Dy5OOxAa1bREQaKaghjbcCtwaxLhERab6gztRFRCQNKNRFRCJEoS4iEiEKdRGRCFGoi4hEiEJdRCRCFOoiIhGiUBcRiRCFuohIhCjURUQiRKEuIhIhCnURkQhRqIuIRIhCXUQkQhTqIiIRolAXEYkQhbqISIQo1EVEIkShLiISIQp1EZEIUaiLiESIQl1EJEIU6iIiEaJQFxGJEIW6iEiEKNRFRCJEoS4iEiEKdRGRCFGoi4hEiEJdRCRCWhzqZra7mT1kZsvN7Cszm29mZwRRnIiINE0QZ+p5wApgINAFGAtMM7M+AaxbRESaIK+lK3DOlQC3Vps1y8yWAv2BZS1dv4iINF7gfU7IKh8AAANoSURBVOpmVgAcChQFvW4REalfi8/UqzOzfOAx4G/OuU/rWW44MBygoKCAWCwWZBlNVlxcHHoN6UL7wovH4yQSCe2LJB0XVdJ9X5hzrv4FzGL4/vJU3nTOnZhcLgeYCnQGznHObW9MAQMGDHBz585tdMGtIRaLUVhYGGoN6UL7wissLCQejzN//vywS0kLOi6qpMu+MLN5zrkBNec3eKbunCtsxMoNeAgoAIY0NtBFRCRYQXW/TAQOBwY550oDWqeIiDRREOPUDwCuBvoBX5hZcXK6rMXViYhIkwQxpHE5YAHUIiIiLaTLBIiIRIhCXUQkQhoc0tjqBZitB5aHWgT0BDaEXEO60L6oon1RRfuiSrrsiwOcc71qzgw91NOBmc1NNd4zG2lfVNG+qKJ9USXd94W6X0REIkShLiISIQp1b1LYBaQR7Ysq2hdVtC+qpPW+UJ+6iEiE6ExdRCRCFOoiIhGiUE/BzA4xs21m9mjYtYQh2+87a2bdzezvZlaS3AeXhl1TGLL9OKhLuueDQj21+4F/h11EiLL9vrP3A+X4S0lfBkw0syPCLSkU2X4c1CWt80GhXoOZ/QCIAy+HXUtYnHMlzrlbnXPLnHMVzrlZQOV9ZyPNzDoAFwC3OOeKnXNvADOAoeFW1vay+TioSybkg0K9GjPrDPwW+O+wa0knWXbf2UOBHc65RdXmfQBk45n6LrLsOKglU/JBob6r24CHnHMrwy4kXTT2vrMR0hHYUmPeZqBTCLWkjSw8DlLJiHzImlA3s5iZuTqmN8ysHzAIuDfsWltbQ/ui2nI5wBR8//K1oRXctorx99mtrjPwVQi1pIUsPQ52kUn5ENTt7NJeQ/daNbPRQB/gc3/LVToCuWbW1zl3bKsX2IZ039l6LQLyzOwQ59x/kvOOJnu7HLL1OKipkAzJB32jNMnM2rPrGdoN+F/iCOfc+lCKCpGZ/Ql/i8JBzrnisOtpS2b2BOCAn+D3wXPACc65rAv2bD4OqsukfMiaM/WGOOe2AlsrfzazYmBbuv3C2kK1+86W4e87W9l0tXPusdAKazsjgYeBdcBG/Bs3GwM924+DnTIpH3SmLiISIVnzQamISDZQqIuIRIhCXUQkQhTqIiIRolAXEYkQhbqISIQo1EVEIkShLiISIQp1EZEI+X+6Vp4ZJEcDuAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plotting the ELU activation function\n",
    "plt.plot(z, elu(z), \"b-\", linewidth=2) # calling the ELU function we created above\n",
    "plt.plot([-5, 5], [0, 0], 'k-')\n",
    "plt.plot([-5, 5], [-1, -1], 'k--')\n",
    "plt.plot([0, 0], [-2.2, 3.2], 'k-')\n",
    "plt.grid(True)\n",
    "plt.title(r\"ELU activation function ($\\alpha=1$)\", fontsize=14)\n",
    "plt.axis([-5, 5, -2.2, 3.2])\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implementing ELU in TensorFlow is trivial, just specify the activation function when building each layer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<keras.layers.core.Dense at 0x7ff1f6e46f98>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keras.layers.Dense(10, activation=\"elu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SELU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This activation function was proposed in this [great paper](https://arxiv.org/pdf/1706.02515.pdf) by Günter Klambauer, Thomas Unterthiner and Andreas Mayr, published in June 2017. During training, a neural network composed exclusively of a stack of dense layers using the SELU activation function and LeCun initialization will self-normalize: the output of each layer will tend to preserve the same mean and variance during training, which solves the vanishing/exploding gradients problem. As a result, this activation function outperforms the other activation functions very significantly for such neural nets, so you should really try it out. Unfortunately, the self-normalizing property of the SELU activation function is easily broken: you cannot use ℓ<sub>1</sub> or ℓ<sub>2</sub> regularization, regular dropout, max-norm, skip connections or other non-sequential topologies (so recurrent neural networks won't self-normalize). However, in practice it works quite well with sequential CNNs. If you break self-normalization, SELU will not necessarily outperform other activation functions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# complementary error function, 1 - erf(x)\n",
    "# calculates the complementary error\n",
    "from scipy.special import erfc\n",
    "\n",
    "# alpha and scale to self normalize with mean 0 and standard deviation 1\n",
    "# (see equation 14 in the paper):\n",
    "alpha_0_1 = -np.sqrt(2 / np.pi) / (erfc(1/np.sqrt(2)) * np.exp(1/2) - 1)\n",
    "scale_0_1 = (1 - erfc(1 / np.sqrt(2)) * np.sqrt(np.e)) * np.sqrt(2 * np.pi) * (2 * erfc(np.sqrt(2))*np.e**2 + np.pi*erfc(1/np.sqrt(2))**2*np.e - 2*(2+np.pi)*erfc(1/np.sqrt(2))*np.sqrt(np.e)+np.pi+2)**(-1/2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0507009873554805"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scale_0_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# defining the SeLU activation function\n",
    "def selu(z, scale=scale_0_1, alpha=alpha_0_1):\n",
    "    return scale * elu(z, alpha)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAEMCAYAAAA70CbBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXwV1f3/8dcHwo4SAY072LrXUtSo1daaujxcftpaxV37pahEqH5FoW7Fioq7tKgICgVRUBFBrPLT/twa694GjWtlsYK4oCwGSQgkJOf3x7kxl0tCCMzNuXfu+/l4zCPDzGTmc08mnxzOnDnHnHOIiEg8tAkdgIiIREdJXUQkRpTURURiREldRCRGlNRFRGJESV1EJEaU1CXrmdlCMxvWCtcZYWYftMJ12pjZ/Wa23MycmRWl+5rNxDPZzGaHjEE2nZJ6jJjZtmY2NpHk1prZ12b2opkdk3RMSSJRpC7Tko5xZtaviWv0N7OKJvY1+X1R2EhSPQgYG+F1eic+S2HKrjuBI6K6zkacAPwOOAnYAXi9Fa6JmRUlPnfPlF2XAue2Rgyy5fJCByCRmgl0Bs4HFgDb4ZNQj5TjHgCuSdlWlfbo0sQ5t7SVrlMBNPoHLWK7A18551olmTfHObcydAzSAs45LTFYgHzAAUc3c1wJMKaZYxzQr4l9/YGKln5fYv9xwCvAt8AK4P8B+6QcsyPwMLAcWA2UAb9MXNelLP0T37MQGJZYfwSYmXLONsBi4PJNiaOR65Qkto8APkg577WJc68F3gd+nbS/d+L7TwWeT3yej4BjNlJGk1OuvbCpn1vi2NkpP9uxwM3AMuAb/P8u2iQd0z6xf1Ei5v8C/5sUa/IyuYnrdABGA18Da4A3gZ8n7S9KfP9RwFuJz10KHBD69yQXFjW/xEd9LfJXZtYxdDBN6IJPBgfjf/FXAk+bWXsAM+sCvIxPMCcDPwZuSHzvY8AoYC6+SWKHxLZUU4H/Y2bdkrYdkTj+0U2JI7EdfPLfATilic9zKfAH4MpErLOAJ8ysb8pxNwF3Az8B/g1MM7OuGznnDcDniWsf1MRxTTkHWAccBlwMDAHOSNr/IPBb4HJgH/z/6srxf5hOTRzzo8S1L23iGrcnzjkA2B//x+zvZrZDynG3AFcBB+D/SD9sZtbCzyMtFfqvipboFvwv5Qp87ekNfC3tkJRjSoBqGv4I1C+Dk45JS029keO7ALUkannAhcAqoGcTx48gqaactH0hDTX1PHwN8vyk/X8FnmtBHL0Tn6VwY9cHvgD+1Ej5Tk05T3HS/p0S236+kXiGkaihp5x3U2rqb6Qc8zzw18T6HolrH9fEdYsS+3s2dZ1EWVUDv03a3xb4BBiZcp5jk475WWLbzqF/T+K+qKYeI865mfjmi5OAZ/G1tTfNLLX9/DGgb8rycLrjM7MfmtkjZvaJmX2HT75tgF0Th+wPvOecW7a513DOrcN/vnMS1+yA/2M3tQVxbMpn2Rpf1q+l7HoV2Ddl23tJ618mvm63qddqofdS/v1l0rX2B+qAf2zB+X8ItCPpczvnavGViJCfWxL0oDRmnHNr8LWz54EbzOyvwAgzu9M5V504bKVzbsFmXuI7oJOZtXPO1dRvNLP8+nNv5Htn45sVivG13HX4Nub2G/mezTEVeMPMdgIOSZz/iVaMI3Xo0+/LyTnnEi0QLa1Q1QGpTRftGjmuJuXfbjOutbma/NxJ+1SRTDMVcPx9hP/jHVU7+1z8fbN/yvYDkvZvwMx6AHsDNzvnXnDO/QfYivUrFu8AfRrpUlevGv9f/Y1yzv0L3/vnLHyN/W/O91zZ1Djq//g1eS3n3Hf42ufPUnb9HF/mUVuKb+dO9pMWnqMM/7P7ZRP7m/3c+GaWapI+t5m1BQ4lPZ9bWkg19ZhIJKvHgUn4//auAgqBK4AXE0moXmcz2z7lFNXOuRVJ/+7dyAO//zrnPjSz54C/mtnl+F/yPYG7gOnOuc+aCPFbfI+MC81sMb5t+Q58LbneI/gHa38zs6vwtej9gFXOuX/g2857mdkBwGeJ7WubuN7DwAX4du3kB52bEsc3+C6ex5rZQmCNa7xb3x34/w3NB+bg+3IfTsMfuCi9BIw2s1/h/3AWA7vgy2STOOfmmdl0/M/uUuBtYGegt3NuCr5HjMM/aH4aqKr/Y5h0jkozGwfcZmbLgE+By4ACInxXQLZA6EZ9LdEs+G5mN+N7V3yL70Y2H/gz0D3puBI27LrmgFeTjmlsvwNOTOzPxyfxBYnrzANuA7o2E+ORwAf4B7kfAMfiH9L2TzpmZ3ybeHni3O8ARUmfcUbi8zXapTHpPD9IHPM1kLcZcVyA/8NRy6Z1aazG9wI5OWl/bxp/4Npc18/GHpS2A+7F/0FaBlxP4w9Km3uY2gHfe+ULfJfGT4CLk/ZfC3yFb+6ZvJFz1HdpXEvTXRp7NlcWWqJfLFHgIiISA2pTFxGJESV1EZEYUVIXEYkRJXURkRgJ3qWxZ8+ernfv3kFjqKyspEuXLkFjyBQqC2/u3LnU1tay776pL0nmpky4LyorYe5ccA522w26dw8VR/iyAJgzZ84y59y2qduDJ/XevXtTWloaNIaSkhKKioqCxpApVBZeUVER5eXlwe/NTBH6vvjqKzjwQJ/QL70URo8OFkrwsqhnZosa267mFxHJaNXVcNppPrH/4hdwxx2hI8psSuoiktGGDoXXXoOddoLp06FdYyPeyPeU1EUkYz30EIwZA+3bw8yZUFAQOqLMF2lSN7OpZvaVmX1nZvPM7IIozy8iuePtt6G42K/fcw8cckjYeLJF1DX1W/CDA20N/AoYaWYHRnwNEYm5ZcvglFNgzRq44AIYODB0RNkj0qTunPvQNYyaVz8I1A+jvIaIxFttLZx1FixaBAcf7JtfZNNF3qXRzMbipzzrhB9h75lGjhkIDAQoKCigpKQk6jBapKKiIngMmUJl4ZWXl1NbW6uySGjN+2L8+B/wwgu7kp9fzdChc3jjjaZGVw4j039H0jJKY9Kg+UXAbS5phpxUhYWFLnRf4Ezpd5oJVBZefT/1srKy0KFkhNa6L2bOhH79oG1beOEFyMRbMVN+R8xsjnOuMHV7Wnq/OOdqnXOv4sfGHpSOa4hIvHz0EfTv79fvuCMzE3o2SHeXxjzUpi4izVi5En7zG6io8O3pQ4aEjih7RZbUzWw7MzvTzLqaWVszOxY/R+SLUV1DROKnrg5++1uYNw/69IEJE8BSp9iWTRblg1KHb2q5D//HYhEwxDn3VITXEJGYuekmeOopyM+HJ56ADBgrK6tFltSdc0uBI6I6n4jE3zPPwHXX+Zr5I4/AD9VYu8WCj9IoIrlpwQI45xw/8uKNN8Lxx4eOKB409ouItLrKSv/GaHk5/PrXcM01oSOKDyV1EWlVzvlX/99/H/bcEx58ENooE0VGRSkirWr0aJg2Dbp2hSefhG7dQkcUL0rqItJq/vEP+MMf/PqDD8I++4SNJ46U1EWkVSxeDGec4Qfsuuoq36Yu0VNSF5G0W7MGTj0Vli6FY46BkSNDRxRfSuoiklbOwcUXw7//Db17w6OP+gG7JD2U1EUkrSZMgIkToWNH/8Zojx6hI4o3JXURSZs33/S1dIDx42H//cPGkwuU1EUkLZYs8e3oNTVwySVw3nmhI8oNSuoiErmaGjj9dPjySzj8cBg1KnREuUNJXUQiN2wYvPIK7LgjTJ8O7dqFjih3KKmLSKSmToW77/aJfMYM2H770BHlFiV1EYlMWRkMHOjX774bDj00bDy5SEldRCKxYoWfkq6qCgYMgOLi0BHlJiV1EdlitbV+btGFC6GwEO69V1PShaKkLiJb7Npr4bnnoGdPmDnTv2gkYSipi8gWeeIJuOUWPyb69Omw666hI8ptSuoistn+8x/4n//x67ffDr/8Zdh4REldRDbTd9/5B6MVFX5I3csvDx2RgJK6iGyGujpfQ587F/bbzw/YpQejmUFJXURa7NZb/VR0+fkwaxZ06RI6IqmnpC4iLfL3v8Pw4b5m/vDDsPvuoSOSZHmhAxCR7PHf/8LZZ/uJL66/Hk44IXREkko1dRHZJKtX+wej334LJ53ka+uSeZTURaRZzsGFF8J778Eee8CUKb5fumQe/VhEpFkzZ+7EI4/4B6KzZkG3bqEjkqYoqYvIRr38Mowb55+GPvAA/OhHgQOSjVJSF5Emff65n8Gors644go47bTQEUlzlNRFpFFr1/o5Rr/5Bg48cAU33RQ6ItkUkSV1M+tgZhPNbJGZrTKzMjM7Pqrzi0jruuQS+Ne/oFcvuPba/5CnDtBZIcqaeh6wGDgC6AYMB6abWe8IryEirWDCBL907OhHYezWrSZ0SLKJIkvqzrlK59wI59xC51ydc2428ClwYFTXEJH0e+stuPhiv37ffXDAAWHjkZZJ23+ozKwA2BP4sJF9A4GBAAUFBZSUlKQrjE1SUVERPIZMobLwysvLqa2tzbmyWLGiHcXFhVRXd+Dkk7+gV6/5lJTovkiW6WVhzrnoT2rWDngW+MQ5t9GZCgsLC11paWnkMbRESUkJRUVFQWPIFCoLr6ioiPLycsrKykKH0mpqauCYY3wXxp/9DF56Cdq39/t0XzTIlLIwsznOucLU7ZH3fjGzNsAUoBq4OOrzi0h6XHGFT+g77ACPP96Q0CW7RNr8YmYGTAQKgBOcc3q6IpIFHnkERo+Gdu1gxgyf2CU7Rd2mPg7YBzjaOVcV8blFJA3efRcuuMCvjx4Nhx0WNh7ZMlH2U+8FFAN9gSVmVpFYzonqGiISrRUr/MiLVVXQvz8MGhQ6ItlSkdXUnXOLAE1oJZIlamvhnHPg0099t8WxYzUlXRxomACRHDVihJ/FqGdP/4JRp06hI5IoKKmL5KAnn4SRI/2Y6NOm+aEAJB6U1EVyzMcfw29/69dvvRWOOipsPBItJXWRHLJqlX8wumqVH0Z32LDQEUnUlNRFcoRzvofLxx/7iS4mTdKD0ThSUhfJEbfdVj/iop+SrmvX0BFJOiipi+SA556DP/7Rr0+d6iePlnhSUheJuU8/hbPOgro6uO46OPHE0BFJOimpi8TY6tVwyin+zdETT4Q//Sl0RJJuSuoiMeUcFBdDWRnsvjtMmeL7pUu86UcsElNjxvj2886d/YPR/PzQEUlrUFIXiaFXXoHLL/frDzwA++0XNh5pPUrqIjHzxRf+xaJ16/zLRaefHjoiaU1K6iIxsnYt9OsHX38NRx4Jt9wSOiJpbUrqIjFy6aXw5puw665+oK68tE0tL5lKSV0kJiZOhPvvhw4dYOZM2Hbb0BFJCErqIjHw73/D4MF+fdw4KNxgjnnJFUrqIlnum2/8C0bV1X46ut/9LnREEpKSukgWW7cOzjgDPv8cDj3UTxwtuU1JXSSLXXUVlJTA9tvDjBnQvn3oiCQ0JXWRLDVtGowa5Xu4PP447Lhj6IgkEyipi2Sh996D88/363/5C/z852HjkcyhpC6SZb791j8YXb3azzX6+9+HjkgyiZK6SBapq4Nzz4VPPoH994f77tOUdLI+JXWRLHL99fDMM9Cjh5+arlOn0BFJplFSF8kSTz0FN9zgx0R/9FHo3Tt0RJKJlNRFssC8eXDeeX795pvhmGPCxiOZS0ldJMOtWgW/+Q189x2ceipccUXoiCSTKamLZDDnYMAA+Ogj2HdfP+GFHozKxiipi2SwO+7wb4puvbV/MLrVVqEjkkwXaVI3s4vNrNTM1prZ5CjPLZJrnn8err7ar0+ZAnvtFTYeyQ5RD6H/JTASOBZQZyuRzbRwIZx1lu+Xfu218KtfhY5IskWkSd059wSAmRUCO0d5bpFcUVXl3xhdvhxOOAFGjAgdkWSTIJNdmdlAYCBAQUEBJSUlIcL4XkVFRfAYMoXKwisvL6e2trbVy8I5uPXWvXnnne3ZcccqLrpoDv/857pWjaExui8aZHpZBEnqzrnxwHiAwsJCV1RUFCKM75WUlBA6hkyhsvDy8/MpLy9v9bK491547jno3BmefbYTffpkxkhdui8aZHpZqPeLSIZ49VUYMsSvT5wIffqEjUeyk5K6SAb48ks47TQ/k9Hll8OZZ4aOSLJVpM0vZpaXOGdboK2ZdQTWOefCNwqKZKjqap/QlyyBoiK47bbQEUk2i7qmPhyoAq4Czk2sD4/4GiKxctll8PrrsPPO8NhjfiYjkc0VdZfGEcCIKM8pEmeTJ8PYsX5u0SeegO22Cx2RZDu1qYsEUloKF13k18eOhYMOChuPxIOSukgAS5f6F4zWroXi4ob5RkW2lJK6SCtbt873blm8GH76U7jrrtARSZwoqYu0smuugZdegoICPwJjhw6hI5I4UVIXaUXTp/vhdPPy4PHHYaedQkckcaOkLtJKPvjAT3gBMGoUHH542HgknpTURVpBebmfkq6yEs49Fy65JHREEldK6iJpVlfnJ41esAD69oX779eUdJI+SuoiaXbjjTB7NmyzjX/BqHPn0BFJnCmpi6TR7Nl+kgszePRR2G230BFJ3Cmpi6TJ/Pm+/Rzgppvg2GPDxiO5QUldJA0qKvyD0ZUr/derrgodkeQKJXWRiDnnX/v/8EPYe28/aJcejEprUVIXidioUf4lo622glmzYOutQ0ckuURJXSRCL74IV17p1x96yNfURVqTkrpIRBYtgjPO8P3S//hHOPnk0BFJLlJSF4lAVRWceiosXw7HHQfXXx86IslVSuoiW8g5GDwY5syBH/wAHn4Y2rYNHZXkKiV1kS10332+h0unTv6N0e7dQ0ckuUxJXWQLvP46XHqpX//rX+EnPwkbj4iSushm+uor6NcPampgyBA4++zQEYkoqYtslupqOO00n9iPOAJuvz10RCKekrrIZhg6FF57zc9c9Nhj0K5d6IhEPCV1kRZ66CEYMwbat4eZM/1coyKZQkldpAXefhuKi/36mDFwyCFh4xFJpaQusomWLYNTToE1a+DCC/0ikmmU1EU2wbp1cNZZfiiAgw+Ge+4JHZFI45TURTbB8OHwwguw3Xa+Hb1Dh9ARiTROSV2kGTNmwG23+Vf/p0+HnXcOHZFI05TURTbio4+gf3+/fuedvk+6SCaLNKmbWXczm2VmlWa2yMz0jp1krdpa4+STobLSvy1aPxyASCbLi/h89wLVQAHQF/i/Zvauc+7DiK8jknaLF3dm5Uro0wcmTNCUdJIdzDkXzYnMugDfAvs55+Yltk0BvnDONTnt7lZbbeUOPPDASGLYXOXl5eTn5weNIVOoLLzS0jIqK6Ft274UFkLHjqEjCkv3RYNMKYuXX355jnOuMHV7lDX1PYF19Qk94V1gg1ZIMxsIDARo164d5eXlEYbRcrW1tcFjyBQqC6+qygHGttuuYc2aNaxZEzqisHRfNMj0sogyqXcFvkvZthLYKvVA59x4YDxAYWGhKy0tjTCMlispKaGoqChoDJlCZQGPPw6nn15EXl4dCxb8ky5dQkcUnu6LBplSFtZEe2CUD0orgNR507cGVkV4DZG0qqnx84sCbL/9WiV0yTpRJvV5QJ6Z7ZG07SeAHpJK1pg0CebP97MYde++NnQ4Ii0WWVJ3zlUCTwA3mFkXM/sZ8GtgSlTXEEmnysqGCaN32029XSQ7Rf3y0WCgE/AN8CgwSN0ZJVvcdZef9KKwELbdNnQ0Ipsn0qTunFvhnDvZOdfFOberc+6RKM8vki7Ll/uhAABuvTVsLCJbQsMEiAC33ALffQfHHANHHRU6GpHNp6QuOW/hQj/hBaiWLtlPSV1y3lVXwdq1fnyXAw4IHY3IllFSl5z2xht+4uiOHX0TjEi2U1KXnFVXB5dd5teHDYNddw0bj0gUlNQlZz32GLz1Fmy/PVx5ZehoRKKhpC45qarKt6UDjBwJXbuGjUckKkrqkpNuvRU++8yPlV4/s5FIHCipS875+OOGrov33OPnHhWJCyV1ySnOwUUXQXU1nH8+/OIXoSMSiZaSuuSUhx6Cl1+Gnj0bhgUQiRMldckZy5bB0KF+/c9/hh49wsYjkg5K6pIzhg71A3cdeSSce27oaETSQ0ldcsKsWb7ppWNHGDdOY6VLfCmpS+x9/TUMHOjXb78d9twzbDwi6aSkLrHmHFx4oW9PP+oo+P3vQ0ckkl5K6hJrDzwATz8N3br59Ta64yXmdItLbM2bB5de6tfHjIFddgkbj0hrUFKXWFq9Gvr1g4oKOP10OOec0BGJtA4ldYmliy+G99+HPfaACRPU20Vyh5K6xM4DD/ilY0eYMQO23jp0RCKtR0ldYuXdd2HwYL8+dqwfhVEklyipS2wsWQInnQRr1sDvfucXkVyjpC6xUFUFJ58MixfDoYf6WrpILlJSl6znnK+Vv/UW9OoFTz7p29NFcpGSumS9667z841utRXMng3bbRc6IpFwlNQlq91zD9x4o39TdNo02G+/0BGJhKWkLllryhT43//16xMmwAknhI1HJBMoqUtWeuqpht4td94JAwaEjUckUyipS9Z59ln/6n9tLfzxjw2zGYlIREndzC42s1IzW2tmk6M4p0hj/vY3+PWvYe1auOQS354uIg2iqql/CYwEJkV0PpENTJ/uB+mqqYHLLoO77tKYLiKpIknqzrknnHNPAsujOJ9IqkmT4KyzYN06uPpqGDVKCV2kMXkhLmpmA4GBAAUFBZSUlIQI43sVFRXBY8gUmVYWzsGDD/bmwQd7A9C//6ccc8wiXn45vdctLy+ntrY2o8oipEy7L0LK9LIIktSdc+OB8QCFhYWuqKgoRBjfKykpIXQMmSKTyqKmBoqL4cEHfT/0MWNg0KDdgN3Sfu38/HzKy8szpixCy6T7IrRML4tmm1/MrMTMXBPLq60RpOSe5ct9v/MHHoDOnf2r/4MGhY5KJPM1W1N3zhW1Qhwi33vnHTjlFFi40L/yP3s2HHRQ6KhEskNUXRrzzKwj0BZoa2YdzSxI045kt6lT4bDDfEI/6CAoLVVCF2mJqLo0DgeqgKuAcxPrwyM6t+SAigo4/3w47zw/HvqAAfDPf2qyaJGWiqQ27ZwbAYyI4lySe+bM8d0V58+HDh18//OBA9VlUWRzaJgACaamBm66yU9qMX++H2GxtNT3eFFCF9k8aveWIN5+2ze3lJX5f19yCdx2G3TqFDYukWynmrq0qooKuPJKOPhgn9B32w2efx7uvlsJXSQKSurSKpyDRx6BvfaC22+Hujo/fsv778PRR4eOTiQ+1Pwiaffmm/CHP8CriVfVDjrIvx168MFh4xKJI9XUJW0+/BB+8xv/IPTVV/2LRJMm+SSvhC6SHqqpS+Tee88/9Jw2zTezdO4MQ4bAFVdAt26hoxOJNyV1icwrr8Ctt8Izz/h/5+XBRRfB8OGwww5hYxPJFUrqskVqauDpp+HPf4bXXvPbOnWCCy+Eyy+HXr3CxieSa5TUZbMsWgQTJsDEibBkid+2zTa+v/kll0DPnmHjE8lVSuqyyaqqfNPKpEl+8mfn/PZ99vHNLAMGQNeuYWMUyXVK6rJRNTXw4ovw6KMwaxasWuW3t2/v5wstLobDD9dr/SKZQkldNlBZ6RP500/7ySmWLWvYV1gIZ5/tR1NUE4tI5lFSFwA+/dQ3qTz00I8pK4O1axv27bOPH0XxzDNhjz3CxSgizVNSz1FffAH/+IdfXnrJT0rh9cAMDjkETjrJLz/+sZpXRLKFknoOWLvWD5711lsNyyefrH/MNtvAkUfC7rt/zGWX7U1BQZhYRWTLKKnHTFUVfPSRHyjrnXd8An/nHaiuXv+4rl3hF7/wifzII6FPH2jbFkpKllBQsHeY4EVkiympZ6nVq31te948+OADn8Tffx8WLPCv5qfaZx/fpPLTn/qv++3n3/gUkXjRr3WGcg6WLoXPPoPFi32yXrDAzxA0fz58/nnj39e2Ley7r28H79PHD5x10EEac0UkVyipB1BVBV9/3bAsWeITd30Cr1+Se6CkysvzE0zssQf86Ec+gf/4x7D33n6eTxHJTUrqW8A5/zLOt9/CihXrf01eX7oUvvmmIYnXv8DTnG22gV13hV12aUjg9UuvXmo+EZENxT4t1NTAmjV+qapa/2v9emlpT776yrdTV1T4pFv/NXk99evKlVBb2/KY2rWDggI/vnhBgV/qk3f911120Sv3ItJywZP6V1/Bn/7kk++WLtXVfklO3JuWdPfb7Pi7dIHu3X2tun5J/nf37tCjR0PyLiiA/Hz1+xaR9Aie1L/8ci433liUsvV0YDCwGjihke/qn1iWAf0a2T8IOANYDJxHmzastxQUDGW77U6irm4u//1vMbW1NXTo0I42bXyTxuGHD6dPn6NZubKMJ58cQtu2/gFkXp7/euWVN1NUdBgffvg61113zXpX/vZbuO660fTt25cXXniBkSNHbhDd/fffz1577cXTTz/NqFGjNtg/ZcoUdtllFx577DHGjRu3wf4ZM2bQs2dPJk+ezOTJkzfY/8wzz9C5c2fGjh3L9OnTN9hfUlICwJ133sns2bPX21dVVcVbb70FwI033siLL7643v4ePXowc+ZMAK6++mreeOON9fbvvPPOTJ06FYAhQ4ZQVla23v4999yT8ePHAzBw4EDmzZu33v6+ffsyevRoAM4991w+T3kifOihh3LLLbcAcOqpp7J8+fL19h911FFce+21ABx//PFUVVWtt//EE09k2LBhABQVFZHq9NNPZ/DgwdTV1bFgwYINjunfvz/9+/dn2bJl9Ou34b03aNAgzjjjDBYvXsx55523wf6hQ4dy0kknMXfuXIqLizfYP3z4cI4++mjKysoYMmTIBvtvvvlmDjvsMF5//XWuueaaDfaPHp2ee6+8vJz8/Py03nudOnXi2WefBXL73lu9ejUnnLBh3mvu3qsXPKm3b+8nUGjTxtdezeDAA+Goo3zXvLvu8tuS9x93HJx4oh+jZPjw9fe3aeNHCzzzTN+WPWDAhtccOtS/KTl3rh+Qqry8kvz8/O/3n3++nwy5rMxPvZZqxx39uCft2qWxYERENoO5+vFTAyksLHSlpaVBYygpKWn0L2cuUll4RUVFlJeXb1Dby1W6LxpkSlmY2RznXGHqdk08LSISI5+nG4EAAAOOSURBVErqIiIxoqQuIhIjSuoiIjGipC4iEiNbnNTNrIOZTTSzRWa2yszKzOz4KIITEZGWiaKmnod/y+cIoBswHJhuZr0jOLeIiLTAFr985JyrBEYkbZptZp8CBwILt/T8IiKy6SJ/o9TMCoA9gQ83csxAYCBAQUHB968Oh1JRURE8hkyhsvDKy8upra1VWSTovmiQ6WUR6RulZtYOeBb4xDm34cAWjdAbpZlFZeHpjdL16b5okCllsdlvlJpZiZm5JpZXk45rA0wBqoGLI41eREQ2SbPNL865ouaOMTMDJgIFwAnOuZotD01ERFoqqjb1ccA+wNHOuarmDhYRkfSIop96L6AY6AssMbOKxHLOFkcnIiItEkWXxkWA5vEREckAGiZARCRGgk+SYWZLgUVBg4Ce+LnxRGWRTGXRQGXRIFPKopdzbtvUjcGTeiYws9LG+nvmIpVFA5VFA5VFg0wvCzW/iIjEiJK6iEiMKKl740MHkEFUFg1UFg1UFg0yuizUpi4iEiOqqYuIxIiSuohIjCipi4jEiJJ6I8xsDzNbY2ZTQ8cSQq7PO2tm3c1slplVJsrg7NAxhZDr90FTMj0/KKk37l7g36GDCCjX5529Fz8vQAFwDjDOzH4UNqQgcv0+aEpG5wcl9RRmdiZQDrwYOpZQnHOVzrkRzrmFzrk659xsoH7e2Vgzsy7AqcC1zrkK59yrwFPAeWEja325fB80JRvyg5J6EjPbGrgBuDx0LJlkU+adjZE9gXXOuXlJ294FcrGmvp4cuw82kC35QUl9fTcCE51zn4cOJFMk5p19GHjQOfdx6HhaQVfgu5RtK4GtAsSSMXLwPmhMVuSHnEnqzc21amZ9gaOBv4SONd007+xGVQBbp2zbGlgVIJaMkKP3wXqyKT9ENZ1dxmturlUzGwL0Bj7zU67SFWhrZvs65w5Ie4CtSPPObtQ8IM/M9nDOzU9s+wm52+SQq/dBqiKyJD9omIAEM+vM+jW0Yfgf4iDn3NIgQQVkZvfhpyg82jlXETqe1mRm0wAHXIAvg2eAw5xzOZfYc/k+SJZN+SFnaurNcc6tBlbX/9vMKoA1mfYDaw1J886uxc87W7+r2Dn3cLDAWs9gYBLwDbAc/4ubiwk91++D72VTflBNXUQkRnLmQamISC5QUhcRiREldRGRGFFSFxGJESV1EZEYUVIXEYkRJXURkRhRUhcRiZH/D3PURwprhitlAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plotting the SeLU activation function\n",
    "plt.plot(z, selu(z), \"b-\", linewidth=2) # calling the SeLU function created\n",
    "plt.plot([-5, 5], [0, 0], 'k-')\n",
    "plt.plot([-5, 5], [-1.758, -1.758], 'k--')\n",
    "plt.plot([0, 0], [-2.2, 3.2], 'k-')\n",
    "plt.grid(True)\n",
    "plt.title(\"SELU activation function\", fontsize=14)\n",
    "plt.axis([-5, 5, -2.2, 3.2])\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "By default, the SELU hyperparameters (`scale` and `alpha`) are tuned in such a way that the mean output of each neuron remains close to 0, and the standard deviation remains close to 1 (assuming the inputs are standardized with mean 0 and standard deviation 1 too). Using this activation function, even a 1,000 layer deep neural network preserves roughly mean 0 and standard deviation 1 across all layers, avoiding the exploding/vanishing gradients problem:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "Z = np.random.normal(size=(500, 100)) # standardized inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Layer 0: mean -0.00, std deviation 1.00\n",
      "Layer 100: mean 0.02, std deviation 0.96\n",
      "Layer 200: mean 0.01, std deviation 0.90\n",
      "Layer 300: mean -0.02, std deviation 0.92\n",
      "Layer 400: mean 0.05, std deviation 0.89\n",
      "Layer 500: mean 0.01, std deviation 0.93\n",
      "Layer 600: mean 0.02, std deviation 0.92\n",
      "Layer 700: mean -0.02, std deviation 0.90\n",
      "Layer 800: mean 0.05, std deviation 0.83\n",
      "Layer 900: mean 0.02, std deviation 1.00\n"
     ]
    }
   ],
   "source": [
    "# plotting the mean, and the standard deviation for each layer\n",
    "np.random.seed(42)\n",
    "Z = np.random.normal(size=(500, 100)) # standardized inputs\n",
    "for layer in range(1000):\n",
    "    W = np.random.normal(size=(100, 100), scale=np.sqrt(1 / 100)) # LeCun initialization\n",
    "    Z = selu(np.dot(Z, W))\n",
    "    means = np.mean(Z, axis=0).mean()\n",
    "    stds = np.std(Z, axis=0).mean()\n",
    "    if layer % 100 == 0:\n",
    "        print(\"Layer {}: mean {:.2f}, std deviation {:.2f}\".format(layer, means, stds))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using SELU is easy:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<keras.layers.core.Dense at 0x7ff1f9b1fef0>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keras.layers.Dense(10, activation=\"selu\",\n",
    "                   kernel_initializer=\"lecun_normal\") # using SeLU with LeCun initialization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's create a neural net for Fashion MNIST with 100 hidden layers, using the SELU activation function:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# setting seed value\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating the model using Keras Sequential API\n",
    "model = keras.models.Sequential()\n",
    "model.add(keras.layers.Flatten(input_shape=[28, 28]))\n",
    "model.add(keras.layers.Dense(300, activation=\"selu\",\n",
    "                             kernel_initializer=\"lecun_normal\"))\n",
    "for layer in range(99):\n",
    "    model.add(keras.layers.Dense(100, activation=\"selu\",\n",
    "                                 kernel_initializer=\"lecun_normal\"))\n",
    "model.add(keras.layers.Dense(10, activation=\"softmax\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/anaconda/envs/tensorflow2/lib/python3.6/site-packages/keras/optimizer_v2/optimizer_v2.py:356: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  \"The `lr` argument is deprecated, use `learning_rate` instead.\")\n"
     ]
    }
   ],
   "source": [
    "# compiling the model with sparse categorical crossentropy since\n",
    "# the predicted labels are integers\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
    "              optimizer=keras.optimizers.SGD(lr=1e-3),\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's train it. Do not forget to scale the inputs to mean 0 and standard deviation 1:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'X_train' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-25e45a416b49>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# scaling the dataset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mpixel_means\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mX_train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeepdims\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mpixel_stds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mX_train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeepdims\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mX_train_scaled\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mpixel_means\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mpixel_stds\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mX_valid_scaled\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mX_valid\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mpixel_means\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mpixel_stds\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'X_train' is not defined"
     ]
    }
   ],
   "source": [
    "# scaling the dataset\n",
    "pixel_means = X_train.mean(axis=0, keepdims=True)\n",
    "pixel_stds = X_train.std(axis=0, keepdims=True)\n",
    "X_train_scaled = (X_train - pixel_means) / pixel_stds\n",
    "X_valid_scaled = (X_valid - pixel_means) / pixel_stds\n",
    "X_test_scaled = (X_test - pixel_means) / pixel_stds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "510/782 [==================>...........] - ETA: 4s - loss: 1.5513 - accuracy: 0.4050"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-35-4e7400086f82>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# training the model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m history = model.fit(X_train_scaled, y_train, epochs=6,\n\u001b[0;32m----> 3\u001b[0;31m                     validation_data=(X_valid_scaled, y_valid))\n\u001b[0m",
      "\u001b[0;32m/usr/local/anaconda/envs/tensorflow2/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1182\u001b[0m                 _r=1):\n\u001b[1;32m   1183\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1184\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1185\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1186\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/anaconda/envs/tensorflow2/lib/python3.6/site-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    883\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    884\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 885\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    886\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    887\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/anaconda/envs/tensorflow2/lib/python3.6/site-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    915\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    916\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 917\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    918\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    919\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/anaconda/envs/tensorflow2/lib/python3.6/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   3038\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[1;32m   3039\u001b[0m     return graph_function._call_flat(\n\u001b[0;32m-> 3040\u001b[0;31m         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[0m\u001b[1;32m   3041\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3042\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/anaconda/envs/tensorflow2/lib/python3.6/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1962\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1963\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1964\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1965\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1966\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/anaconda/envs/tensorflow2/lib/python3.6/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    594\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    595\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 596\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    597\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    598\u001b[0m           outputs = execute.execute_with_cancellation(\n",
      "\u001b[0;32m/usr/local/anaconda/envs/tensorflow2/lib/python3.6/site-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 60\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     61\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# training the model\n",
    "history = model.fit(X_train_scaled, y_train, epochs=6,\n",
    "                    validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training the model\n",
    "history1 = model.fit(X_train_scaled, y_train, epochs=6,\n",
    "                    validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training the model\n",
    "history1 = model.fit(X_train_scaled, y_train, epochs=6,\n",
    "                    validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now look at what happens if we try to use the ReLU activation function instead:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# trying the same but this time using ReLU activation function\n",
    "model = keras.models.Sequential()\n",
    "model.add(keras.layers.Flatten(input_shape=[28, 28]))\n",
    "model.add(keras.layers.Dense(300, activation=\"relu\", kernel_initializer=\"he_normal\"))\n",
    "for layer in range(99):\n",
    "    model.add(keras.layers.Dense(100, activation=\"relu\", kernel_initializer=\"he_normal\"))\n",
    "model.add(keras.layers.Dense(10, activation=\"softmax\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compiling the model with sparse categorical crossentropy since\n",
    "# the predicted labels are integers\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
    "              optimizer=keras.optimizers.SGD(lr=1e-3),\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training the model\n",
    "history = model.fit(X_train_scaled, y_train, epochs=10,\n",
    "                    validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Not great at all, we suffered from the vanishing/exploding gradients problem."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Batch Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating a model\n",
    "# applying BN after every hidden layer and as the first\n",
    "# layer in the model (after flattening the input images)\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    \n",
    "    keras.layers.Dense(300, activation=\"relu\"),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    \n",
    "    keras.layers.Dense(100, activation=\"relu\"),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    \n",
    "    keras.layers.Dense(10, activation=\"softmax\")\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2368"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(784+300+100)*2 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1010"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "101*10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_3\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_3 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization (BatchNo (None, 784)               3136      \n",
      "_________________________________________________________________\n",
      "dense_111 (Dense)            (None, 300)               235500    \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 300)               1200      \n",
      "_________________________________________________________________\n",
      "dense_112 (Dense)            (None, 100)               30100     \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, 100)               400       \n",
      "_________________________________________________________________\n",
      "dense_113 (Dense)            (None, 10)                1010      \n",
      "=================================================================\n",
      "Total params: 271,346\n",
      "Trainable params: 268,978\n",
      "Non-trainable params: 2,368\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "268978.0"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "785*300 + 301*100 + 101 * 10 + (784*4 + 300*4 + 100*4)/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2368.0"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(784*4 + 300*4 + 100*4)/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3136"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "784*4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_5 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_3 (Batch (None, 784)               3136      \n",
      "_________________________________________________________________\n",
      "dense_409 (Dense)            (None, 300)               235500    \n",
      "_________________________________________________________________\n",
      "batch_normalization_4 (Batch (None, 300)               1200      \n",
      "_________________________________________________________________\n",
      "dense_410 (Dense)            (None, 100)               30100     \n",
      "_________________________________________________________________\n",
      "batch_normalization_5 (Batch (None, 100)               400       \n",
      "_________________________________________________________________\n",
      "dense_411 (Dense)            (None, 10)                1010      \n",
      "=================================================================\n",
      "Total params: 271,346\n",
      "Trainable params: 268,978\n",
      "Non-trainable params: 2,368\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary() # generating model summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1184"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights_and_bias = 235500 + "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "#moving_means, #moving_variance, #gammas, #betas = 1184"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Nontrainable: #moving_means + #moving_variance = 2368"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Trainable: 2368 + weights + biases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('batch_normalization/gamma:0', True),\n",
       " ('batch_normalization/beta:0', True),\n",
       " ('batch_normalization/moving_mean:0', False),\n",
       " ('batch_normalization/moving_variance:0', False)]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# checking model layer details\n",
    "bn1 = model.layers[1]\n",
    "[(var.name, var.trainable) for var in bn1.variables]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor: shape=(), dtype=float32, numpy=0.9999898>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=0.00623951>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=9.18706e-06>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=4.122953e-07>)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i=0 \n",
    "bn1.gamma[i] , bn1.beta[i], bn1.moving_mean[i], bn1.moving_variance[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.9999898, 0.00623951, 9.18706e-06, 4.122953e-07)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w = bn1.get_weights()\n",
    "w[0][0], w[1][0], w[2][0], w[3][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((784,), (784,), (784,), (784,))"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bn1.get_weights()[0][0].shape,bn1.get_weights()[1][0].shape,bn1.get_weights()[2][0].shape, bn1.get_weights()[3].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When you create a BN layer in Keras, it also creates two operations that will be called by Keras at each iteration during training. These operations will update the moving averages. Since we are using the TensorFlow backend, these operations are\n",
    "TensorFlow operations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bn1.updates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/anaconda/envs/tensorflow2/lib/python3.6/site-packages/keras/optimizer_v2/optimizer_v2.py:356: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  \"The `lr` argument is deprecated, use `learning_rate` instead.\")\n"
     ]
    }
   ],
   "source": [
    "# compiling the model\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
    "              optimizer=keras.optimizers.SGD(lr=1e-3),\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "782/782 [==============================] - 4s 4ms/step - loss: 1.0855 - accuracy: 0.6489 - val_loss: 0.6913 - val_accuracy: 0.7700\n",
      "Epoch 2/5\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.6950 - accuracy: 0.7675 - val_loss: 0.5743 - val_accuracy: 0.8080\n",
      "Epoch 3/5\n",
      "782/782 [==============================] - 3s 3ms/step - loss: 0.6108 - accuracy: 0.7904 - val_loss: 0.5234 - val_accuracy: 0.8250\n",
      "Epoch 4/5\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.5622 - accuracy: 0.8055 - val_loss: 0.4890 - val_accuracy: 0.8390\n",
      "Epoch 5/5\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.5330 - accuracy: 0.8140 - val_loss: 0.4663 - val_accuracy: 0.8468\n"
     ]
    }
   ],
   "source": [
    "# training the model\n",
    "history = model.fit(X_train, y_train, epochs=5,\n",
    "                    validation_data=(X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.5044 - accuracy: 0.8234 - val_loss: 0.4512 - val_accuracy: 0.8504\n",
      "Epoch 2/10\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.4835 - accuracy: 0.8323 - val_loss: 0.4389 - val_accuracy: 0.8564\n",
      "Epoch 3/10\n",
      "782/782 [==============================] - 3s 3ms/step - loss: 0.4706 - accuracy: 0.8364 - val_loss: 0.4300 - val_accuracy: 0.8582\n",
      "Epoch 4/10\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.4544 - accuracy: 0.8394 - val_loss: 0.4206 - val_accuracy: 0.8628\n",
      "Epoch 5/10\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.4454 - accuracy: 0.8443 - val_loss: 0.4125 - val_accuracy: 0.8636\n",
      "Epoch 6/10\n",
      "782/782 [==============================] - 3s 3ms/step - loss: 0.4367 - accuracy: 0.8450 - val_loss: 0.4071 - val_accuracy: 0.8658\n",
      "Epoch 7/10\n",
      "782/782 [==============================] - 3s 3ms/step - loss: 0.4298 - accuracy: 0.8485 - val_loss: 0.4019 - val_accuracy: 0.8670\n",
      "Epoch 8/10\n",
      "782/782 [==============================] - 3s 3ms/step - loss: 0.4192 - accuracy: 0.8518 - val_loss: 0.3971 - val_accuracy: 0.8676\n",
      "Epoch 9/10\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.4110 - accuracy: 0.8548 - val_loss: 0.3936 - val_accuracy: 0.8684\n",
      "Epoch 10/10\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.4047 - accuracy: 0.8574 - val_loss: 0.3888 - val_accuracy: 0.8670\n"
     ]
    }
   ],
   "source": [
    "# training the model\n",
    "history1 = model.fit(X_train, y_train, epochs=10,\n",
    "                    validation_data=(X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "l = model.layers[2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sometimes applying BN before the activation function works better (there's a debate on this topic). Moreover, the layer before a `BatchNormalization` layer does not need to have bias terms, since the `BatchNormalization` layer some as well, it would be a waste of parameters, so you can set `use_bias=False` when creating those layers:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# defining the model\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    \n",
    "    keras.layers.Dense(300, use_bias=False),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    keras.layers.Activation(\"relu\"),\n",
    "    \n",
    "    keras.layers.Dense(100, use_bias=False),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    keras.layers.Activation(\"relu\"),\n",
    "    \n",
    "    keras.layers.Dense(10, activation=\"softmax\")\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_4\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_4 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_3 (Batch (None, 784)               3136      \n",
      "_________________________________________________________________\n",
      "dense_114 (Dense)            (None, 300)               235200    \n",
      "_________________________________________________________________\n",
      "batch_normalization_4 (Batch (None, 300)               1200      \n",
      "_________________________________________________________________\n",
      "activation (Activation)      (None, 300)               0         \n",
      "_________________________________________________________________\n",
      "dense_115 (Dense)            (None, 100)               30000     \n",
      "_________________________________________________________________\n",
      "batch_normalization_5 (Batch (None, 100)               400       \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "dense_116 (Dense)            (None, 10)                1010      \n",
      "=================================================================\n",
      "Total params: 270,946\n",
      "Trainable params: 268,578\n",
      "Non-trainable params: 2,368\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compiling the model\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
    "              optimizer=keras.optimizers.SGD(lr=1e-3),\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "782/782 [==============================] - 4s 4ms/step - loss: 1.2970 - accuracy: 0.5952 - val_loss: 0.8480 - val_accuracy: 0.7508\n",
      "Epoch 2/5\n",
      "782/782 [==============================] - 3s 3ms/step - loss: 0.8324 - accuracy: 0.7486 - val_loss: 0.6866 - val_accuracy: 0.7902\n",
      "Epoch 3/5\n",
      "782/782 [==============================] - 3s 3ms/step - loss: 0.7182 - accuracy: 0.7736 - val_loss: 0.6105 - val_accuracy: 0.8066\n",
      "Epoch 4/5\n",
      "782/782 [==============================] - 3s 3ms/step - loss: 0.6554 - accuracy: 0.7899 - val_loss: 0.5616 - val_accuracy: 0.8172\n",
      "Epoch 5/5\n",
      "782/782 [==============================] - 3s 3ms/step - loss: 0.6122 - accuracy: 0.8012 - val_loss: 0.5295 - val_accuracy: 0.8230\n"
     ]
    }
   ],
   "source": [
    "# training the model\n",
    "history = model.fit(X_train, y_train, epochs=5,\n",
    "                    validation_data=(X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# New experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# defining the model\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    \n",
    "    keras.layers.Dense(100, use_bias=False),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    keras.layers.Activation(\"relu\"),\n",
    "    \n",
    "    keras.layers.Dense(100, use_bias=False),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    keras.layers.Activation(\"relu\"),\n",
    "\n",
    "    keras.layers.Dense(100, use_bias=False),\n",
    "    keras.layers.BatchNormalization(),\n",
    "    keras.layers.Activation(\"relu\"),\n",
    "\n",
    "    keras.layers.Dense(10, activation=\"softmax\")\n",
    "])\n",
    "# compiling the model\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
    "              optimizer=keras.optimizers.SGD(lr=1e-3),\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.3832 - accuracy: 0.8664 - val_loss: 0.3717 - val_accuracy: 0.8646\n",
      "Epoch 2/30\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.3788 - accuracy: 0.8688 - val_loss: 0.3702 - val_accuracy: 0.8672\n",
      "Epoch 3/30\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.3747 - accuracy: 0.8670 - val_loss: 0.3669 - val_accuracy: 0.8664\n",
      "Epoch 4/30\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.3676 - accuracy: 0.8720 - val_loss: 0.3650 - val_accuracy: 0.8682\n",
      "Epoch 5/30\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.3658 - accuracy: 0.8698 - val_loss: 0.3620 - val_accuracy: 0.8672\n",
      "Epoch 6/30\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.3599 - accuracy: 0.8736 - val_loss: 0.3602 - val_accuracy: 0.8688\n",
      "Epoch 7/30\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.3580 - accuracy: 0.8723 - val_loss: 0.3589 - val_accuracy: 0.8702\n",
      "Epoch 8/30\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.3567 - accuracy: 0.8730 - val_loss: 0.3564 - val_accuracy: 0.8704\n",
      "Epoch 9/30\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.3464 - accuracy: 0.8776 - val_loss: 0.3567 - val_accuracy: 0.8708\n",
      "Epoch 10/30\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.3466 - accuracy: 0.8794 - val_loss: 0.3537 - val_accuracy: 0.8692\n",
      "Epoch 11/30\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.3489 - accuracy: 0.8759 - val_loss: 0.3523 - val_accuracy: 0.8714\n",
      "Epoch 12/30\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.3370 - accuracy: 0.8816 - val_loss: 0.3529 - val_accuracy: 0.8732\n",
      "Epoch 13/30\n",
      "313/782 [===========>..................] - ETA: 1s - loss: 0.3405 - accuracy: 0.8802"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-66-f43ba95cdce5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# training the model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m history = model.fit(X_train, y_train, epochs=30,\n\u001b[0;32m----> 3\u001b[0;31m                     validation_data=(X_valid, y_valid))\n\u001b[0m",
      "\u001b[0;32m/usr/local/anaconda/envs/tensorflow2/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1182\u001b[0m                 _r=1):\n\u001b[1;32m   1183\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1184\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1185\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1186\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/anaconda/envs/tensorflow2/lib/python3.6/site-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    883\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    884\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 885\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    886\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    887\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/anaconda/envs/tensorflow2/lib/python3.6/site-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    915\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    916\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 917\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    918\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    919\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/anaconda/envs/tensorflow2/lib/python3.6/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   3038\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[1;32m   3039\u001b[0m     return graph_function._call_flat(\n\u001b[0;32m-> 3040\u001b[0;31m         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[0m\u001b[1;32m   3041\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3042\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/anaconda/envs/tensorflow2/lib/python3.6/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1962\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1963\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1964\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1965\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1966\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/anaconda/envs/tensorflow2/lib/python3.6/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    594\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    595\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 596\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    597\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    598\u001b[0m           outputs = execute.execute_with_cancellation(\n",
      "\u001b[0;32m/usr/local/anaconda/envs/tensorflow2/lib/python3.6/site-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 60\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     61\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# training the model\n",
    "history = model.fit(X_train, y_train, epochs=30,\n",
    "                    validation_data=(X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gradient Clipping"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "All Keras optimizers accept `clipnorm` or `clipvalue` arguments:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = keras.optimizers.SGD(clipvalue=1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = keras.optimizers.SGD(clipnorm=1.0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To ensure that Gradient Clipping does not change the direction of the gradient vector, you should clip by norm by setting clipnorm instead of clipvalue. This will clip the whole gradient if its ℓ2 norm is greater than the threshold you picked."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reusing Pretrained Layers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reusing a Keras model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's split the fashion MNIST training set in two:\n",
    "* `X_train_A`: all images of all items except for sandals and shirts (classes 5 and 6).\n",
    "* `X_train_B`: a much smaller training set of just the first 200 images of sandals or shirts.\n",
    "\n",
    "The validation set and the test set are also split this way, but without restricting the number of images.\n",
    "\n",
    "We will train a model on set A (classification task with 8 classes), and try to reuse it to tackle set B (binary classification). We hope to transfer a little bit of knowledge from task A to task B, since classes in set A (sneakers, ankle boots, coats, t-shirts, etc.) are somewhat similar to classes in set B (sandals and shirts). However, since we are using `Dense` layers, only patterns that occur at the same location can be reused (in contrast, convolutional layers will transfer much better, since learned patterns can be detected anywhere on the image, as we will see in the CNN chapter)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loading the Fashion-MNIST dataset\n",
    "(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.fashion_mnist.load_data()\n",
    "\n",
    "# trimming the data since it takes lot of time\n",
    "X_train_full = X_train_full[:30000]\n",
    "y_train_full = y_train_full[:30000]\n",
    "\n",
    "X_test = X_test[:5000]\n",
    "y_test = y_test[:5000]\n",
    "\n",
    "# scaling the dataset\n",
    "X_train_full = X_train_full / 255.0\n",
    "X_test = X_test / 255.0\n",
    "\n",
    "# dividing the dataset into traingin and validation set\n",
    "X_valid, X_train = X_train_full[:5000], X_train_full[5000:]\n",
    "y_valid, y_train = y_train_full[:5000], y_train_full[5000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.arange(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False, False, False, False,  True,  True, False, False,  True,\n",
       "        True, False, False])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = np.array([1,2,3,4,5,6,7,8, 5, 6, 8, 9])\n",
    "x = np.arange(len(y))\n",
    "y_5_or_6 = (y == 5) | (y == 6)\n",
    "y_5_or_6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([5, 6, 5, 6]), array([4, 5, 8, 9]))"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[y_5_or_6], x[y_5_or_6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True,  True,  True,  True, False, False,  True,  True, False,\n",
       "       False,  True,  True])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "~y_5_or_6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([1, 2, 3, 4, 7, 8, 8, 9]), array([ 0,  1,  2,  3,  6,  7, 10, 11]))"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[~y_5_or_6], x[~y_5_or_6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# defining the dataset\n",
    "def split_dataset(X, y):\n",
    "    y_5_or_6 = (y == 5) | (y == 6) # sandals or shirts\n",
    "    y_A = y[~y_5_or_6]\n",
    "    y_A[y_A > 6] -= 2 # class indices 7, 8, 9 should be moved to 5, 6, 7\n",
    "    y_B = (y[y_5_or_6] == 6).astype(np.float32) # binary classification task: is it a shirt (class 6)?\n",
    "    return ((X[~y_5_or_6], y_A),\n",
    "            (X[y_5_or_6], y_B))\n",
    "\n",
    "(X_train_A, y_train_A), (X_train_B, y_train_B) = split_dataset(X_train, y_train)\n",
    "(X_valid_A, y_valid_A), (X_valid_B, y_valid_B) = split_dataset(X_valid, y_valid)\n",
    "(X_test_A, y_test_A), (X_test_B, y_test_B) = split_dataset(X_test, y_test)\n",
    "X_train_B = X_train_B[:200]\n",
    "y_train_B = y_train_B[:200]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(19875, 28, 28)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_A.shape # checking shape of training set A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200, 28, 28)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_B.shape # checking shape of training set B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4, 0, 5, 7, 7, 7, 4, 4, 3, 4, 0, 1, 6, 3, 4, 3, 2, 6, 5, 3, 4, 5,\n",
       "       1, 3, 4, 2, 0, 6, 7, 1], dtype=uint8)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_A[:30] # checking first 30 y-labels for training set A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1., 1., 0., 0., 0., 0., 1., 1., 1., 0., 0., 1., 1., 0., 0., 0., 0.,\n",
       "       0., 0., 1., 1., 0., 0., 1., 1., 0., 1., 1., 1., 1.], dtype=float32)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_B[:30] # checking first 30 y-labels for training set B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.random.set_seed(42)\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# defining the model\n",
    "model_A = keras.models.Sequential()\n",
    "model_A.add(keras.layers.Flatten(input_shape=[28, 28]))\n",
    "for n_hidden in (300, 100, 50, 50, 50):\n",
    "    model_A.add(keras.layers.Dense(n_hidden, activation=\"selu\"))\n",
    "model_A.add(keras.layers.Dense(8, activation=\"softmax\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/anaconda/envs/tensorflow2/lib/python3.6/site-packages/keras/optimizer_v2/optimizer_v2.py:356: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  \"The `lr` argument is deprecated, use `learning_rate` instead.\")\n"
     ]
    }
   ],
   "source": [
    "# compiling the model\n",
    "model_A.compile(loss=\"sparse_categorical_crossentropy\",\n",
    "                optimizer=keras.optimizers.SGD(lr=1e-3),\n",
    "                metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "622/622 [==============================] - 3s 4ms/step - loss: 0.7691 - accuracy: 0.7588 - val_loss: 0.4674 - val_accuracy: 0.8495\n",
      "Epoch 2/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.4378 - accuracy: 0.8546 - val_loss: 0.3845 - val_accuracy: 0.8729\n",
      "Epoch 3/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.3788 - accuracy: 0.8678 - val_loss: 0.3737 - val_accuracy: 0.8707\n",
      "Epoch 4/20\n",
      "622/622 [==============================] - 3s 5ms/step - loss: 0.3479 - accuracy: 0.8789 - val_loss: 0.3245 - val_accuracy: 0.8939\n",
      "Epoch 5/20\n",
      "622/622 [==============================] - 2s 4ms/step - loss: 0.3288 - accuracy: 0.8849 - val_loss: 0.3178 - val_accuracy: 0.8939\n",
      "Epoch 6/20\n",
      "622/622 [==============================] - 2s 4ms/step - loss: 0.3149 - accuracy: 0.8917 - val_loss: 0.3018 - val_accuracy: 0.8989\n",
      "Epoch 7/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.3045 - accuracy: 0.8942 - val_loss: 0.2941 - val_accuracy: 0.9018\n",
      "Epoch 8/20\n",
      "622/622 [==============================] - 2s 4ms/step - loss: 0.2956 - accuracy: 0.8979 - val_loss: 0.2906 - val_accuracy: 0.9041\n",
      "Epoch 9/20\n",
      "622/622 [==============================] - 2s 4ms/step - loss: 0.2879 - accuracy: 0.9000 - val_loss: 0.2865 - val_accuracy: 0.9046\n",
      "Epoch 10/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2816 - accuracy: 0.9028 - val_loss: 0.2941 - val_accuracy: 0.9028\n",
      "Epoch 11/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2756 - accuracy: 0.9056 - val_loss: 0.2796 - val_accuracy: 0.9076\n",
      "Epoch 12/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2707 - accuracy: 0.9059 - val_loss: 0.3372 - val_accuracy: 0.8886\n",
      "Epoch 13/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2668 - accuracy: 0.9087 - val_loss: 0.2726 - val_accuracy: 0.9088\n",
      "Epoch 14/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2611 - accuracy: 0.9101 - val_loss: 0.2704 - val_accuracy: 0.9096\n",
      "Epoch 15/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2575 - accuracy: 0.9102 - val_loss: 0.2946 - val_accuracy: 0.9033\n",
      "Epoch 16/20\n",
      "622/622 [==============================] - 3s 4ms/step - loss: 0.2541 - accuracy: 0.9118 - val_loss: 0.2678 - val_accuracy: 0.9116\n",
      "Epoch 17/20\n",
      "622/622 [==============================] - 6s 9ms/step - loss: 0.2503 - accuracy: 0.9140 - val_loss: 0.2669 - val_accuracy: 0.9098\n",
      "Epoch 18/20\n",
      "622/622 [==============================] - 4s 6ms/step - loss: 0.2468 - accuracy: 0.9129 - val_loss: 0.2756 - val_accuracy: 0.9028\n",
      "Epoch 19/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2436 - accuracy: 0.9158 - val_loss: 0.2654 - val_accuracy: 0.9086\n",
      "Epoch 20/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2408 - accuracy: 0.9176 - val_loss: 0.2605 - val_accuracy: 0.9111\n"
     ]
    }
   ],
   "source": [
    "# training the model\n",
    "history = model_A.fit(X_train_A, y_train_A, epochs=20,\n",
    "                    validation_data=(X_valid_A, y_valid_A))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "622/622 [==============================] - 2s 4ms/step - loss: 0.2380 - accuracy: 0.9177 - val_loss: 0.2589 - val_accuracy: 0.9143\n",
      "Epoch 2/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2350 - accuracy: 0.9192 - val_loss: 0.2608 - val_accuracy: 0.9123\n",
      "Epoch 3/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2333 - accuracy: 0.9196 - val_loss: 0.2786 - val_accuracy: 0.9051\n",
      "Epoch 4/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2307 - accuracy: 0.9205 - val_loss: 0.2579 - val_accuracy: 0.9153\n",
      "Epoch 5/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2282 - accuracy: 0.9205 - val_loss: 0.2611 - val_accuracy: 0.9128\n",
      "Epoch 6/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2262 - accuracy: 0.9221 - val_loss: 0.2534 - val_accuracy: 0.9158\n",
      "Epoch 7/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2241 - accuracy: 0.9220 - val_loss: 0.2532 - val_accuracy: 0.9150\n",
      "Epoch 8/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2222 - accuracy: 0.9230 - val_loss: 0.2578 - val_accuracy: 0.9183\n",
      "Epoch 9/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2201 - accuracy: 0.9251 - val_loss: 0.2557 - val_accuracy: 0.9170\n",
      "Epoch 10/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2184 - accuracy: 0.9250 - val_loss: 0.2604 - val_accuracy: 0.9145\n",
      "Epoch 11/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2164 - accuracy: 0.9256 - val_loss: 0.2553 - val_accuracy: 0.9163\n",
      "Epoch 12/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2146 - accuracy: 0.9257 - val_loss: 0.3178 - val_accuracy: 0.8896\n",
      "Epoch 13/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2137 - accuracy: 0.9274 - val_loss: 0.2527 - val_accuracy: 0.9173\n",
      "Epoch 14/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2106 - accuracy: 0.9260 - val_loss: 0.2507 - val_accuracy: 0.9163\n",
      "Epoch 15/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2094 - accuracy: 0.9284 - val_loss: 0.2905 - val_accuracy: 0.9063\n",
      "Epoch 16/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2085 - accuracy: 0.9267 - val_loss: 0.2518 - val_accuracy: 0.9185\n",
      "Epoch 17/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2063 - accuracy: 0.9291 - val_loss: 0.2515 - val_accuracy: 0.9175\n",
      "Epoch 18/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2044 - accuracy: 0.9294 - val_loss: 0.2618 - val_accuracy: 0.9086\n",
      "Epoch 19/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2029 - accuracy: 0.9298 - val_loss: 0.2522 - val_accuracy: 0.9168\n",
      "Epoch 20/20\n",
      "622/622 [==============================] - 2s 3ms/step - loss: 0.2017 - accuracy: 0.9302 - val_loss: 0.2463 - val_accuracy: 0.9170\n"
     ]
    }
   ],
   "source": [
    "# training the model\n",
    "history = model_A.fit(X_train_A, y_train_A, epochs=20,\n",
    "                    validation_data=(X_valid_A, y_valid_A))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_A.save(\"my_model_A.h5\") # saving the model we created"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rw-r--r-- 1 sandeepgiri9034 sandeepgiri9034 1136024 Nov 10 02:59 my_model_A.h5\r\n"
     ]
    }
   ],
   "source": [
    "!ls -al my_model_A.h5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# defining the model\n",
    "model_B = keras.models.Sequential()\n",
    "model_B.add(keras.layers.Flatten(input_shape=[28, 28]))\n",
    "for n_hidden in (300, 100, 50, 50, 50):\n",
    "    model_B.add(keras.layers.Dense(n_hidden, activation=\"selu\"))\n",
    "model_B.add(keras.layers.Dense(1, activation=\"sigmoid\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compiling the model with binary crossentropy\n",
    "# that can accept either logits (i.e values from last linear node, z)\n",
    "# or probabilities from the last Sigmoid node\n",
    "model_B.compile(loss=\"binary_crossentropy\",\n",
    "                optimizer=keras.optimizers.SGD(lr=1e-3),\n",
    "                metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "7/7 [==============================] - 1s 40ms/step - loss: 0.8140 - accuracy: 0.5550 - val_loss: 0.6947 - val_accuracy: 0.5974\n",
      "Epoch 2/5\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.5910 - accuracy: 0.6650 - val_loss: 0.5299 - val_accuracy: 0.7424\n",
      "Epoch 3/5\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.4586 - accuracy: 0.8100 - val_loss: 0.4380 - val_accuracy: 0.8256\n",
      "Epoch 4/5\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.3817 - accuracy: 0.8650 - val_loss: 0.3705 - val_accuracy: 0.8682\n",
      "Epoch 5/5\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.3229 - accuracy: 0.9050 - val_loss: 0.3190 - val_accuracy: 0.9118\n"
     ]
    }
   ],
   "source": [
    "# training the model\n",
    "history = model_B.fit(X_train_B, y_train_B, epochs=5,\n",
    "                      validation_data=(X_valid_B, y_valid_B))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_1 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 300)               235500    \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 100)               30100     \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 50)                5050      \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 50)                2550      \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 50)                2550      \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 8)                 408       \n",
      "=================================================================\n",
      "Total params: 276,158\n",
      "Trainable params: 276,158\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_A.summary() # generating model summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_3\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_3 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_18 (Dense)             (None, 300)               235500    \n",
      "_________________________________________________________________\n",
      "dense_19 (Dense)             (None, 100)               30100     \n",
      "_________________________________________________________________\n",
      "dense_20 (Dense)             (None, 50)                5050      \n",
      "_________________________________________________________________\n",
      "dense_21 (Dense)             (None, 50)                2550      \n",
      "_________________________________________________________________\n",
      "dense_22 (Dense)             (None, 50)                2550      \n",
      "_________________________________________________________________\n",
      "dense_23 (Dense)             (None, 1)                 51        \n",
      "=================================================================\n",
      "Total params: 275,801\n",
      "Trainable params: 275,801\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_B.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model_A = keras.models.load_model(\"my_model_A.h5\") # loading our saved model\n",
    "model_B_on_A = keras.models.Sequential(model_A.layers[:-1]) # creating new model based on existing layer\n",
    "model_B_on_A.add(keras.layers.Dense(1, activation=\"sigmoid\")) # adding new layer to new model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model_A and model_B_on_A now share some layers. When you train\n",
    "# model_B_on_A, it will also affect model_A. To avoid that, you need to clone\n",
    "# model_A before you reuse its layers. To do this, you clone model A’s\n",
    "# architecture with clone_model(), then copy its weights\n",
    "# (since clone_model() does not clone the weights)\n",
    "model_A_clone = keras.models.clone_model(model_A)\n",
    "model_A_clone.set_weights(model_A.get_weights())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# freezing reused layers\n",
    "for layer in model_B_on_A.layers[:-1]:\n",
    "    layer.trainable = False\n",
    "\n",
    "# compiling the model\n",
    "model_B_on_A.compile(loss=\"binary_crossentropy\",\n",
    "                     optimizer=keras.optimizers.SGD(lr=1e-3),\n",
    "                     metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_9\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_1 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 300)               235500    \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 100)               30100     \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 50)                5050      \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 50)                2550      \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 50)                2550      \n",
      "_________________________________________________________________\n",
      "dense_39 (Dense)             (None, 1)                 51        \n",
      "=================================================================\n",
      "Total params: 275,801\n",
      "Trainable params: 51\n",
      "Non-trainable params: 275,750\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_B_on_A.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "7/7 [==============================] - 0s 20ms/step - loss: 0.5306 - accuracy: 0.7600 - val_loss: 0.5263 - val_accuracy: 0.7556\n",
      "Epoch 2/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.4876 - accuracy: 0.8100 - val_loss: 0.4855 - val_accuracy: 0.8002\n",
      "Epoch 3/50\n",
      "7/7 [==============================] - 0s 15ms/step - loss: 0.4489 - accuracy: 0.8300 - val_loss: 0.4508 - val_accuracy: 0.8256\n",
      "Epoch 4/50\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.4160 - accuracy: 0.8600 - val_loss: 0.4192 - val_accuracy: 0.8611\n",
      "Epoch 5/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.3861 - accuracy: 0.8800 - val_loss: 0.3924 - val_accuracy: 0.8834\n",
      "Epoch 6/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.3606 - accuracy: 0.9000 - val_loss: 0.3685 - val_accuracy: 0.8966\n",
      "Epoch 7/50\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.3379 - accuracy: 0.9100 - val_loss: 0.3476 - val_accuracy: 0.9077\n",
      "Epoch 8/50\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.3179 - accuracy: 0.9150 - val_loss: 0.3281 - val_accuracy: 0.9189\n",
      "Epoch 9/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.2994 - accuracy: 0.9150 - val_loss: 0.3117 - val_accuracy: 0.9260\n",
      "Epoch 10/50\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.2837 - accuracy: 0.9300 - val_loss: 0.2968 - val_accuracy: 0.9341\n",
      "Epoch 11/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.2696 - accuracy: 0.9300 - val_loss: 0.2827 - val_accuracy: 0.9442\n",
      "Epoch 12/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.2561 - accuracy: 0.9400 - val_loss: 0.2704 - val_accuracy: 0.9483\n",
      "Epoch 13/50\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.2443 - accuracy: 0.9500 - val_loss: 0.2591 - val_accuracy: 0.9544\n",
      "Epoch 14/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.2336 - accuracy: 0.9500 - val_loss: 0.2489 - val_accuracy: 0.9584\n",
      "Epoch 15/50\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.2238 - accuracy: 0.9650 - val_loss: 0.2393 - val_accuracy: 0.9594\n",
      "Epoch 16/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.2145 - accuracy: 0.9700 - val_loss: 0.2302 - val_accuracy: 0.9604\n",
      "Epoch 17/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.2058 - accuracy: 0.9750 - val_loss: 0.2220 - val_accuracy: 0.9625\n",
      "Epoch 18/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.1979 - accuracy: 0.9750 - val_loss: 0.2138 - val_accuracy: 0.9655\n",
      "Epoch 19/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.1900 - accuracy: 0.9750 - val_loss: 0.2066 - val_accuracy: 0.9665\n",
      "Epoch 20/50\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.1832 - accuracy: 0.9800 - val_loss: 0.2001 - val_accuracy: 0.9665\n",
      "Epoch 21/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.1770 - accuracy: 0.9800 - val_loss: 0.1943 - val_accuracy: 0.9686\n",
      "Epoch 22/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.1714 - accuracy: 0.9850 - val_loss: 0.1888 - val_accuracy: 0.9706\n",
      "Epoch 23/50\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.1662 - accuracy: 0.9850 - val_loss: 0.1833 - val_accuracy: 0.9726\n",
      "Epoch 24/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.1609 - accuracy: 0.9900 - val_loss: 0.1784 - val_accuracy: 0.9757\n",
      "Epoch 25/50\n",
      "7/7 [==============================] - 0s 16ms/step - loss: 0.1562 - accuracy: 0.9900 - val_loss: 0.1739 - val_accuracy: 0.9767\n",
      "Epoch 26/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.1519 - accuracy: 0.9900 - val_loss: 0.1694 - val_accuracy: 0.9787\n",
      "Epoch 27/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.1476 - accuracy: 0.9900 - val_loss: 0.1651 - val_accuracy: 0.9787\n",
      "Epoch 28/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.1435 - accuracy: 0.9950 - val_loss: 0.1611 - val_accuracy: 0.9797\n",
      "Epoch 29/50\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.1398 - accuracy: 0.9950 - val_loss: 0.1576 - val_accuracy: 0.9807\n",
      "Epoch 30/50\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.1364 - accuracy: 0.9950 - val_loss: 0.1541 - val_accuracy: 0.9807\n",
      "Epoch 31/50\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.1331 - accuracy: 0.9950 - val_loss: 0.1507 - val_accuracy: 0.9817\n",
      "Epoch 32/50\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.1298 - accuracy: 0.9950 - val_loss: 0.1475 - val_accuracy: 0.9817\n",
      "Epoch 33/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.1268 - accuracy: 0.9950 - val_loss: 0.1445 - val_accuracy: 0.9838\n",
      "Epoch 34/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.1239 - accuracy: 0.9950 - val_loss: 0.1418 - val_accuracy: 0.9838\n",
      "Epoch 35/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.1213 - accuracy: 0.9950 - val_loss: 0.1391 - val_accuracy: 0.9838\n",
      "Epoch 36/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.1187 - accuracy: 0.9950 - val_loss: 0.1364 - val_accuracy: 0.9838\n",
      "Epoch 37/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.1162 - accuracy: 0.9950 - val_loss: 0.1340 - val_accuracy: 0.9838\n",
      "Epoch 38/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.1139 - accuracy: 0.9950 - val_loss: 0.1315 - val_accuracy: 0.9838\n",
      "Epoch 39/50\n",
      "7/7 [==============================] - 0s 15ms/step - loss: 0.1115 - accuracy: 0.9950 - val_loss: 0.1292 - val_accuracy: 0.9838\n",
      "Epoch 40/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.1093 - accuracy: 0.9950 - val_loss: 0.1271 - val_accuracy: 0.9838\n",
      "Epoch 41/50\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.1073 - accuracy: 0.9950 - val_loss: 0.1250 - val_accuracy: 0.9848\n",
      "Epoch 42/50\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.1054 - accuracy: 0.9950 - val_loss: 0.1231 - val_accuracy: 0.9848\n",
      "Epoch 43/50\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.1035 - accuracy: 0.9950 - val_loss: 0.1211 - val_accuracy: 0.9848\n",
      "Epoch 44/50\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.1017 - accuracy: 0.9950 - val_loss: 0.1193 - val_accuracy: 0.9858\n",
      "Epoch 45/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.1000 - accuracy: 0.9950 - val_loss: 0.1176 - val_accuracy: 0.9858\n",
      "Epoch 46/50\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.0983 - accuracy: 0.9950 - val_loss: 0.1158 - val_accuracy: 0.9868\n",
      "Epoch 47/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.0966 - accuracy: 0.9950 - val_loss: 0.1141 - val_accuracy: 0.9868\n",
      "Epoch 48/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.0951 - accuracy: 0.9950 - val_loss: 0.1126 - val_accuracy: 0.9868\n",
      "Epoch 49/50\n",
      "7/7 [==============================] - 0s 13ms/step - loss: 0.0936 - accuracy: 0.9950 - val_loss: 0.1111 - val_accuracy: 0.9868\n",
      "Epoch 50/50\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.0921 - accuracy: 0.9950 - val_loss: 0.1095 - val_accuracy: 0.9868\n"
     ]
    }
   ],
   "source": [
    "# training the model\n",
    "history = model_B_on_A.fit(X_train_B, y_train_B, epochs=50,\n",
    "                           validation_data=(X_valid_B, y_valid_B))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "7/7 [==============================] - 1s 40ms/step - loss: 0.0624 - accuracy: 0.9950 - val_loss: 0.0763 - val_accuracy: 0.9929\n",
      "Epoch 2/5\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.0589 - accuracy: 0.9950 - val_loss: 0.0729 - val_accuracy: 0.9929\n",
      "Epoch 3/5\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.0559 - accuracy: 0.9950 - val_loss: 0.0699 - val_accuracy: 0.9929\n",
      "Epoch 4/5\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.0532 - accuracy: 0.9950 - val_loss: 0.0669 - val_accuracy: 0.9929\n",
      "Epoch 5/5\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.0506 - accuracy: 0.9950 - val_loss: 0.0645 - val_accuracy: 0.9929\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# unfreezing reused layers\n",
    "for layer in model_B_on_A.layers[:-1]:\n",
    "    layer.trainable = True\n",
    "\n",
    "# compiling after reducing learning rate to avoid damaging the reused weights\n",
    "model_B_on_A.compile(loss=\"binary_crossentropy\",\n",
    "                     optimizer=keras.optimizers.SGD(lr=1e-3),\n",
    "                     metrics=[\"accuracy\"])\n",
    "\n",
    "# training the model\n",
    "history = model_B_on_A.fit(X_train_B, y_train_B, epochs=5,\n",
    "                           validation_data=(X_valid_B, y_valid_B))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So, what's the final verdict?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "7/7 [==============================] - 0s 19ms/step - loss: 0.0483 - accuracy: 0.9950 - val_loss: 0.0622 - val_accuracy: 0.9929\n",
      "Epoch 2/5\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.0462 - accuracy: 0.9950 - val_loss: 0.0602 - val_accuracy: 0.9929\n",
      "Epoch 3/5\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.0444 - accuracy: 1.0000 - val_loss: 0.0583 - val_accuracy: 0.9929\n",
      "Epoch 4/5\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.0428 - accuracy: 1.0000 - val_loss: 0.0564 - val_accuracy: 0.9929\n",
      "Epoch 5/5\n",
      "7/7 [==============================] - 0s 14ms/step - loss: 0.0411 - accuracy: 1.0000 - val_loss: 0.0548 - val_accuracy: 0.9929\n"
     ]
    }
   ],
   "source": [
    "history = model_B_on_A.fit(X_train_B, y_train_B, epochs=5,\n",
    "                           validation_data=(X_valid_B, y_valid_B))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31/31 [==============================] - 0s 2ms/step - loss: 0.3065 - accuracy: 0.9266\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.3065095841884613, 0.9265770316123962]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_B.evaluate(X_test_B, y_test_B) # evaluating the model A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31/31 [==============================] - 0s 2ms/step - loss: 0.1343 - accuracy: 0.9835\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.13432174921035767, 0.9834539890289307]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_B_on_A.evaluate(X_test_B, y_test_B) # evaluating the model B"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Great! We got quite a bit of transfer: the error rate dropped by a factor of 4!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "error1 = 1 - 0.943\n",
    "error2 = 1 - 0.979"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.05700000000000005"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "error1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.066666666666663"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(100 - 96.95) / (100 - 99.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.02100000000000002"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "error2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Faster Optimizers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Momentum optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = keras.optimizers.SGD(lr=0.001, momentum=0.9)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Nesterov Accelerated Gradient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = keras.optimizers.SGD(lr=0.001, momentum=0.9, nesterov=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AdaGrad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = keras.optimizers.Adagrad(lr=0.001)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RMSProp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = keras.optimizers.RMSprop(lr=0.001, rho=0.9)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adam Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adamax Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = keras.optimizers.Adamax(lr=0.001, beta_1=0.9, beta_2=0.999)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Nadam Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = keras.optimizers.Nadam(lr=0.001, beta_1=0.9, beta_2=0.999)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learning Rate Scheduling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Power Scheduling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```lr = lr0 / (1 + steps / s)**c```\n",
    "* Keras uses `c=1` and `s = 1 / decay`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = keras.optimizers.SGD(lr=0.01, decay=1e-4) # implementing Power Scheduling by setting decay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# defining the model\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.Dense(300, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n",
    "    keras.layers.Dense(100, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n",
    "    keras.layers.Dense(10, activation=\"softmax\")\n",
    "])\n",
    "\n",
    "# compiling the model\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'X_train_scaled' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-41-778c4560c40e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m#training the model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m history = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n\u001b[0m\u001b[1;32m      5\u001b[0m                     validation_data=(X_valid_scaled, y_valid))\n",
      "\u001b[0;31mNameError\u001b[0m: name 'X_train_scaled' is not defined"
     ]
    }
   ],
   "source": [
    "n_epochs = 5 # setting the epoch\n",
    "\n",
    "#training the model\n",
    "history = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n",
    "                    validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAEeCAYAAAAjNKpiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deZxcVZn/8c+300vSW/Y9SFhigCCECYqKSEBFcYYhyow/FRVmBBwYxhlWwdERmXEURxSZQZZBREQZGQTDsImIYZNNFiEBCRESyL6SpDt78vz+uLc71ZXq7upOdd3u9Pf9etWrq845t+qpS9NPzrnnnqOIwMzMLAsVWQdgZmb9l5OQmZllxknIzMwy4yRkZmaZcRIyM7PMOAmZmVlmnITM+hBJ8yWd3wPv+1eSunS/hqRTJTW199qsGE5C1qdIulFSpI+tkl6T9B1JdVnHVgxJp0l6TlKTpLWSXpD0b1nHVSI/B/bNOgjrWyqzDsCsGx4APgtUAUcB1wN1wJlZBtVCUnVEbClQ/rfAlcA5wG9I4j8YeE95I+wZEbER2Jh1HNa3uCdkfdHmiFgaEW9GxM+AnwIzWiolvV/Sk5I2SVom6XuSqtO6j0haL6kyfb1/2qu6Juf4f5P0QM7rgyTdnR63XNItksbk1N8o6S5JX5K0EFjYTtx/CdweEddGxLyIeDki/jcizs1tJOmjafwbJa2S9H+SBuY0GSjpWknrJC2UdEHe8YMlXZfGul7SQ5IOz2vzOUkLJG2QdBcwOq/+Ekmz88o6HG4rMDx3iaTZkj4p6U9pLL+UNCKnTWX632dN+viepKslzWrvc2zP4iRke4KNJL0KJI0H7gWeAw4DPg98Cvhm2vZRYCDQ8kd5OrAy/UlO2az0/cYCDwOzgXcBHwTqgZmScv//ORo4BPgI8IF24lwKvEtSu0NWkj4C3An8GpgGHAM8RNv/V88BXgT+DLgM+Lak96THC7gbGA/8RXoOHgYeTL8Lko4AbgSuA6YC/wdc2l5Mu2ki8P+AjwHHpfF8I6f+fOBU4DTg3STf89M9FIv1RhHhhx995kHyx/OunNfvIkkiP09ffwN4FajIaXMqsBmoTV8/AVycPr8Z+BpJIhsL1KZt35fWXwr8Ji+GoUAA78qJaQVQ00nsY4HH02NfTT/7c0BVTpvHgP/p4D3mA7fklb0KfCV9fizQBAzKa/M8cGH6/GfAr/Pqr0/+HLS+vgSYndfmVKCpC68vATYBg3PK/hmYl/N6CXBRzmsBrwCzsv5d86M8D/eErC/6SHphfxPJH/WHgX9I6w4EnoiIHTntHwWqgf3T17PY2fM5mqTn9GRa9l5gG/BUWj8NeH/6eU3pcNObad1+OZ8xOyI2dxR0RCyJiPcA7wCuIPmDey3wlKTatNlhJNeLOvJC3uvFwKiceGuBFXkxH5wT74Ek5y1X/utSWRARawvFKmkwMIad55qIiNzXtufzxATrix4GzgC2AosjYmuRx7VMQZ4FnC3pQKAReCYtOwZYDjweOycWVJAMbxWaFr0s53lzscFHxGyS4b2rJL0PeAT4BEmPqhj53zfYOVxXkcZ1VIHj1hUbI7CDJEnmqurC8S06itXMScj6pA0RMa+dupeBT0iqyOkNvQ/YAvwpff0oUANcCDwaEdvTC+H/TfIH/L6c93uWJEEs6EKy64qX0p/16c/nSK4p/Xc33+9ZkkkGOyLitXbavExy/SVX/usVwGhJSnsnkFw/KpmIWCtpKfBO4EFovab1TpLrZ9YP+F8ktqf5ATAO+IGkAyX9OfAt4L8iYgNARDSR9H4+A/w2Pe4JYALJH+NZOe93FTAY+LmkIyTtK+mD6eyzhq4Els76+qqkIyXtLendwE3ABuD+tNk3gL9OZ+gdJGmKpHNyhus68wDJdaWZko6XtI+k90j6uqSW3tGVwAclXSxpkqTTSSYO5JoFDAO+LGk/SZ8H/qor37dI3wculPQxSZOBy0munXmjs37CScj2KBGxCDie5NrK88ANwC3Al/OaziIZCZiVHreJ5LrQZtpeo1gMHEkyPHUfMIckMW1OH13xa+AI4FZgLnBHWv6hiJibft49JAnheJJe0UMkw4Q7dnm3AtJey0dJehb/TXKR/1ZgMsn1GCLiCZJZg2eSXF/6OMkkgtz3eTmtPyNt8yHg37v4fYvxHeAnwI9I/iEAyXnZ1AOfZb2Qdva0zcyyJ+k5kmHSf+i0sfV5viZkZpmRtDfwYZIeXxVwOsn9VqdnGZeVT1mH4yQNk3SHpOb0bu2CN6UpcVl6t/iq9Lly6q+T9IqkHZJOLXD8OZKWpneU3yCppge/lpl13w6Se6WeIpkm/m7g+Ij4faZRWdmU+5rQVSSzlEYDJwNXS5pSoN0ZJMuwHEryr6ITgC/k1P8BOItkJlAbkj4MXEQyw2hvkgUVv166r2BmpRLJ0kvvi4jBEdEQEUdExP2dH2l7irJdE1KyyvEa4OCWi7CSfgIsioiL8tr+DrgxIq5LX38eOD0i3p3X7lHg+oi4MafsZ8D8iPhy+voDwE8jYgxmZtarlPOa0NuBbS0JKPUHkjvW801J63LbFeoxFTIFmJl37GhJwyNiVW5DSWeQ9LqoGNQ4rXLwqNa6hipRPQCqB4jqClD+bXsZ2bFjBxUVvX9So+Msrb4QZ1+IERxnqc2dO3dlRIzs7vHlTEL17HrH9lqg0L0W9Wldbrv6vBvnOvqc/GNJP6dNEkp7WtcB1IydFGNPuQKA6soKBlZWsG7TNgC2CvYZUcdBYxs5aFwjU8YN5qCxjYxsKP+lplmzZjF9+vSyf25XOc7S6gtx9oUYwXGWmqQFu3N8OZNQE8kSKbkagfVFtG0kWRixmLHDQsfSzufsYlDVAL758Xdw4tRxLHprIy8tXsdLS9bx0uJ1PP/mW9z1wpLWtiMbanISUyMHjW1k4vA6Kip6SbfJzKyXK2cSmgtUSpoUEa+mZYeS3PyXb05a91Qn7QppOfbWnGOX5Q/FFTJ+yCAu+PBkZhw2HoAJQ2uZMLSW46bsvJy0dsPWJCmliemlJet47OHX2LYjyY+11QM4YExDmx7T5DENDKwaUGT4Zmb9R9mSUEQ0S7oduFTSaSTrUJ1IsmpxvpuAcyXdQ7J8x3nAf7ZUKtmgrIJkgcUqJRt+bUnXCrsJuFHST0nuEP8KRSwMObGxgscuOrbT7zG4tor37Dec9+w3vLVs87btvLqsqU1imvncYm5+4g0AKgT7jaznoLS3NGXcYA4a18iwuupOP8/MbE9W7ptVzyJZRmU5yfWZMyNiTrqm1b0R0bKI47UkU6tfTF9fn5a1uJ+dExreS3Jd5xiSPUjuk/RtkjXBBgG/INkvpsfUVA7g4PGDOXj84NayiGDhmo3MWby2NTE9/fpqZj6/uLXNmMaBOYkpGdbba2ith/PMrN8oaxKKiNXkbMOcU/4IO1cRbln/6sL0Ueh9pnfyOd8Fvrs7se4uSew1rJa9htXykYPHtpavad7Spsf00uJ1PDR3BdvT4bz6mkoOHNvQpsc0aXQ9NZUezjOzPY+X7SmzoXXVHLn/CI7cf0Rr2aat25m7bH2bxHTbMwv58ePJpJPKCrH/qHoOGttI9YatVO+1koPGNjKk1sN5Zta3OQn1AgOrBnDIhCEcMmFIa9mOHcEbqzcwZ/E6XlqSDOk99qeVLFu3hf955UkgmUhxYDo7r2VIb8LQQai33NRkZtYJJ6FeqqJCTBxRx8QRdfz5ITuH8+781W8Zuu/BSXJKe04P/nEZ6WgeDQMrW6eNtwzp7T+qnurK3n/Tm5n1P05CfUxjjThq0kiOmrTzBuWNW7bzyrL1bSZB/M9Tb7Jx63YAqgaISaMaWhPTQekkiMaB3dmt2cysdJyE9gCDqgcwda8hTN1r53De9h3B/FXNbXpMs15Zzm3PLGxts9ewQUlSGju49YbbsYMHejjPzMrGSWgPNaBC7Deynv1G1vOXh45rLV++fhMvLV6XXmtax8uL13H/S8toWYtiSG1Vmph2LlG078g6qgZ4OM/MSs9JqJ8Z1TCQUZMHMn3yzsVamzdv449L16cz85IhvZ88sYDN25IdpasrK5g8uqHNUN6BYxupr9n11+eXzy3iP371Cove2sj4Jx5sswKFmVk+JyGjrqaSaXsPZdreQ1vLtm3fwesrm3lpybrWIb37X1rKz3//ZmubicNr21xnenP1Br517x/ZuDVJXove2sjFtyf3GzsRmVkhTkJWUOWACiaNbmDS6AZOnJokkIhg2brNrVPG56SPe15c2u77bNy6ncvu+6OTkJkV5CRkRZPEmMEDGTN4IMceMLq1fP2mrfxx6Xr++prHCx63ZO0m3vWNB9h/VD2TRtWz/+gG9h9Zz6TR9Qyvq/ZECLN+zEnIdlvDwCreOXEY44cMYtFbG3epbxxYyfvfPpJ5y5v4xbOLaNq8rbVuaG0V+4+qZ/9RDUmCGpUkpzGNnqVn1h84CVnJXPDhyVx8+4ut9ydBsj/TpSce3DocFxEsXbeJecubeHVZE68ub+JPy5u4d/YSbtmwtfW4+ppK9kt7Tq3JaVQDE4YO8gKvZnsQJyErmZZE0zo7Lm9/JkiG9MYOHsTYwYPa3HAbEaxq3pIkp+VNzFu2nnkrmnh47oo29zYNrKpg3xFJb6klOe0/qoG9h9d6GrlZH+QkZCU147DxzDhsfJe3JpbEiPoaRtTX8O59h7epW7txK/OWNzFv+XpeXdbEvBVN/H7+mjbbYlQNEBOH1zFpdJKUWq4/7TOizhsKmvViTkLW6w0eVLXLFHJI7m96bUUzry5fn/Seljfx8pL13Dd7aetaehWCtw2rTa45ja5vnRCx38h66grc52Rm5eX/C63Pqqup5B0TBvOOCYPblG/aup35q5rbXHN6dfl6Hpq7nK3bo7Xd+CGDGFa5hUebXmqdELH/yAYG13pNPbNycRKyPc7AqgEcMKaRA8Y0tinfun0Hb6zekAzpLV/PvOVNPPfaUm5+cgGb0htsAUY21LSZENHSi/J0crPScxKyfqNqQEXrenowBoBZs2bx/vcfzaK3NibDesuaWidH5E8nH1JblSamndecPJ3cbPc4CVm/V1Gxcyv23JtwW1aIeDVnQsS8ZU3cN3sJa9qZTt6anDyd3KwoTkJm7chdISJ3OjnAqqbNvLq87TWn/OnkNZVJzyt3QoSnk5u15SRk1g3D62sY3sl08pZhvQ6nk49MljFqbzq5VyW3PZ2TkFkJtTedfMOWbfxpeTKdvCU5dTSdfP9R9azbuIVfPLuodUsNr0pueyInIbMyqK3ufDr5vOUtkyJ2nU7eYuPW7fzLzNlUV1YwcXgdew+v9f1O1qf5t9csQ+1NJ9+2fQeT/vledk1DsG7TNs766bOtr0c11LQmpIkj6pg4vI6JI2rZe3hdwY0HzXoT/4aa9UKVAyoY186q5GMHD+T6Uw5n/soNzF/VzIJVzcxfuYGH5q7gf3MmRgCMqK9hnzQhTcxJUnsPr6VhoG/Ktew5CZn1Uu2tSv6ljxzAlHGDmTJu8C7HNG/exoJVG1iwqpnXVzWzYOUGXl/VzCOvruC2Zza3aTuivjpNTnkJakQtjU5QViZOQma9VDGrkuerq6lMtlwf17hL3YYtOQlqZfJz/qpmHpu3kl88u6lN2+F11W2G9/YeXss+I+rYe3gdgwc5QVnpOAmZ9WLdXZW8kNrqSg4c28iBY3dNUBu3bGfB6mRYryU5zV+5gcf/tIrbn13Upu3Q2qqd155yrj81by10BcusY05CZsag6sITJCCZwffG6g28vrK5TS/qqddX88vnFxE5ueefH7+/dXhv7+F1ae8p6UUNqa0u4zeyvsJJyMw6NLBqAG8f3cDbRzfsUrdp63beTBPUb556kcohY5i/qpmn569h5h8Wt0lQgwdVtV57yp8oMbS2yuvv9VNOQmbWbQOrBjBpdAOTRjdQveKPTJ/+jta6TVu3s3DNhtZZfC1DfM8sWMOdeQmqcWBla3LaJ+1FJQmqlmFevXyP5iRkZj1iYNWAdPWHXXtQm7dt583VG9PhvWYWrEoS1fNvruHuFxa3riIB0DCwcpfJES29KG+v0fc5CZlZ2dVUDkj3aqrfpW7Lth28uWZD6/1PSS9qAy8sXMs9Ly5pk6DqaypzZvHVphMlkoQ1sr6mTYLyOny9k5OQmfUq1ZW5+z61tWXbDha9tZH5K1uG95IENWfRWu6bvZTtORmqrnpA6+SIzdu289DcFa1LIXkdvt6jrElI0jDgh8BxwErg4oj4WYF2Ar4FnJYWXQ9cFJGMIkuamr7PgcDLwOcj4vm0rgb4PvAxoAp4DPi7iGg7z9TM+pzqygr2GZEklnxbt+9g0ZqN6U26SXKav6qZl5as4/WVzbu037h1O1/6xQs8+fpqJgwdlD5q2WvoIEbU13gvqDIpd0/oKmALMBqYCtwt6Q8RMSev3RnADOBQIIBfA68D10iqBmYCVwA/AL4AzJQ0KSK2AP8IvAc4BFgLXAf8J/DxHv5uZpahqgEVybDciDqY3LZun4vuLrgO3+ZtO7h/zlJWNW9pU15dWcGEIYOYMKy2TYJqeZ4/1GfdV7YkJKkOOAk4OCKagEcl3Ql8Frgor/kpwOURsTA99nLgdOAaYHoa9xVpz+hKSecDxwL3AfsAv4qIZemxPwe+28Nfz8x6sfbW4Rs/ZBCPXXQszZu3seitjSxcs4GFazamj+T57EVrWZ2XpGoqKxifl5icpLpHEeW5y1nSYcBjEVGbU3Y+cHREnJDXdi1wXEQ8mb4+HPhtRDRIOietOz6n/V1p/eVp2+8Dfw28RTKUtzwi/qlATGeQ9LoYOXLktFtvvbW0X7oHNDU1UV+/61h5b+M4S6svxNmbY/zd4q3cOHsLW3bsLKuugFMPrua94zpfhmjTtmDVxmDFxh2s3BjpYwer0p/rt7ZtX1UBIwaJEYMq0p+5zytorKbTJNWbz2euY4455pmIOLy7x5dzOK4eWJdXthbYdf5m0nZtXrv69FpRfl3++7wKvAksArYDLwJnFwooIq4jGa5j8uTJsbvLopRDKZZvKQfHWVp9Ic7eHON04KDc2XFFrMPXFU2bt7FoTW5PameP6vlVG1izoW2WGlhVwfghub2nnb2ovYbVMryumoceeqjXns9SKmcSagLy1wRpBNYX0bYRaIqIkNTZ+1wF1ADDgWbgQuBe4Ijdit7M+rRSrsOXr76mksljGpg8ptC/qTtOUi8sfKtgkhpWHbz99afyklTyc0+6P6qcSWguUJlOIHg1LTsUyJ+UQFp2KPBUgXZzgPMkKXaOJR5CknwgmfDwzxGxGkDSfwKXShoREStL+o3MzIrQWZJav2lrck1q9c4E9dyrb7CyaTPPv/kWbxVIUvnXo/bKed2XVpkoWxKKiGZJt5MkhNNIksWJwHsLNL8JOFfSPSSz484jmeEGMItkmO2Lkq4hmbAA8GD682ngc5JmARuAs4DFTkBm1ls1DKzigDFVbRaQnTVrOdOnHwUUTlIL12zkzTUbeO6Nt1i7sW2SGlQ1oMCEid6ZpMo9Rfss4AZgObAKODMi5kg6Crg3Ilquwl0L7EtyPQeSyQXXAkTEFkkz0rJvkdwnNCOdng1wPnAlybWhamA2yT1DZmZ9UqEklWvdpq3pcN+uQ37PFpGk9hrWNlkVs6BsywoU1WP2n7Y7362sSSgdIptRoPwRkgkHLa+D5FrOhe28z3NAwS8eEauAk0sRr5lZX9A4sIrGsVUF94qC9pPUm6s38syCNazbtK1N+9rqAbv0nnKT1EOvLOfLd8xus+tvd3nZHjOzPVxnSWrtxq15Eyd2Pv/9/NW7JClBwZt/u8NJyMysnxs8qIrBg6oKbgsPuyapS+96qWSf7SRkZmYdyk9SP3z09YIrUHRHRUnexczM+o0LPjyZQVUDSvJe7gmZmVmXtKw08R+/eoUlu/le7gmZmVmXzThsPI9ddCxbls57Znfex0nIzMwy4yRkZmaZcRIyM7PMOAmZmVlmnITMzCwzRSchSaMlnS/pakkj0rIjJe3Tc+GZmdmerKgkJGka8ArJwqCfZ+emch8CvtEzoZmZ2Z6u2J7Qd4DvR8RhwOac8l8BR5Y8KjMz6xeKTULTgB8XKF8CjC5dOGZm1p8Um4Q2AkMLlB9AskGdmZlZlxWbhGYCX5NUk74OSROBy4Bf9EBcZmbWDxSbhM4HhgErgFrgUWAe8BbwlZ4JzczM9nRFraIdEeuA90k6FvgzkuT1bEQ80JPBmZnZnq2oJCTpc8DPI+JB4MGc8mrgkxFxUw/FZ2Zme7Bih+N+BAwuUN6Q1pmZmXVZsUlIQBQofxuwtnThmJlZf9LhcJykF0mSTwAPSdqWUz0A2Bu4p+fCMzOzPVln14RuS38eDNwNNOXUbQHm4ynaZmbWTR0moYj4OoCk+SQTEzaVIygzM+sfip2iXWjJHjMzs91S7Cra1ZK+LmmupE2Stuc+ejpIMzPbMxU7O+5fgVOAy4EdwAXAVcAq4KyeCc3MzPZ0xSahTwB/FxHXAtuBmRHxReBrJHsKmZmZdVmxSWg08FL6vAkYkj6/Dziu1EGZmVn/UGwSegMYlz6fB3w4ff4ekm0ezMzMuqzYJHQH8IH0+feBr0t6HbgRuL4H4jIzs36g2CnaF+c8v03SmyTbes+NiLt6KjgzM9uzFdsTaiMinoyI70bEXZLqij1O0jBJd0hqlrRA0qfbaSdJl0lalT4uk6Sc+qmSnpG0If05Ne/4P5P0sKQmScsk/WN3vqeZmfWsbiUhAEkDJV0AvN6Fw64iWe5nNHAycLWkKQXanQHMAA4FDgFOAL6Qfm41yU6vN5NsOf5jYGZajqQRJBMmrgWGA/sD93f1+5mZWc/rMAmlN6l+Q9LTkn4naUZa/jngNeCfgO8V80Fpj+kk4KsR0RQRjwJ3Ap8t0PwU4PKIWBgRi0juTzo1rZtOMox4RURsjogrSVb5PjatPxf4VUT8NK1fHxEvFxOjmZmVlyIK7dCQVkr/Dvw98GuSa0AjgP8mmaTw78DPImJrUR8kHQY8FhG1OWXnA0dHxAl5bdcCx0XEk+nrw4HfRkSDpHPSuuNz2t+V1l8u6UHgReCdJL2gJ4G/j4g3CsR0Bkmvi5EjR0679dZbi/kqmWpqaqK+vj7rMDrlOEurL8TZF2IEx1lqxxxzzDMRcXh3j+9sYsIngFMj4g5JhwLPkQyBTYmIbR0fuot6YF1e2VqSjfEKtV2b164+vS6UX5f/PhNItiD/EEky+jZwC0kSbSMirgOuA5g8eXJMnz69+G+TkVmzZuE4S8dxlk5fiBEcZ2/TWRLaC3gaICL+IGkLcFk3EhAkN7k25pU1AuuLaNsINEVESOrsfTYCd0TE0wCSvg6slDQ4IrwBn5lZL9LZxIQqYHPO6610fyfVuUClpEk5ZYcCcwq0nZPWFWo3Bzgkd7YcyeSFlvoXaLsLbPvjjWZmlqli7hP6pqQN6fNq4JL0mk2rdB25DkVEs6TbgUslnQZMBU4E3lug+U3AuZLuIUki5wH/mdbNIlm/7ouSrgFOT8sfTH/+CPiFpCtJEtNXgUfdCzIz6306S0IPA/vlvP4d8La8Nl3paZwF3AAsJ1mB+8yImCPpKODeiGi5CnctsC/JNR1IVmW4FiAitqSz9K4HvgW8DMyIiC1p/YOSvkyyE2wt8ChQ8H4kMzPLVmc7q04v5YdFxGqS+3/yyx8hmXDQ8jqAC9NHofd5DpjWwedcDVy9u/GamVnP6vbNqmZmZrvLScjMzDLjJGRmZplxEjIzs8w4CZmZWWaK2k9IUv607BYBbIqIFaULyczM+ouikhAwnw7uB5K0juQm0Qu7uaSPmZn1Q8UmoU+RLAR6Dcmq1ABHkKxAfQkwBPgKyfptXyttiGZmtqcqNgmdCZwTEbfnlD0o6RXgHyPiaEnLga/jJGRmZkUqdmLCEexcQifXbJJ9ewAeJ9lGwczMrCjFJqEFpJu/5TkdaNksbiSwuhRBmZlZ/1DscNx5JCtTf5R0fyHgcJLFTU9KX78T6P1bk5qZWa9RVBKKiLvTfYDOAianxXcC17Rsmx0RP+iZEM3MbE9VbE+IiHgTuLgHYzEzs36m6CQkqZZkI7pR5F1Lyps1Z2ZmVpRiV0z4IHALMLxAdQADShmUmZn1D8XOjvs+yU6lEyKiIu/hBGRmZt1S7HDcROAvI2JxD8ZiZmb9TLE9ocfYOSvOzMysJIrtCV0DfEfSOJKVE7bmVkbEs6UOzMzM9nzFJqHb0p/XFajzxAQzM+uWYpPQPj0ahZmZ9UvFrpiwoKcDMTOz/qfdJCTp48D/RcTW9Hm7fLOqmZl1R0c9oduAMcBydl4TKsTXhMzMrFvaTUIRUVHouZmZWak4uZiZWWa6soDpBOD9FF7A9LsljsvMzPqBYhcwPRm4AdgGrCC5DtQiACchMzPrsmJ7QpcClwNfjYjtPRiPmZn1I8VeExoNXO8EZGZmpVRsEroHOKInAzEzs/6n2OG4XwOXSZpC4QVMfbOqmZl1WbFJ6Nr055cL1PlmVTMz65aihuMK7KbarZ1VJQ2TdIekZkkLJH26nXaSdJmkVenjMknKqZ8q6RlJG9KfUwu8R7WklyUtLDY+MzMrr06TkKQqSU9KKsWmdlcBW0gmOpwMXJ0O8eU7A5gBHAocApwAfCGNpxqYCdwMDAV+DMxMy3NdQDKd3MzMeqlOk1BEbCXZyiE6a9sRSXXASSTTvJsi4lHgTuCzBZqfAlweEQsjYhHJ9PBT07rpJMOIV0TE5oi4EhBwbM5n7QN8Bvjm7sRsZmY9SxGd5xZJ/wEQERd0+4Okw4DHIqI2p+x84OiIOCGv7VrguIh4Mn19OPDbiGiQdE5ad3xO+7vS+stzXv8QWAPcHBET2onpDJJeFyNHjpx26623dvfrlU1TUxP19fVZh9Epx1lafSHOvhAjOM5SO+aYY56JiMO7e3yxExPqgJMlfQh4BmjOrYyILxbxHvXAuryytUBDO23X5rWrT68L5de1eR9JHwMGRMQdkqZ3FFBEXEe6W+zkyZNj+vQOm/cKs2bNwnGWjuMsnb4QIzjO3qbYJHQg8Gz6fN+8umKH6ZqAxryyRqON9AoAAA7ISURBVGB9EW0bgaaICEntvk865Pdt4KNFxmRmZhkqdmfVY0rwWXOBSkmTIuLVtOxQYE6BtnPSuqcKtJsDnCdJsXMs8RCSSQ+TgInAI+lkumpgsKSlwLsjYn4JvoeZmZVI2bZyiIhm4HbgUkl1ko4ETgR+UqD5TcC5ksZLGgecB9yY1s0CtgNflFQj6ey0/EFgNrAXMDV9nAYsS5+/2RPfy8zMuq8rWzkcA3wKeBtJD6NVRBxb8KBdnUWyGvdyYBVwZkTMkXQUcG9EtFyFu5Zk2O/F9PX1aRkRsUXSjLTsW8DLwIyI2JK2XZoT82pgR0S0lpmZWe9R7FYOpwLXAHeQTJGeCbydZOr2zcV+WESsJrn/J7/8EZIJBy2vA7gwfRR6n+eAaUV83iyg4Mw4MzPLXrHDcecDZ0fEp0jWjbs4Ig4jSUBNPRWcmZnt2YpNQvsCD6TPN7Oz1/Jf7LyJ1MzMrEuKTUKr2Hk/zyLg4PT5cGBQqYMyM7P+odiJCY8Ax5FMFLgVuDK9cfUDJNs8mJmZdVmxSehsYGD6/JvANuBIkoT0bz0Ql5mZ9QPF3qy6Ouf5DuCyHovIzMz6jaJvVpU0WtL5kq6WNCItOzJdsdrMzKzLikpCkqYBr5DsAfR5dq7d9iHgGz0TmpmZ7emK7Ql9B/h+em/Q5pzyX5FcGzIzM+uyYpPQNJIdTPMtIdkl1czMrMuKTUIbSbbSzncAyTpwZmZmXVZsEpoJfE1STfo6JE0kmSX3ix6Iy8zM+oGurB03DFgB1AKPAvNIdjT9Ss+EZmZme7pi7xNaB7xP0rHAn5Ekr2cj4oGOjzQzM2tf0fsJAUTEgySbxwEgaW/gPyLiE6UOzMzM9ny7u7PqEOCkUgRiZmb9T9m29zYzM8vnJGRmZplxEjIzs8x0ODFB0p2dHN/YSb2ZmVm7Opsdt6qI+tdLFIuZmfUzHSahiPibcgViZmb9j68JmZlZZpyEzMwsM05CZmaWGSchMzPLjJOQmZllxknIzMwy4yRkZmaZcRIyM7PMOAmZmVlmnITMzCwzTkJmZpaZsiYhScMk3SGpWdICSZ9up50kXSZpVfq4TJJy6qdKekbShvTn1Jy6CyTNlrRe0uuSLijHdzMzs64rd0/oKmALMBo4Gbha0pQC7c4AZgCHAocAJwBfAJBUDcwEbgaGAj8GZqblAAI+l9Z9BDhb0id76guZmVn3lS0JSaoDTgK+GhFNEfEocCfw2QLNTwEuj4iFEbEIuBw4Na2bTrL69xURsTkiriRJPMcCRMS3I+LZiNgWEa+QJKwje/CrmZlZNykiyvNB0mHAYxFRm1N2PnB0RJyQ13YtcFxEPJm+Phz4bUQ0SDonrTs+p/1daf3lee8j4Fng2oi4pkBMZ5D0uhg5cuS0W2+9tUTftuc0NTVRX1+fdRidcpyl1Rfi7AsxguMstWOOOeaZiDi8u8d3tqldKdUD6/LK1gIN7bRdm9euPk0q+XUdvc8lJL29HxUKKCKuA64DmDx5ckyfPr3DL9AbzJo1C8dZOo6zdPpCjOA4e5tyJqEmdt0OvBFYX0TbRqApIkJSUe8j6WySa0NHRcTm3QnczMx6RjknJswFKiVNyik7FJhToO2ctK5QuznAIbmz5UgmL7S+j6S/BS4CPhARC0sQu5mZ9YCyJaGIaAZuBy6VVCfpSOBE4CcFmt8EnCtpvKRxwHnAjWndLGA78EVJNWmPB+BBAEknA/8OfCgiXuup72NmZruv3FO0zwIGAcuBW4AzI2KOpKPSYbYW1wL/B7wIzAbuTsuIiC0k07c/B7wF/C0wIy0H+DdgOPC0pKb0scukBDMzy145rwkREatJEkh++SMkEw5aXgdwYfoo9D7PAdPaqdunJMGamVmP87I9ZmaWGSchMzPLjJOQmZllxknIzMwy4yRkZmaZcRIyM7PMOAmZmVlmnITMzCwzTkJmZpYZJyEzM8uMk5CZmWXGScjMzDLjJGRmZplxEjIzs8w4CZmZWWachMzMLDNOQmZmlhknITMzy4yTkJmZZcZJyMzMMuMkZGZmmXESMjOzzDgJmZlZZpyEzMwsM05CZmaWGSchMzPLjJOQmZllxknIzMwy4yRkZmaZcRIyM7PMOAmZmVlmnITMzCwzTkJmZpYZJyEzM8tMWZOQpGGS7pDULGmBpE+3006SLpO0Kn1cJkk59VMlPSNpQ/pzarHHmplZ71HuntBVwBZgNHAycLWkKQXanQHMAA4FDgFOAL4AIKkamAncDAwFfgzMTMs7PNbMzHqXsiUhSXXAScBXI6IpIh4F7gQ+W6D5KcDlEbEwIhYBlwOnpnXTgUrgiojYHBFXAgKOLeJYMzPrRSrL+FlvB7ZFxNycsj8ARxdoOyWty203JafuhYiInPoX0vL7Ojm2DUlnkPScADZLml3cV8nUCGBl1kEUwXGWVl+Isy/ECI6z1CbvzsHlTEL1wLq8srVAQztt1+a1q0+v7eTX5b9Pu8fmJS4i4jrgOgBJv4+Iw4v/OtlwnKXlOEunL8QIjrPUJP1+d44v5zWhJqAxr6wRWF9E20agKU0inb1PR8eamVkvUs4kNBeolDQpp+xQYE6BtnPSukLt5gCH5M14OySvvr1jzcysFylbEoqIZuB24FJJdZKOBE4EflKg+U3AuZLGSxoHnAfcmNbNArYDX5RUI+nstPzBIo7tyHVd/1aZcJyl5ThLpy/ECI6z1HYrTpVzlErSMOAG4EPAKuCiiPiZpKOAeyOiPm0n4DLgtPTQ64EvtQypSTosLTsIeBn4fEQ8V8yxZmbWe5Q1CZmZmeXysj1mZpYZJyEzM8tMv0pCpVq7rpfEeImkrZKach77liPG9PPPlvR7SZsl3dhJ23MkLZW0TtINkmrKFGbRcUo6VdL2vPM5vUwx1kj6Yfrfe72k5yUd30H7TM5nV+LM8nymn3+zpCXpOZor6bQO2mb5+1lUnFmfzzSGSZI2Sbq5nfru/d2MiH7zAG4Bfk5yQ+v7SG5knVKg3ReAV4AJwHjgJeDvelmMlwA3Z3guP06yRt/VwI0dtPswsIxk1YqhJLMbv9UL4zwVeDSjc1mX/vecSPIPw78gue9tYm86n12MM7PzmX7+FKAmfX4AsBSY1pvOZxfjzPR8pjHcDzzS3t+d7v7d7Dc9IZVu7breEmOmIuL2iPglySzHjpwC/DAi5kTEGuBfKeNafl2IMzMR0RwRl0TE/IjYERF3Aa8D0wo0z+x8djHOTKXnZ3PLy/SxX4GmWf9+FhtnpiR9EngL+E0Hzbr1d7PfJCHaX7uu0LpyRa8/V2JdiRHgBEmrJc2RdGbPh9cthc7laEnDM4qnI4dJWpkOi3xVUjmXtWolaTTJ70Khm6x7zfnsJE7I+HxK+oGkDcAfgSXAPQWaZX4+i4wTMjqfkhqBS4FzO2narb+b/SkJlWrtup7UlRhvBQ4ERgKnA/8i6VM9G163FDqXUPg7Zelh4GBgFElv9FPABeUOQlIV8FPgxxHxxwJNesX5LCLOzM9nRJxFcl6OIrlRfnOBZpmfzyLjzPJ8/itJb3FhJ+269XezPyWhUq1d15OKjjEiXoqIxRGxPSJ+B3wf+Ksejq87Cp1LKHzeMxMRr0XE6+kw04sk//Ir6/mUVEGygsgW4Ox2mmV+PouJszeczzSO7emw9gSg0GhB5ucTOo8zq/OpZMPQDwLfK6J5t/5u9qckVKq163pSV2LMFyT7KvU2hc7lsojotddoUmU9n+m/Fn9IsuHjSRGxtZ2mmZ7PLsSZL+vfz0oKX2vpbb+f7cWZr1znczrJRJQ3JC0FzgdOkvRsgbbd+7uZ5WyLcj+A/yGZfVYHHEn7M8/+jmQ5oPHAuPRElmt2XLExnkgym0fAu4BFwCllPJeVwEDgmyT/Kh4IVBZo9xGSGT8HAUNI1vgr5+yjYuM8HhidPj8AmA18rYxxXgM8AdR30i7r81lsnJmdT5Ihq0+SDA8NIJkB1wz8ZW86n12MM5PzCdQCY3Ie3wFuA0YWaNutv5tl+cXtLQ9gGPDL9D/0G8Cn0/KjSLqNLe0EfBtYnT6+TbrEUS+K8RaSGV9NJBc0v1jmc3kJO2fztDwuAd6WxvS2nLbnkkyDXQf8iHRKam+KM/2fa1l63l8jGe6oKlOMe6dxbUpjanmc3JvOZ1fizPh8jgQeIpnNtQ54ETg9retN57PoOLM8n3kxX0I6RbvA36Ru/d302nFmZpaZ/nRNyMzMehknITMzy4yTkJmZZcZJyMzMMuMkZGZmmXESMjOzzDgJme2hJIWk3riUk1krJyGzHiDpxjQJ5D+eyDo2s94kk6XqzfqJB9h1L6gtWQRi1lu5J2TWczZHxNK8x2poHSo7W9LdkjakW2Z/JvdgSe+Q9ICkjem+UTdKGpzX5hRJLyrZvnyZpB/nxTBM0v8q2S7+tfzPMMuak5BZdr5OsnPuVOA64CZJh0PrLru/Ilk/7F3Ax4D3Aje0HCzpC8C1JOudHQJ8lGRhy1z/AswkWdH458ANkt7Wc1/JrGu8dpxZD5B0I/AZksU+c10VEV+SFMD1EXF6zjEPAEsj4jOSTidZtHJCRKxP66cDvwUmRcQ8SQtJFpO8qJ0YgmRF6IvT15UkC2WeERE3l/DrmnWbrwmZ9ZyHgTPyyt7Kef54Xt3jwJ+nzw8EXmhJQKnfATuAgyStI1ky/zedxPBCy5OI2CZpBckWAma9gpOQWc/ZEBHzeuB9uzJ8kb/pXOBheOtF/Mtolp13F3j9cvr8ZeAdkhpy6t9L8v/syxGxnGQjww/0eJRmPcg9IbOeUyNpTF7Z9ohYkT7/uKSngVnAX5EklCPSup+STFy4SdK/kOyiey1we07v6hvA9yQtA+4m2QXzAxFxeU99IbNScxIy6zkfBJbklS0CJqTPLwFOAq4EVgB/ExFPA0TEBkkfBq4AniKZ4DAT+MeWN4qIqyVtAc4DLiPZzfKenvoyZj3Bs+PMMpDOXPvriLgt61jMsuRrQmZmlhknITMzy4yH48zMLDPuCZmZWWachMzMLDNOQmZmlhknITMzy4yTkJmZZeb/A7oK0OXF48E0AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plotting the learning rate by epoch for power scheduling\n",
    "learning_rate = 0.01\n",
    "decay = 1e-4\n",
    "batch_size = 32\n",
    "n_steps_per_epoch = len(X_train) // batch_size\n",
    "epochs = np.arange(n_epochs)\n",
    "lrs = learning_rate / (1 + decay * epochs * n_steps_per_epoch)\n",
    "\n",
    "plt.plot(epochs, lrs,  \"o-\")\n",
    "plt.axis([0, n_epochs - 1, 0, 0.01])\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Learning Rate\")\n",
    "plt.title(\"Power Scheduling\", fontsize=14)\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exponential Scheduling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```lr = lr0 * 0.1**(epoch / s)```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# defining exponential decay\n",
    "def exponential_decay_fn(epoch):\n",
    "    return 0.01 * 0.1**(epoch / 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# defining exponential decay without hardcoding\n",
    "def exponential_decay(lr0, s):\n",
    "    def exponential_decay_fn(epoch):\n",
    "        return lr0 * 0.1**(epoch / s)\n",
    "    return exponential_decay_fn\n",
    "\n",
    "exponential_decay_fn = exponential_decay(lr0=0.01, s=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# defining the model\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.Dense(300, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n",
    "    keras.layers.Dense(100, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n",
    "    keras.layers.Dense(10, activation=\"softmax\")\n",
    "])\n",
    "\n",
    "# compiling the model\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"nadam\", metrics=[\"accuracy\"])\n",
    "n_epochs = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 25000 samples, validate on 5000 samples\n",
      "Epoch 1/5\n",
      "25000/25000 [==============================] - 4s 145us/sample - loss: 0.9703 - accuracy: 0.7344 - val_loss: 0.8451 - val_accuracy: 0.7642\n",
      "Epoch 2/5\n",
      "25000/25000 [==============================] - 3s 114us/sample - loss: 0.6797 - accuracy: 0.7863 - val_loss: 1.2915 - val_accuracy: 0.6454\n",
      "Epoch 3/5\n",
      "25000/25000 [==============================] - 3s 117us/sample - loss: 0.6090 - accuracy: 0.8094 - val_loss: 1.7539 - val_accuracy: 0.6522\n",
      "Epoch 4/5\n",
      "25000/25000 [==============================] - 3s 115us/sample - loss: 0.5586 - accuracy: 0.8259 - val_loss: 1.5897 - val_accuracy: 0.7072\n",
      "Epoch 5/5\n",
      "25000/25000 [==============================] - 3s 118us/sample - loss: 0.4868 - accuracy: 0.8445 - val_loss: 0.5465 - val_accuracy: 0.8442\n"
     ]
    }
   ],
   "source": [
    "# LearningRateScheduler update the optimizer’s learning_rate\n",
    "# attribute at the beginning of each epoch\n",
    "lr_scheduler = keras.callbacks.LearningRateScheduler(exponential_decay_fn)\n",
    "\n",
    "# training the model\n",
    "history = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n",
    "                    validation_data=(X_valid_scaled, y_valid),\n",
    "                    callbacks=[lr_scheduler])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAEeCAYAAAAjNKpiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deZxU1Zn/8c/TK02vNDuN7AqKsghqxDiCxi2JEaOTSXSMTsZl4jiZcf3pJE6MmSxoTNSJo5AYl5g4EqNB4xYVcV8iGEFEUREUBFGWhmZfnt8f53ZTltXd1U1V3Wr6+3697qur7jn31nNvL0+fe889x9wdERGROBTEHYCIiHReSkIiIhIbJSEREYmNkpCIiMRGSUhERGKjJCQiIrFREhLJATM708wa2rjNLDP7ZbZiij5jsZldnIX9nmJmbXr+I/kcteecScejJCRZZWa3mZmnWF6MO7ZsiY7vlKTVdwNDsvBZZ5nZq2bWYGb1ZjbXzP47058Tk6ycM8kvRXEHIJ3C48DpSeu2xhFIXNx9E7Apk/s0s28BNwAXAE8AxcD+wKGZ/Jy4ZOOcSf5RS0hyYYu7r0haVgOY2RFmts3MJjZWNrNzzWydmQ2J3s8ys5vN7HozWxMt15hZQcI23czs9qhsk5k9bmYjE8rPjFoLR5nZ62a2wcyeNLPBiYGa2QlmNtvMNpvZe2b2IzMrSShfbGbfM7OpUYxLzeySxPLo5R+iFtHixM9PqDfUzGaY2Yooljlm9uU2ntevAPe6+1R3f8fdF7j7H9z9wqRj+qKZvRSdl1Vm9oCZdUmo0qW544m2rzazaWa20szWm9lTZjY+qc43zWyJmW00sz8DvZPKrzSz15PWtXi5LcU5uzL63n3dzN6NYvmTmfVIqFNkZr9I+Dn5hZndZGazWj+dEgclIYmVuz8FXAP8NkokI4CfA//m7osSqp5G+Hk9FDgXOAf4j4Ty24BDgBOBg4GNwCNmVpZQpxS4HPhWtJ8a4ObGQjM7Fvgd8EtgZFTvFODHSWFfAMwDDgSmAFebWWPr46Do69lA34T3ySqAh4GjgdHAH4F7o+NP1wrg4MZknYqZHQfcDzwGjAMmAU/x6d/9Zo/HzAx4EKgDvgyMBZ4GZppZ36jOIYTzPw0YAzwAXNWG42iLQcA/ACcBx0Tx/Cih/GLgTOAs4HOE4zw1S7FIJri7Fi1ZWwh/nLYDDUnLlIQ6xcBfgXuBOcDdSfuYBSwELGHd94Cl0eu9AQf+LqG8GqgHzorenxnVGZ5Q5zRgS+N+CX9cr0j67MlRvI11FgN3JdV5G/hewnsHTkmqcybQ0Mq5ejFpP7OAX7ZQvy/wQvR5bwN3At8EihPqPAf8Xwv7aPF4gCOj4y9LqvM34NLo9e+Bx5LKfx3+vDS9vxJ4vaVzksb7K4HNQHXCuu8C7yS8Xw5clvDegLeAWXH/LmhJvaglJLnwNOE/5MTlmsZCd99G+G/1y0AvQksn2Yse/VWJvADUmVkVsC+wM1rXuM96wn/3+yVss8Xd30p4/yFQAnSL3o8DvhtdtmuILgX9HigH+iRsNzcptg+juNNmZuVmdrWZvRFdNmoAxgMD0t2Huy9390OBA4DrCH9wpwIvm1nXqNpYwv2ilrR0POOArsDHSedlf2BoVGdfEs59JPl9piyJvrefidXMqgnfp5cbC6OfmZeRvKWOCZILG939nVbqNF46qQF6Amsz9NmJiWt7M2UFCV9/APwhxX4+Tni9LcV+2voP3c+A4wiXj94mXD68g5AU28TdXwdeB240s88DzwBfI7RC09HS8RQAHwGHp9huXRvC3ElIkomK27B9o0yce8kj+uZJ7KLOAb8E/pVw7+JOM0v+B+mQ6P5Eo88BH7r7OmABu+4XNe6zitBCeKMNocwBRni4yZ+8JCewlmwDClup83ngDnf/o7vPBZayq2WxOxqPtyL6+ipw1G7sbw6hk8HOFOdkZVRnAeH7kSj5/cdA76Tv4ZjdiOszohbSChLuw0Wf19x9OckDaglJLpSaWZ+kdTvc/WMzKwR+Czzl7lPN7B7CZbTvA1ck1O8HXGdm/0tILpcA/w3g7m+b2QxgqpmdQ2hF/Yjwn/rv2xDnVcCfzWwJMJ3QctofONjdL23DfhYDR5nZU4RLgGtS1FkInBTFvY1wvF1S1GuWmd1EuBw1k5DE+hLulW0E/hJV+xHwgJm9QzgXRrihP9XdN6bxMY8T7ivNMLNLgTcJl7yOAx5392cI3cSfN7PLgXuAiYSOA4lmAbXAf5rZ/0V1kp+lyoTrgUvNbCEhIZ9LOC/Ls/BZkgFqCUkufIHwRyBxeTUq+09gGPDPAO6+CjgDuCy6tNTod4TWxUvAr4BbgF8klP8T4dr//dHXrsBxHp41SYu7Pwp8idCD7OVouQx4P/1DBeCiaB8fsOs4k10IrCRcOnuY0CnhmTZ+zmOEHoHTCUntvmj90e6+EMDdHyIkhOOjWJ6KYtuZzgdE91S+SEh0vyLc5J8ODCckQNz9RcL379uE+0tfJXQiSNzPgqj8nKjO0Xy212Em/IzwT82thHMK4bxszsJnSQY09vgRyVvRMx6vu/v5ccciHY+ZvQo86+7/Fncs8lm6HCciewwzGwgcS2jxFROe1xoVfZU8pCQkInuSnYRnpa4h3G54Azje3V+JNSppli7HiYhIbNQxQUREYqPLcZGamhofNmxY3GG0asOGDZSXl8cdRqsUZ2Z1hDg7QoygODNt9uzZn7h7z/ZuryQU6d27N6+8kv+XjWfNmsXEiRPjDqNVijOzOkKcHSFGUJyZFj1X1266HCciIrFREhIRkdgoCYmISGyUhEREJDZKQiIiEhslIRERiY2SkIiIxEZJSEREYqMkJCIisVESEhGR2CgJiYhIbJSEREQkNkpCIiISGyUhERGJjZKQiIjEJqdJyMxqzew+M9tgZkvM7NRm6pmZTTGzVdEyxcwsoXyamb1lZjvN7MwU219gZivMbJ2Z/cbMSluLbfG6nRz205n86dVlu3WMIiKSvly3hG4EtgK9gdOAm8xsZIp65wCTgdHAKOAE4NyE8teA84A5yRua2bHAZcBRwEBgCPCDdIJbtnYTl987T4lIRCRHcpaEzKwcOBm4wt0b3P1Z4H7g9BTVzwCudfel7r4MuBY4s7HQ3W909yeAzc1se4u7z3f3NcAPE7dtzaZtO7jm0bfSrS4iIrvB3D03H2Q2FnjO3bsmrLsYOMLdT0iqWw8c4+4vRe/HA0+6e2VSvWeBX7v7bQnrXgN+7O53R+97AB8DPdx9VdL25xBaXZT0GTau7xnXNZUdN6iIfbsXsk+3QsqKjHzR0NBARUVF3GG0SnFmVkeIsyPECIoz0yZNmjTb3ce3d/uiTAbTigpgXdK6eqCymbr1SfUqzMy89ayZaluiz/lUEnL3acA0gNK+ezftt6SwgJkf7OSRxdspLDBG96/m0KHdmTC0B+MGdqNLcWErIWRPR5l3XnFmVkeIsyPECIoz3+QyCTUAVUnrqoD1adStAhrSSEDNbUszn/MZZcWF/OSrB3DsyD7MeX8Nz7/7Cc+/u4qbn1rEjU++S0lhAQcOrOHQIT2YMKw7o/vXUFKkToYiIu2RyyS0ECgys73d/e1o3Whgfoq686Oyl1upl0rjttMTtv0o+VJcKnU1ZVxy7HAmj60D4LBhPThsWA8A1m/exiuLdyWl655YyC8eD0nroMG1HDqkOxOGdmf/umoKC/Ln8p2ISD7LWRJy9w1mdi9wlZmdBYwBTgQmpKh+B3ChmT0EOHAR8D+NhWZWQuhUYUCxmXUBtrr7zmjb28zsd8CHwPeA21qLb1BVAc9ddmSz5ZVdipk0oheTRvQCYO3Grby4aDUvRElpyiNvRvWKOGRw9+jyXXeG966kQElJRCSlXLaEIHSr/g2wknB/5tvuPt/MDgcedvfGu3BTCV2r50Xvfx2ta/QX4Ijo9QTCfZ1JwCx3f8TMrgaeBMqAPwLfz/SB1HQt4bj9+3Dc/n0AWLl+86eS0uMLPgKgtryEQ4d053NRUhrSo5yER55ERDq1nCYhd19NeP4nef0zhA4Fje8duDRaUu1nYiuf83Pg57sTa1v1quzCV0b34yuj+wHhmaMX3l3F8+9+wgvvruLBecsB6F1VyoShPTh0SGgt7VXbtaXdiojs0XLdEuo06mrKOGVcf04Z1x93Z8mqjTwfJaWnF37MfdEDsXvVljFhSA8OHRqSUu+qLjFHLiKSO0pCOWBmDOpRzqAe5Zx6yADcnbdXNvD8O+HS3cOvL+fuVz4AYGjP8tBSGtqdzw3pTm15SczRi4hkj5JQDMyMfXpXsk/vSs48bDA7djoLlq9r6nn3xzlL+e2LSwDYt28VE4Z259Ah3Tl4SG3MkYuIZJaSUB4oLDD2r6tm/7pqzvm7oWzbsZO5S+ubOjn89sUl3PLsexRY6MX30uY3OXRId8YP6kbXEn0LRaTj0l+wPFRcWMC4gd0YN7Ab5x+5N5u37eDV99fywruf8PCcRfzq6UXcNOtdiguNsXt1a+p5N3ZADaVF8Y3mICLSVkpCHUCX4sKmjgsHliznoEM/zytL1jT1vPvlzLe54Ym3KS0q4KBBtU11R9VVU1So0RxEJH8pCXVA5aVFHLFPT47YpycA9Zu28fJ7q5uSUuMo4BWlRRw8uJYJUSeH/fpW6cFZEckrSkJ7gOqyYo7erzdH79cbgFUNW3hx0a6kNPPNlQDUdC3mc4O7M2FY6OgwrFeFHpwVkVgpCe2BuleU8qVRffnSqL4ArKjfzAuLPuH5d1bx/LureGT+CgB6VJSGnnfRPaUBtV2VlEQkp5SEOoE+1V04aWx/ThrbH4APVm9s6g7+wruruP+1D4HwgO2hUXfwCcO607e6LM6wRaQTUBLqhPaq7co/1A7gHw4KD86++/GGT415d8/spQAM7lHe1Er63JDu9KgojTlyEdnTKAl1cmbGsF4VDOtVwemHDmLnTufNFeub7ifd/7cP+f1L7wMwvHdlU1I6ZHB3qrsWxxy9iHR0SkLyKQUFxn79qtivXxVnHT6E7Tt28vqH65qS0v/99X1ue34xZrB/v+rQShranYMH1VJeWsSfXl3GNY++xbK1m6h7cean5mcSEUmmJCQtKiosYMxeNYzZq4bzJg5jy/YdvPZBfdM9pVufW8zUpxdRVGD071bG0jWb2L4zTIC7bO0mLr83zMahRCQiqSgJSZuUFhVy8OBaDh5cy398ATZt3cHs6MHZXz2zqCkBNdq0bQdXPTCfCUO700sjhItIEiUh2S1lJYV8fu8efH7vHtw0692UdVZv3MbBP36CvWrLGD+wlgMHdmP8wG7s07tSU6GLdHJKQpIx/WrKWLZ202fW96wo5dwjhjB7yRqefeeTprmUKkuLGDOghvEDaxk3sBtjBtRQUaofSZHORL/xkjGXHDucy++dx6ZtO5rWlRUX8t0v7cvksXWcdTi4Ox+s3sTs91fzyuI1zF6yhuueWIg7FFiYuqJx8NZxA7tRV1OmB2hF9mBKQpIxjZ0PmnrH1ZR9pnecmTGge1cGdO/a9PDsus3bePX9tcxesobZS1Zzz+yl3PFCmE+pT1WXpoQ0flA39u1bRbEGZRXZYygJSUZNHlvH5LF1zJo1i4kTJ6a1TVWX4k8NyLp9x07eXLE+SkpheXDeciC0rEbvVd10Ce/AAd30vJJIB6YkJHmnqLCgaZK/MyYMAmB5/SZmL1nDK4vXMOf9Ndz01LvsiHri7d2rgvGDujEuSkyDumsMPJGOQklIOoS+1WV8eVQZXx7VD4CNW7fztw/WMmfJGl5ZsoYH5y7nrpc/AKB7eUlTD7zxg7oxsl81XYo12Z9IPlISkg6pa0kRE4b2YMLQHgDs3Om883FDU2tp9pLVPPbGRwCUFBZwQP/qT3V40Dh4IvlBSUj2CAUFxj69K9mndyXfOHgAAJ80bPnUfaXbnlvMtKcXATCoe1fGDaylcss2+n20nmE9KzThn0gMlIRkj9WjopRjR/bh2JF9ANi8bQfzP6xv6ho+662VrNqwldvmP01Vl6KmS3jjBtYyeq9qupbo10Mk2/RbJp1Gl+LCqPNCLRCeWbr7oScp6rMPs5eE55ZmvfUxAIUFxsh+VRw4oFvU6aGb5lcSyQIlIem0zIw+5QVMHNefU8aFZ5bqN25jzvuhpfTKktVNo4ZDmPQv8b7SiD6VFOmZJZHdoiQkkqC6azGTRvRi0oheAGzbsZMFy9eFS3jvr+Hl91Y3zURbXlLImAE1TV3Dxw6ooaqLnlkSaQslIZEWFBcWMKp/DaP61/AtBuPufFi/mVcWr27q8PDLmW+z08EsTPzXOLrDuAG17FWrYYdEWqIkJNIGZkZdTRl1Y+o4cUwYjqhhy3b+Fg079MqS1cz424f8LpqNtmdlKeMS7iuN7FdNSZEu4Yk0ymkSMrNa4BbgGOAT4HJ3/32Kegb8FDgrWvVr4DJ396h8TLSffYEFwD+7+9+islLgeuAkoBh4DvgXd1+WxUOTTqyitKhpOguAHTudhR+t55Ula6KHaVfzyPwVAJQWFTC6fw3jBnVj3ICQmLqVl8QZvkisct0SuhHYCvQGxgAPmtlr7j4/qd45wGRgNODAY8B7wM1mVgLMAK4D/hc4F5hhZnu7+1bg34FDgVFAPTAN+B/gq1k+NhEg9Kzbt28V+/at4vTPDQRg5brNUUspXML79TOLuGlHGHZoSM/yMLpDNNfS0J7lTZfwNF267OlyloTMrBw4Gdjf3RuAZ83sfuB04LKk6mcA17r70mjba4GzgZuBiVHc10UtoxvM7GLgSOARYDDwqLt/FG17N/DzLB+eSIt6VXXh+AP6cvwBfYHwzNLcpfW8smQ1c5as4bE3PmL6K0sB6Na1mAMHdKOspIC/vLGSrdt3ApouXfZMFl3hyv4HmY0FnnP3rgnrLgaOcPcTkurWA8e4+0vR+/HAk+5eaWYXRGXHJ9T/c1R+bVT3euDvgbWES3kr3f0/UsR0DqHVRc+ePcdNnz49swedBQ0NDVRUVMQdRqsUZ9u4O8s3OO+s3cHba3by9todrNiQ+nezuhSuPaIrRXk2wkO+nMvWKM7MmjRp0mx3H9/e7XN5Oa4CWJe0rh6obKZufVK9iuheUXJZ8n7eBj4AlgE7gHnA+akCcvdphMt1DB8+3NOdeiBObZkiIU6Kc/cNvuxBUqWh+i1w3szN7Ne3itH9qzmgfw2j+lcztGdFrNOl5/O5TKQ480vaScjMehMunQ0FrnD3T8zsMOBDd38vjV00AFVJ66qA9WnUrQIa3N3NrLX93AiUAt2BDcClwMPAIWnEKJI3mpsuvVvXYk4Z15+5S+u5Z/ZSbo8mAOxaUsjIflVRl/JqDqirZlD3co2JJ3ktrSRkZuOAJwidA0YC1xB6tx0N7AOcmsZuFgJFUQeCt6N1o4HkTglE60YDL6eoNx+4yMzMd11LHEVIPhA6PHzX3VdHsf8PcJWZ9XD3T9I5XpF80Nx06d8/YWTTPaGdO51FnzQwd2k9c5fWM29ZPb97aQm3PBvuI1V2KeKAumoO6F/NqLqQnPp307NLkj/SbQn9DLje3b9vZoktl0eBf0pnB+6+wczuJSSEswjJ4kRgQorqdwAXmtlDhN5xFxF6uAHMIlxm+46Z3UzosAAwM/r6V+CbZjYL2AicR2itKQFJh5LOdOkFBcawXpUM61XJVw8MQw9t37GTt1c2MG9pPXOXrWXe0npufXYxW3eExNSta3G4hNeYnPpX06eqixKTxCLdJDQO+OcU65cTulun6zzgN8BKYBXwbXefb2aHAw+7e+NduKnAEML9HAidC6YCuPtWM5scrfsp4TmhyVH3bICLgRsI94ZKgNcJzwyJdDjtmS69qLCgqYv41w7aC4At23ewcEVDU1Kau7T+U7PT9qws/VRSOqCuhp6VmnNJsi/dJLQJ6JZi/QhCQklLdIlscor1zxA6HDS+d8K9nEub2c+rhMSYqmwVcFq6MYl0BqVFhRzQPySZxrujm7ft4I3l65qS0tyla5n51koaL3L3q+4SJaWacEmvrloP1krGpZuEZgDfN7O/j967mQ0CpgB/zEJcIpJlXYoLOXBANw4csOv/yw1btjP/w3XMXbqWectCcnp0/kdN5QNqu0b3l0JCO6CumkoN2iq7Id0kdDHwEPAx0BV4lnAZ7jnge9kJTURyrby0iIMH13Lw4NqmdfWbtjF/WT1zl4XW0msfrOXBucubyof0LGdUXTVlm7dRvng1I/tVaUJASVtaPynuvg74vJkdCRwIFABz3P3xbAYnIvGrLitmwrAeTBjWo2nd6g1bQ0vpg7XMXVbPi4tWs2LdVu568wUKDPbuVdl0f2lU/xpG9KmkS3FhjEch+SrdLtrfBO5295ns6oVGNI7b1939jizFJyJ5qLa8hCP26ckR+/RsWvenR2ZSMWAkc5fVM2/pWp58cyX3zA5DERUVGMP7VDZ1ehjVv5rhfSop1qSAnV66beZbCeOyJXdCqIzKlIREOrmaLgVM3K83X9gvdJh1d5bXb2bu0rVNzzA9NG8Fd738AQAlRaEX36i6XS2moT3LNVttJ5NuEjJIOYLIAD47hI6ICGZGv5oy+tWUcdz+YeBWd+eD1Zt4ranjw1rue3UZv30xjPpQVhxGfTigfzWj+9dwQP9qBmvUhz1ai0nIzOYRko8DT5nZ9oTiQmAgocOCiEirzIwB3bsyoHtXThjdD2gc9WED85ZFLaal9dz18vvc+txiIMzXtH/druGIRtXVaMbaPUhrLaF7oq/7Aw8SxnRrtBVYjLpoi8huCKM+VDCsVwUnjd016sM7Hzc0JaW5y+q57bldoz5UlxVHl/B23WPqW61RHzqiFpOQu/8AwMwWEzombM5FUCLSuRUVFjCiTxUj+lTxtfFh1Iet23ey8KP10f2l0Gqa+tQitkejPvSoKOGAuupdA7j2r6ZXZZemfWqCwPyUbhft27MdiIhIS0qKCti/rpr966oJt6PDqA8Llq9rerB27tK1PLXwY6K8RJ+qLozqX01RgfH4gpVNLSlNEJg/0u2iXQJ8F/gG4bv/qUek3V0PAIhIznUpLmTsgG6MTRr14Y3l65qS0ryl9Sz6ZMNntt20bQdXPTCf0XvVMLC2qzo/xCTd3nE/BP4B+AnwC+ASYBDwdeCKrEQmItIO5aVFHDSoloMG7Rr1obkJAldv3Makn82irLiQ4X0qo4Ffw9cRfSo1JFEOpJuEvgb8i7s/YmY/A2a4+7tmtoAwp9DUrEUoIrKbmpsgsGdlKZccM5w3lq/jzRXreGjecu56+f2m8v7dykJiihLUiL5VajVlWLpJqDfwRvS6AaiJXj9CGMRURCRvNTdB4He/uO+n7gm5OyvWbWbB8nUsWL4++rqOJxZ81HSfqWtJaDWN6FPFfn0rGaFW025JNwm9D/SLvr4DHAvMBg4lTPMgIpK30pkgEMJzTH2ry+hbXcaRI3ZNlbZ52w4WfrSeN5evV6spw9JNQvcBRwEvAtcDd5nZ2UAdYapvEZG81p4JAht1KS6Mun7XNK1Tqykz0u2ifXnC63vM7APgMGChu/85W8GJiOSrjLSaElpOAzppq6ldk364+0vASwBmVu7un+3/KCLSCWWq1bRpzQ7Gbd62x7ea2j3zlJl1Af6N0F27V8YiEhHZw7S31fTjl/6yx7eaWhvAtAT4PnAMsA242t3/FM0v9FPCwKa/yHqUIiJ7oOZaTcvrNzP9L89R3HPQHn+vqbWW0JXAvwKPEe4B/cHMfkXopHA58Ht335bVCEVEOpHGKTDG9Cpi4sRhTevTude0V20ZI/p0rFZTa0noa8CZ7n6fmY0GXgW6ASPdfXvLm4qISKa01Gp6c0X695r27VvF8DxqNbWWhPYC/grg7q+Z2VZgihKQiEj8EicObGsPvXxpNbWWhIqBLQnvt6GZVEVE8louWk2NU2OU9Bk2bndiTad33E/MbGP0ugS40sw+lYjc/Tu7E4SIiGRXa62mxO7jrbWalq7ZxM8fe4tN23budlytJaGngaEJ75+ncSKPXVINTisiIh1Ae1tNmdLazKoTM/txIiKS71prNX3ll89l7LMKMrYnERHZozW2mupqyjK2TyUhERFpk0uOHU5ZcWYm1M5pEjKzWjO7z8w2mNkSMzu1mXpmZlPMbFW0TDEzSygfY2azzWxj9HVM0vYHmtnTZtZgZh+Z2b9n+9hERDqLyWPr+MlXD8hIiyjXLaEbga2ESfJOA24ys5Ep6p0DTAZGA6OAE4BzoWkooRnAnYQHZ28HZkTrMbMehMn2pgLdgWHAX7J3SCIinc/ksXU8d9mRbF3xzuzd2U/OkpCZlQMnA1e4e4O7PwvcD5yeovoZwLXuvtTdlwHXAmdGZRMJHSquc/ct7n4DYMCRUfmFwKPu/ruofL27L8jagYmISLuZe+v97cwsuVt2Iwc2u/vHaexjLPCcu3dNWHcxcIS7n5BUtx44JpoyAjMbDzzp7pVmdkFUdnxC/T9H5dea2UxgHnAQoRX0EvCv7v4+SczsHEKri549e46bPn16a4cRu4aGBioqKuIOo1WKM7M6QpwdIUZQnJk2adKk2e4+vr3bpzuVw2JaeB7IzNYBtwKXtjCkTwWwLmldPVDZTN36pHoV0X2h5LLk/fQHDgSOJiSjq4G7CAOwfoq7TwOmAQwfPtzbOttiHNozK2QcFGdmdYQ4O0KMoDjzTbpJ6BuEP+Y3E01mBxxCaEVcCdQA3wPWE6Z+SKUBqEpaVxVt01rdKqDB3d3MWtvPJuA+d/8rgJn9APjEzKrdXUMOiYjkkXST0LeBC9z93oR1M83sLeDf3f0IM1sJ/IDmk9BCoMjM9nb3t6N1o4H5KerOj8peTlFvPnCRmZnvupY4itDpAWAun261aUQHEZE8lW7HhEMIl7aSvU649wLwAuFSWErRFOD3AleZWbmZHQacCPw2RfU7gAvNrM7M+gEXAbdFZbOAHcB3zKzUzM6P1s+Mvt4KnBR14y4GrgCeVStIRCT/pJuElhDdwE9yNtB4w78nsLqV/ZwHlAErCfdpvu3u883s8OgyW6OpwAOExPc68GC0DnffSui+/U1gLfAtYHK0HqsLmowAABI9SURBVHefCfxntM1KQueElM8jiYhIvNK9HHcR8Ecz+yLR/ELAeMLgpidH7w8CWuxe5u6rCQkkef0zhA4Hje8duDRaUu3nVaDZ4cPd/SbgppZiERGR+KWVhNz9QTPbm9CSGR6tvh+4ubHrs7v/b3ZCFBGRPVW6LSHc/QPg8izGIiIinUzaScjMugJjgF4k3UtK6jUnIiKSlrSSkJl9gdCRoHuKYgcyM5yqiIh0Kun2jrue0Nusv7sXJC1KQCIi0i7pXo4bBHzF3T/MYiwiItLJpNsSeo5dveJEREQyIt2W0M3Az6LRC+YB2xIL3X1OpgMTEZE9X7pJ6J7o67QUZeqYICIi7ZJuEhqc1ShERKRTSnfEhCXZDkRERDqfZpOQmX0VeMDdt0Wvm6WHVUVEpD1aagndA/QhjER9Twv1dE9IRETapdkk5O4FqV6LiIhkipKLiIjEpi0DmPYH/o7UA5j+PMNxiYhIJ5DuAKanAb8BtgMfE+4DNXJASUhERNos3ZbQVcC1wBXuviOL8YiISCeS7j2h3sCvlYBERCST0k1CDwGHZDMQERHpfNK9HPcYMMXMRpJ6AFM9rCoiIm2WbhKaGn39zxRlelhVRETaJd2x4/Q8kYiIZFyrycXMis3sJTPTpHYiIpJRrSYhd99GmMrBW6srIiLSFuleZrsdODubgYiISOeTbseEcuA0MzsamA1sSCx09+9kOjAREdnzpZuE9gXmRK+HJJXpMp2IiLRLur3jJmU7EBER6XzU9VpERGKTdhIys0lmNs3MHjGzmYlLG/ZRa2b3mdkGM1tiZqc2U8/MbIqZrYqWKWZmCeVjzGy2mW2Mvo5JsY8SM1tgZkvTjU9ERHIrrSRkZmcCDwOVwETCdA7dgAOBN9rweTcCWwkDop4G3BQNBZTsHGAyMBoYBZwAnBvFUgLMAO6MYrgdmBGtT3RJFKeIiOSpdFtCFwPnu/s3COPGXe7uYwmJoCGdHZhZOXAyYTqIBnd/FrgfOD1F9TOAa919qbsvI0wjcWZUNpFwL+s6d9/i7jcABhyZ8FmDgX8EfpLm8YmISAzMvfXObWa2EdjP3Reb2SfAke4+18xGALPcvU8a+xgLPOfuXRPWXQwc4e4nJNWtB45x95ei9+OBJ9290swuiMqOT6j/56j82oT3twBrgDvdvX8zMZ1DaHXRs2fPcdOnT2/1XMStoaGBioqKuMNoleLMrI4QZ0eIERRnpk2aNGm2u49v7/bpdtFeRbgUB7AM2B+YC3QHytLcRwWwLmldfcJ+k+vWJ9WriO4LJZd9aj9mdhJQ6O73mdnElgJy92nANIDhw4f7xIktVs8Ls2bNQnFmjuLMnI4QIyjOfJNuEnoGOIYwjcN04IbowdWjCNM8pKMBqEpaVwWsT6NuFdDg7m5mze4nuuR3NfDFNGMSEZEYpXtP6Hzgruj1T4BrCK2g6cBZae5jIVBkZnsnrBsNzE9Rd35UlqrefGBUYm85QueF+cDewCDgGTNbAdwL9DWzFWY2KM04RUQkR9J9WHV1wuudwJS2fpC7bzCze4GrzOwsYAxwIjAhRfU7gAvN7CHCiAwXAf8Tlc0CdgDfMbOb2TWm3UxgJ7BXwn4mAL8k9OJTTzkRkTzTlueEepvZxWZ2k5n1iNYdFvVES9d5hHtIKwktq2+7+3wzOzy6zNZoKvAA4fLf68CD0TrcfSuh+/Y3gbXAt4DJ7r7V3be7+4rGBVgN7Ize72hDnCIikgNptYTMbBzwBPAeMJJwOe4T4GhgHyDlQ6fJohbV5BTrnyF0OGh878Cl0ZJqP68C49L4vFlAyp5xIiISv3RbQj8Dro+eDdqSsP5R4LCMRyUiIp1CukloHGFkgmTLCaMfiIiItFm6SWgTYYicZCMI93dERETaLN0kNAP4vpmVRu896vI8BfhjFuISEZFOoC1jx9USujl3BZ4F3iGMVPC97IQmIiJ7unSfE1oHfN7MjiQ8c1MAzHH3x7MZnIiI7NnSHbYHAHefSXgoFAAzGwhc4+5fy3RgIiKy59vdmVVrCNMziIiItJmm9xYRkdgoCYmISGyUhEREJDYtdkwws/tb2T55Xh8REZG0tdY7blUa5e9lKBYREelkWkxC7v5PuQpEREQ6H90TEhGR2CgJiYhIbJSEREQkNkpCIiISGyUhERGJjZKQiIjERklIRERioyQkIiKxURISEZHYKAmJiEhslIRERCQ2SkIiIhIbJSEREYmNkpCIiMRGSUhERGKT0yRkZrVmdp+ZbTCzJWZ2ajP1zMymmNmqaJliZpZQPsbMZpvZxujrmISyS8zsdTNbb2bvmdkluTg2ERFpu1y3hG4EtgK9gdOAm8xsZIp65wCTgdHAKOAE4FwAMysBZgB3At2A24EZ0XoAA74ZlR0HnG9mX8/WAYmISPvlLAmZWTlwMnCFuze4+7PA/cDpKaqfAVzr7kvdfRlwLXBmVDaRMCPsde6+xd1vICSeIwHc/Wp3n+Pu2939LULCOiyLhyYiIu1k7p6bDzIbCzzn7l0T1l0MHOHuJyTVrQeOcfeXovfjgSfdvdLMLojKjk+o/+eo/Nqk/RgwB5jq7jeniOkcQquLnj17jps+fXqGjjZ7GhoaqKioiDuMVinOzOoIcXaEGEFxZtqkSZNmu/v49m5flMlgWlEBrEtaVw9UNlO3PqleRZRUksta2s+VhNberakCcvdpwDSA4cOH+8SJE1s8gHwwa9YsFGfmKM7M6QgxguLMN7lMQg1AVdK6KmB9GnWrgAZ3dzNLaz9mdj7h3tDh7r5ldwIXEZHsyGXHhIVAkZntnbBuNDA/Rd35UVmqevOBUYm95QidF5r2Y2bfAi4DjnL3pRmIXUREsiBnScjdNwD3AleZWbmZHQacCPw2RfU7gAvNrM7M+gEXAbdFZbOAHcB3zKw0avEAzAQws9OAHwNHu/uibB2PiIjsvlx30T4PKANWAncB33b3+WZ2eHSZrdFU4AFgHvA68GC0DnffSui+/U1gLfAtYHK0HuC/ge7AX82sIVo+0ylBRETil8t7Qrj7akICSV7/DKHDQeN7By6NllT7eRUY10zZ4IwEKyIiWadhe0REJDZKQiIiEhslIRERiY2SkIiIxEZJSEREYqMkJCIisVESEhGR2CgJiYhIbJSEREQkNkpCIiISGyUhERGJjZKQiIjERklIRERioyQkIiKxURISEZHYKAmJiEhslIRERCQ2SkIiIhIbJSEREYmNkpCIiMRGSUhERGKjJCQiIrFREhIRkdgoCYmISGyUhEREJDZKQiIiEhslIRERiY2SkIiIxEZJSEREYqMkJCIisclpEjKzWjO7z8w2mNkSMzu1mXpmZlPMbFW0TDEzSygfY2azzWxj9HVMutuKiEj+yHVL6EZgK9AbOA24ycxGpqh3DjAZGA2MAk4AzgUwsxJgBnAn0A24HZgRrW9xWxERyS85S0JmVg6cDFzh7g3u/ixwP3B6iupnANe6+1J3XwZcC5wZlU0EioDr3H2Lu98AGHBkGtuKiEgeKcrhZ+0DbHf3hQnrXgOOSFF3ZFSWWG9kQtlcd/eE8rnR+kda2fZTzOwcQssJYIuZvZ7eocSqB/BJ3EGkQXFmVkeIsyPECIoz04bvzsa5TEIVwLqkdfVAZTN165PqVUT3dpLLkvfT7LZJiQt3nwZMAzCzV9x9fPqHEw/FmVmKM3M6QoygODPNzF7Zne1zeU+oAahKWlcFrE+jbhXQECWR1vbT0rYiIpJHcpmEFgJFZrZ3wrrRwPwUdedHZanqzQdGJfV4G5VU3ty2IiKSR3KWhNx9A3AvcJWZlZvZYcCJwG9TVL8DuNDM6sysH3ARcFtUNgvYAXzHzErN7Pxo/cw0tm3JtLYfVSwUZ2YpzszpCDGC4sy03YrTcnmVysxqgd8ARwOrgMvc/fdmdjjwsLtXRPUMmAKcFW36a+D/NV5SM7Ox0br9gAXAP7v7q+lsKyIi+SOnSUhERCSRhu0REZHYKAmJiEhsOlUSytTYdXkS45Vmts3MGhKWIbmIMfr8883sFTPbYma3tVL3AjNbYWbrzOw3ZlaaozDTjtPMzjSzHUnnc2KOYiw1s1ui7/d6M/ubmR3fQv1Yzmdb4ozzfEaff6eZLY/O0UIzO6uFunH+fKYVZ9znM4phbzPbbGZ3NlPevr+b7t5pFuAu4G7CA62fJzzIOjJFvXOBt4D+QB3wBvAveRbjlcCdMZ7LrxLG6LsJuK2FescCHxFGrehG6N340zyM80zg2ZjOZXn0/RxE+Mfwy4Tn3gbl0/lsY5yxnc/o80cCpdHrEcAKYFw+nc82xhnr+Yxi+AvwTHN/d9r7d7PTtIQsc2PX5UuMsXL3e939T4Reji05A7jF3ee7+xrgh+RwLL82xBkbd9/g7le6+2J33+nufwbeA8alqB7b+WxjnLGKzs+WxrfRMjRF1bh/PtONM1Zm9nVgLfBEC9Xa9Xez0yQhmh+7LtW4cmmPP5dhbYkR4AQzW21m883s29kPr11SncveZtY9pnhaMtbMPokui1xhZrkc1qqJmfUm/Cykesg6b85nK3FCzOfTzP7XzDYCbwLLgYdSVIv9fKYZJ8R0Ps2sCrgKuLCVqu36u9mZklCmxq7LprbEOB3YF+gJnA38l5l9I7vhtUuqcwmpjylOTwP7A70IrdFvAJfkOggzKwZ+B9zu7m+mqJIX5zONOGM/n+5+HuG8HE54UH5Limqxn88044zzfP6Q0Fpc2kq9dv3d7ExJKFNj12VT2jG6+xvu/qG773D354HrgVOyHF97pDqXkPq8x8bdF7n7e9FlpnmE//xyej7NrIAwgshW4PxmqsV+PtOJMx/OZxTHjuiydn8g1dWC2M8ntB5nXOfTwoShXwB+kUb1dv3d7ExJKFNj12VTW2JM5oR5lfJNqnP5kbvn7T2aSE7PZ/Tf4i2ECR9PdvdtzVSN9Xy2Ic5kcf98FpH6Xku+/Xw2F2eyXJ3PiYSOKO+b2QrgYuBkM5uTom77/m7G2dsi1wvwf4TeZ+XAYTTf8+xfCMMB1QH9ohOZq95x6cZ4IqE3jwEHA8uAM3J4LouALsBPCP8VdwGKUtQ7jtDjZz+ghjDGXy57H6Ub5/FA7+j1COB14Ps5jPNm4EWgopV6cZ/PdOOM7XwSLll9nXB5qJDQA24D8JV8Op9tjDOW8wl0BfokLD8D7gF6pqjbrr+bOfnBzZcFqAX+FH2j3wdOjdYfTmg2NtYz4GpgdbRcTTTEUR7FeBehx1cD4Ybmd3J8Lq9kV2+exuVKYEAU04CEuhcSusGuA24l6pKaT3FGv1wfRed9EeFyR3GOYhwYxbU5iqlxOS2fzmdb4oz5fPYEniL05loHzAPOjsry6XymHWec5zMp5iuJumin+JvUrr+bGjtORERi05nuCYmISJ5REhIRkdgoCYmISGyUhEREJDZKQiIiEhslIRERiY2SkMgeyszczPJxKCeRJkpCIllgZrdFSSB5eTHu2ETySSxD1Yt0Eo/z2bmgtsYRiEi+UktIJHu2uPuKpGU1NF0qO9/MHjSzjdGU2f+YuLGZHWBmj5vZpmjeqNvMrDqpzhlmNs/C9OUfmdntSTHUmtkfLEwXvyj5M0TipiQkEp8fEGbOHQNMA+4ws/HQNMvuo4Txww4GTgImAL9p3NjMzgWmEsY7GwV8kTCwZaL/AmYQRjS+G/iNmQ3I3iGJtI3GjhPJAjO7DfhHwmCfiW509/9nZg782t3PTtjmcWCFu/+jmZ1NGLSyv7uvj8onAk8Ce7v7O2a2lDCY5GXNxOCEEaEvj94XEQbKPMfd78zg4Yq0m+4JiWTP08A5SevWJrx+IansBeBL0et9gbmNCSjyPLAT2M/M1hGGzH+ilRjmNr5w9+1m9jFhCgGRvKAkJJI9G939nSzsty2XL5InnXN0GV7yiH4YReLzuRTvF0SvFwAHmFllQvkEwu/sAndfSZjI8KisRymSRWoJiWRPqZn1SVq3w90/jl5/1cz+CswCTiEklEOist8ROi7cYWb/RZhFdypwb0Lr6kfAL8zsI+BBwiyYR7n7tdk6IJFMUxISyZ4vAMuT1i0D+kevrwROBm4APgb+yd3/CuDuG83sWOA64GVCB4cZwL837sjdbzKzrcBFwBTCbJYPZetgRLJBveNEYhD1XPt7d78n7lhE4qR7QiIiEhslIRERiY0ux4mISGzUEhIRkdgoCYmISGyUhEREJDZKQiIiEhslIRERic3/Bzo3DmfzJe/XAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plotting the learning rate by epoch for exponential scheduling\n",
    "plt.plot(history.epoch, history.history[\"lr\"], \"o-\")\n",
    "plt.axis([0, n_epochs - 1, 0, 0.011])\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Learning Rate\")\n",
    "plt.title(\"Exponential Scheduling\", fontsize=14)\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The schedule function can take the current learning rate as a second argument:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def exponential_decay_fn(epoch, lr):\n",
    "    return lr * 0.1**(1 / 20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you want to update the learning rate at each iteration rather than at each epoch, you must write your own callback class:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 25000 samples, validate on 5000 samples\n",
      "Epoch 1/5\n",
      "25000/25000 [==============================] - 4s 154us/sample - loss: 0.8872 - accuracy: 0.7535 - val_loss: 1.0513 - val_accuracy: 0.6950\n",
      "Epoch 2/5\n",
      "25000/25000 [==============================] - 3s 134us/sample - loss: 0.6712 - accuracy: 0.7894 - val_loss: 0.9573 - val_accuracy: 0.7528\n",
      "Epoch 3/5\n",
      "25000/25000 [==============================] - 3s 128us/sample - loss: 0.5962 - accuracy: 0.8142 - val_loss: 0.6722 - val_accuracy: 0.7912\n",
      "Epoch 4/5\n",
      "25000/25000 [==============================] - 3s 124us/sample - loss: 0.4969 - accuracy: 0.8376 - val_loss: 0.6724 - val_accuracy: 0.7862\n",
      "Epoch 5/5\n",
      "25000/25000 [==============================] - 3s 124us/sample - loss: 0.4499 - accuracy: 0.8501 - val_loss: 0.6368 - val_accuracy: 0.8200\n"
     ]
    }
   ],
   "source": [
    "K = keras.backend\n",
    "\n",
    "class ExponentialDecay(keras.callbacks.Callback):\n",
    "    def __init__(self, s=40000):\n",
    "        super().__init__()\n",
    "        self.s = s\n",
    "\n",
    "    def on_batch_begin(self, batch, logs=None):\n",
    "        # Note: the `batch` argument is reset at each epoch\n",
    "        lr = K.get_value(self.model.optimizer.lr)\n",
    "        K.set_value(self.model.optimizer.lr, lr * 0.1**(1 / s))\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs=None):\n",
    "        logs = logs or {}\n",
    "        logs['lr'] = K.get_value(self.model.optimizer.lr)\n",
    "\n",
    "# defining the model\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.Dense(300, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n",
    "    keras.layers.Dense(100, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n",
    "    keras.layers.Dense(10, activation=\"softmax\")\n",
    "])\n",
    "lr0 = 0.01\n",
    "optimizer = keras.optimizers.Nadam(lr=lr0) # using Nadam optimizer\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"]) # compiling the model\n",
    "n_epochs = 5\n",
    "\n",
    "s = 20 * len(X_train) // 32 # number of steps in 20 epochs (batch size = 32)\n",
    "exp_decay = ExponentialDecay(s)\n",
    "\n",
    "# training the model\n",
    "history = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n",
    "                    validation_data=(X_valid_scaled, y_valid),\n",
    "                    callbacks=[exp_decay])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_steps = n_epochs * len(X_train) // 32\n",
    "steps = np.arange(n_steps)\n",
    "lrs = lr0 * 0.1**(steps / s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEeCAYAAAC30gOQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3xc1Zn/8c8jyXJRdZF7b3LDBZvebAimZEnIkk0W2CQkIbApm/1RwqaxYdNJwgbIJpQlCSEJ2RACoYaODDa4YIyL3IuMLTfJRbLcy/P741zJ42EkjUFXxf6+X695SXPPuXeeezUzj+45555r7o6IiEhTy2jpAERE5PikBCMiIrFQghERkVgowYiISCyUYEREJBZKMCIiEgslGGnzzOwaM6s5xnVKzOx/4oopeo0yM7s5hu1+3MyO6fqC5GP0fo7ZB2Fm3zGz3zTX66V4fTezj7fA6zZ6nM3sy2b2VHPF1JyUYNowM3sw+uAkP2a2dGxxqeeL4s/A4Bhe61ozm2dmNWZWZWYLzOz7Tf06LSSWY5aKmXUHbgLa9LEzs9vMbFEMm34AmGhm58Sw7RaV1dIByAf2EvCppGX7WyKQluLue4A9TblNM/sccDdwA/Ay0A4YA5zRlK/TUuI4Zg24Fpjt7qvjfiEza+fuB+J+nabk7vvM7GHgq8DrLR1PU9IZTNu3z903JT22AZjZeWZ2wMwm11Y2s+vNrNrMBkfPS8zsXjO7y8y2R4+fmllGwjqdzex3UdkeM3vJzEYnlF8T/Zd/gZktMrNdZvaqmQ1KDNTMLjOzuWa218zWmNkPzCw7obzMzL5tZvdFMa43s68llke//iU6kylLfP2EekPM7Akz2xTF8raZ/cMxHtePAI+5+33uvtLdl7j7X9z9xqR9utTMZkXHZauZPWVmHRKqdKhvf6L1C8zsfjPbYmY7zWyamU1KqvNpM1trZrvN7GmgR1L5e/6zbqxpJsUxuy362/2zma2KYvmbmXVLqJNlZj9PeJ/83MzuMbOSRo7lVcBRTUBpvu+yzez26LjtNrM5ZnZRQvnk6H1wqZnNNrP9wEXUr6eZPRNta62Z/UtSTD82s2XR37LMzH5S+7c0s2uA7wCj7UhLwTVRWUF0HDZG7+0lZvbJpG03+NkAngQ+YmadGjmWbYu769FGH8CDwNON1PkhsA7oDIwAdgGfSSgvAXYCv4jKPwFUATcm1HkCWAqcC5xE+DCsAzpG5dcABwhnU6cCY4F5wPMJ27gIqAY+CwwBpgDLgJ8l1CkDtgJfAYYC/wY4cEZUXhQ9vxboCRQlvH5NwnbGAf8axToU+BbhrG5E0n7/TwPH7V5gOTC4gToXAwcJTT+jov2+GeiU5v4YMB14JjpuQ4HvRcepV1TnNOBwtA/DgeujbXpCHLcBi5JiSz4mjT2/DagBHo/24wxgLXBfQp2vA9uBK4Bi4K7ovVLSwDHqEsV/VtLyEhp/3/0RmEl43w2OjuN+YFxUPjk6nguBqVGdonri8Oi4XR8dx29FcU1KqHMrcBYwELgUeBf4XlTWEfgZ4XPQM3p0jP6GM4DF0fthMHAJ8LF0PxtRvU7AIeCClv5eadLvqJYOQI8P8McLCeZg9MWQ+Lg9oU47YA7wGPA28OekbZQQvkgtYdm3gfXR78OiD+e5CeUF0ZfBtdHza6I6xQl1rgb21W4XeA24Nem1L4/ira1TBvwpqc4K4NsJzx34eFKda0j4sqznWM1M2k4JDSeYXsCb0eutAP4AfBpol1BnBvB/DWyjwf0Bzo/2v2NSnXeAW6LfHwZeTCp/gHgSzF6gIGHZt4CVCc83Al9PeG6EfxJKGjgG46NjOOgY33dDCAmgf9J6fwN+Ff0+Odr2FWl8Vhz436RlLwF/aGCdf03a/1TH+cIozpH1bOMaGvlsJCzfBny+sX1pSw81kbV9rxE+xImPn9YWemiPvgr4B6A74T+4ZDM9eodH3gT6mFk+MJLwAXozYZtVhP8aRyWss8/dlyU83wBkE86cACYC34qa0mqi5pmHgRzCf4O1FiTFtiGKO21mlhM1byyOml5qgElA/3S34e4b3f0MwlnQnYQv0/uA2QnNGBMI/TMNaWh/JhL+c61IOi5jCF+wEI7/m0nbSH7eVNZGf9v3xGpmBYS/0+zawug9M5uGdYx+7k1R1tD77mTCMV+cdGw+zJFjU+utRmJI3H7y87r3sIXRedOjptUa4Oc0/p6ZAGx09yUN1Gnss1FrD0eO13FBnfxt3253X9lIndMJ/W2FhGamHU302olfDgfrKctI+PlfwF9SbKci4ffkDlrn2PsKf0ZorriZcMawG3iI8KE+Ju6+CFgE/NLMziZ0wn6CcPaYjob2JwPYDKQaPVR9DGEeJnwZJ2p3DOvXaopjn6wy+tmZcAaUrozo9U9JEVfy4IRd7y+0I8zsdOD/CO/RGwifkY8Q3ksfVGOfjVpdOPqz0ObpDOY4F3Um/g/wZeBF4A9mlvyPxWlmlvgFdTqwwd2rgSWE90nd6KnoP8yTCO3O6Xqb0AeyMsUj+QPYkANAZiN1zgYecve/uvsCYD3v/a/3/ajd39zo5zzggg+wvbcJHfaHUxyTLVGdJYS/R6Lk5xVAj6S/4fgPENd7RGc2mwhf+ABEr3dKvSsFqwjJclSKsobed/MISbNnimNT/j53I9VxrD3zOAsod/fvufscd18BDEiqv5/3vvfmAb3MbOT7jAkIA1OADoT3xHFDZzBtX3sz65m07JC7V5hZJvB7YJq732dmjxKatr5D6NCs1Ru408x+RUgcXyO6ZsHdV5jZE8B9ZnYd4T+7HxC+NB4+hji/CzxtZmuBRwj/1Y0BTnX3W45hO2XABWY2jdD0sD1FneXAx6K4DxD2t0OKevUys3sITRmvEBJUL0IfwW7ghajaD4CnzGwl4VgYobP5PnffncbLvETox3nCzG7hSAfyxcBL7v46Yaj0G2b2DeBRQr/Dx5K2U0L47/ebZvZ/UZ04Liq8C7jFzJYTku31hONS75mJux82s5cISf/RpOKG3nfLzeyPwINmdhPhi7cLYd9Wu/tj7yP+fzSzOYTj9XHCPwenRWXLCc1zVxOazi4CrkxavwwYYGYnEwYA7CQ0kc4C/mpmN0TbGQrkuPvfjiG2c6L9WvE+9qvV0hlM2/chwgc88TEvKvsm4c3+eQB33wp8Bvh61NxT64+E/8xmAf8L/JrQ/lzrs4S29iejn52Aiz1cS5EWd3+e0H4+JdrGbMKopHfT31UgXLA3hTCKbV49dW4EthCas/5O6OA/1usLXiR8+TxC+NJ4PFp+obsvB3D3Zwlf9pdEsUyLYjuczgtE/Q+XEpLY/xI6zB8hjNDaENWZSfj7fZHQn/OPhM7mxO0sicqvi+pcSBg92NR+RviH5beEYwrhuKTqX0l0P/DJ6B+eROm8734L/ISQfJ8mjChb+z7jv40wAm4B4Xh91t3nALj7U4S+yzs5cgz/M2n9vwLPEpJKBXClux8m/P1nEAaCLCEk4mNtjr2ScAyOK7Wjd+QEFV3DsMjdv9LSsUjbY2bzgOnu/m+N1HuTMPrr99HzEvS+A8DMxhCS1vCkQRZtnprIRCQtZjaA0HQ0jTCI4AuE6zq+kMbq1xNGXMl79QY+fbwlF1CCEZH0HSZcC/RTQvP6YuASd290mHA02CJ5yLYA7v5C47XaJjWRiYhILNTJLyIisVATWaSwsNCHDh3a0mGkZdeuXeTk5LR0GGlRrPFQrE2vrcQJrSvWuXPnVrp7UaoyJZhIjx49eOutdGecaFklJSVMnjy5pcNIi2KNh2Jtem0lTmhdsUbXtqWkJjIREYmFEoyIiMRCCUZERGKhBCMiIrFQghERkVgowYiISCyUYEREJBZKMCIiEgslGBERiYUSjIiIxEIJRkREYqEEIyIisVCCERGRWCjBiIhILJRgREQkFs2aYMysi5k9bma7zGytmV1VTz0zs9vNbGv0uN3MLKH8fjNbZmaHzeyaFOvfYGabzKzazH5jZu1j3C0REUmhuc9gfgnsB3oAVwP3mNnoFPWuAy4HxgFjgcuA6xPK5wNfAt5OXtHMLgK+DlwADAAGA//VdLsgIiLpaLYEY2Y5wBXAre5e4+7TgSeBT6Wo/hngDndf7+7lwB3ANbWF7v5Ld38Z2FvPur9291J33w58L3Hd+pTXHOaul1awcsvOY9wzERFJxdy9eV7IbAIww907JSy7GTjP3S9LqlsFTHX3WdHzScCr7p6XVG868IC7P5iwbD7wQ3f/c/S8G1ABdHP3rUnrX0c4WyK759CJvT5zJwB9c41TemZxSs8seue2vm6qmpoacnNzWzqMtCjWeCjWptdW4oTWFeuUKVPmuvukVGVZzRhHLlCdtKwKyKunblVSvVwzM288I6Zal+h1jkow7n4/cD9A/yHD/ROT+vJ86WbW1xxg/coDPL7yAMU98rj0pF58eGwvhnZvHX/Q1nQ/7sYo1ngo1qbXVuKEthNrcyaYGiA/aVk+kKpNKrluPlCTRnKpb13qeZ06HbOMn3x8HN+//DBvrKrkmQUbeWHxZpZt3smyzTv5+UvLKe6Rx4fH9uLSk1pPshERaa2aM8EsB7LMbJi7r4iWjQNKU9QtjcpmN1Ivldp1H0lYd3Ny81h9srMymFzcncnF3fnBwcPMWFXJsws28nzpppBsXtzJf7+4nBE9j5zZDClSshERSdZsCcbdd5nZY8B3zexaYDzwUeDMFNUfAm40s2cBB24CflFbaGbZhAEKBrQzsw7Afnc/HK37oJn9EdgAfBt48P3EnJ2VwZTi7kwp7s4PPnYSM2rPbEo3sXTTTpZuOpJsPnxSLy5VshERqdOcZzAQhhb/BthC6A/5oruXmtk5wN/dvfbb+T7C8OKF0fMHomW1XgDOi34/k9CPMgUocffnzOwnwKtAR+CvwHc+aOCJyWb/x05ixspKnlkYzmxqk80dSjYiInWaNcG4+zbC9S3Jy18ndM7XPnfgluiRajuTG3md/wb++4PE2pDsrAymjOjOlBHd+WGUbJ5esJEXFh+dbIp75HHRmJ5cMqYnI3rmkXCtqIjIca+5z2COO4nJZv/Bk5i+soJnFmzihcWb6gYI3P3yCgZ27RQlm16M61ugZCMixz0lmCaUnZXB+SN6cP6IHuw/eBJvrt7Kc4s28kLpZsq27ua+aau5b9pqehV04KLRPbl4TE9OGdiFzAwlGxE5/ijBxCQ7K4Pzhhdx3vAivn+5M6dsG88t2sTzpZvYWLWXB98o48E3yuiak83U0T24eEwvzhjcleys1ndhp4jI+6EE0wwyM4zTB3fl9MFd+c9/GMWC8ir+vmgjzy3axNqtu/nT7HX8afY68jtk8aGRPbh4TE/OHV5Eh3aZLR26iMj7pgTTzDIyjPH9Chnfr5CvXzyCpZt28vdFm3h+UeizeWxeOY/NK6dTdiZTirtz0ZienD+iO7nt9acSkbZF31otyMwY2Sufkb3yufHC4ayuqAnJpnQTC9ZX8czCjTyzcCPZWRmcO6wbU0f35IIR3Vs6bBGRtCjBtCKDi3L58pShfHnKUNZv383zpZt5btFG3lq7nZeWbOGlJVvIMBhamMHKzNVcOKoHA7rmtHTYIiIpKcG0Un07d+LzZw/i82cPYsvOvbxQupkXFm/mzVWVLN9+mO8/s4TvP7OE4h55TB3dg6mjejKmT76GP4tIq6EE0wZ0z+vAv5w+gH85fQDVew9wz+PTWE9XSpZuqbvW5hevrKRXQQcuHBWSzWmDu9AuUyPSRKTlKMG0Mfkd2nFaryz+Y/IE9h88zMzVW3lh8SZeXLyZjVV7eejNtTz05lryOmRx/ojuTB3Vk/OKizRIQESanb512rDsrAzOHV7EucOL+O5HxrCwvIoXFm/ihdLNrNhSwxPvbOCJdzaQnZnBmUO7MnVUTz40qjvd8zq0dOgicgJQgjlOZGQY4/oVMq5fIV+7aARrKnfxYpRs5r67nZJlFZQsq+Bbf4Px/QqZOqonF47qofvaiEhslGCOU4O65XDduUO47twhVOzcxytLN/NC6WZeX1nJvHd3MO/dHdz+3FIGdcvhghHduWBkDyYN7Kx+GxFpMkowJ4CivPZ88pT+fPKU/uzad5DXV1TwQulmXl66hTWVu3hg+hoemL6G/A5ZTC7uzgUjuzN5eHcKOrVr6dBFpA1TgjnB5LTP4uIxvbh4TC8OHjrM3LXbeXnpFl5aspnVFbt4cv4Gnpy/gcwM45SBnblgRA8uGNmdwbq3jYgcIyWYE1hWZganDe7KaYO78s1LR7KmchcvL9nMy0u2MLtsGzNXh8cPnl3C4G45XDAyakob0JksNaWJSCOUYKTOoG45XHvOYK49ZzBVew4wbXkFLy/ZTMmyClZX7mL162v439fVlCYi6VGCkZQKOrbjI+N685FxvTl46DBvrd3OKw00pX1oZA8uGNmDQd00dY2IBEow0qiszIy62w0kNqW9tGQzc8q21zWlff+ZJQwuCqPSphR3Z9LALi0duoi0ICUYOWZHNaXtPsC0FQlNaRW7WF0RmtJysjMZ0Rk2dXqXycXd6VmgCzxFTiRKMPKBFHR6b1Paq8u2ULK0gmWbdzJ3M8x9bCEAI3vlM6W4iCkjujOhX6EGCogc55RgpMkkNqV945KRlO/Yw/1PTqfcOzNjZSVLNlazZGM1vypZRX6HLM4dXsSU4u6cV1xEt9z2LR2+iDQxJRiJTZ/Cjkzp347Jkyex98Ah5pRt49WlFZQs28Lqyl08vWAjTy/YiBmM7VPA5OLunD+iOyf1KSAjQ7cdEGnrlGCkWXRol8k5w4o4Z1gR/3nZKMoqd/Hqsi28uqyCmau3Mn99FfPXV3HXyyvompPNecXh7ObcYUUaBi3SRinBSIsY2C2Hz3YbxGfPGsTu/Qd5c9XWkHCWVlC+Yw+PvV3OY2+Xk2EwcUBnJhd357zhRYzqla+zG5E2QglGWlyn7CwuiK6jcXdWbqmpSzZzyrYxp2w7c8q289Pnl9EtN5tzhhVx3vAizh7WTX03Iq2YEoy0KmbGsB55DOuRx3XnDmHn3gPMWFnJq0sreG1FBRur9vL4vHIen1cOwJg++Zw3vIhzhxVx8gDNBi3SmijBSKuW16Fd3eSctWc305ZXMG15BbPWbGNReTWLyqv55auryG2fxZlDunLu8HCG069Lp5YOX+SEpgQjbUbi2c215wxmz/5DzFqzldeWVzJt+RZWVezihcWbeWHxZgAGd8upSzanDe5Cp2y93UWakz5x0mZ1zM5kcnF3Jhd3B0ZRvmMPry2vYNqyCmasrAwTdFbu4sE3ysjOzODUQV04d3g3zh1eRHGPPMw0WEAkTs2aYMysC/BrYCpQCXzD3R9OUc+AHwPXRoseAL7u7h6Vj4+2MxJYAnze3d+JytoDdwEfA9oBM4B/dffyGHdNWoE+hR258tT+XHlqfw4eOsw763YwbXkFry2vYEF5FdNXVjJ9ZSU/fHYpPfLbc+6wIs4dXsRZQ7vRJSe7pcMXOe409xnML4H9QA9gPPCMmc1399KketcBlwPjAAdeBNYA95pZNvAEcCfwK+B64AkzG+bu+4F/B84AxgJVwP3AL4B/jHnfpBXJysxg0sAuTBrYhZumFrNt135eX1HBa8sreW1FBZur9/GXuev5y9z1mMHo3vn0z95PVp9KJg3sTId2mS29CyJtXrMlGDPLAa4Axrh7DTDdzJ4EPgV8Pan6Z4A73H19tO4dwBeAe4HJUdx3Rmc0d5vZzcD5wHPAIOB5d98crftn4L9j3j1p5brkZPPR8X346Pg+uDtLNu5k2vIKpq+sYE7Z9jBYAHj217NonxWa084e2o2zhnbTtTci75NFrU7xv5DZBGCGu3dKWHYzcJ67X5ZUtwqY6u6zoueTgFfdPc/MbojKLkmo/3RUfkdU9y7gn4AdhOa1Le7+/1LEdB3hbImioqKJjzzySNPudExqamrIzW0btzBuC7HuO+Ss2H6IeRv3sqI6k3d3Hj6qPC8bRnXJZFS3TMZ0zaRrx5YfCt0WjmutthJrW4kTWlesU6ZMmevuk1KVNWcTWS5QnbSsCsirp25VUr3cqG8muSx5OyuAdUA5cAhYCHwlVUDufj+hCY3i4mKfPHlymrvSskpKSlCsTesijsRaWbOPGSsrmbGykukrKtlQtZdZmw4xa9MhIIxOO3tYOLs5Y0hX8js0/1Q2beW4QtuJta3ECW0n1rQTjJn1IDRnDQFudfdKMzsL2ODua9LYRA2Qn7QsH9iZRt18oMbd3cwa284vgfZAV2AXcAvwd+C0NGIUoVtu+6Oa01ZX7mLGykpeX1HJzFVb60anPfTmWjIzjHF9Czh7aDfOHlbEhP6FuthTJJJWgjGzicDLhI720cBPCaPALgSGA1elsZnlQFbUGb8iWjYOSO7gJ1o2Dpidol4pcJOZmR9p3xtLSCwQBg98y923RbH/AviumXVz98p09leklpkxpCiXIUW5fPqMgRw8dJj563cwfcVWpq+sYN67O3g7etz9ykpysjM5bXDXuv6b4T1yNRxaTljpnsH8DLjL3b9jZolnHM8Dn01nA+6+y8weI3zZX0tIBB8FzkxR/SHgRjN7ljCK7CbCSDCAEkLT11fN7F5C5z/AK9HPOcCnzawE2A18iXCWpeQiH1hWZgYTB3Rh4oAu/PuHhlGz7yCzVm/l9RVhCPTKLTW8snQLryzdAkC33GzOGNKNM4d05cwhXenfpZMSjpww0k0wE4HPp1i+kTDkOF1fAn4DbAG2Al9091IzOwf4u7vX9lrdBwwm9J9A6Ki/D8Dd95vZ5dGyHxOug7k8GqIMcDNwN6EvJhtYRLgmRqTJ5bY/MlEnwKaqvUyP+m/eWFXJ5up9PDV/A0/N3wCEa3XOiJLNmUO66TbSclxLN8HsATqnWD6CkCzSEjVbXZ5i+euEzvva507oO7mlnu3MIyS9VGVbgavTjUmkKfUs6MDHJ/bl4xP71vXfvLFqK2+uquTNVVsp37GHR+eu59G564EwYOCMKNmcPrgLXTU7tBxH0k0wTwDfMbN/ip67mQ0Ebgf+GkNcIm1eYv/Np04fwOHDzpJN1by5aitvrNrKrNVHBgz8cda7AIzomcdZQ0OT2qmDupDXAiPURJpKugnmZuBZoALoBEwnNI3NAL4dT2gix5eMDGN07wJG9y7g2nMGc+DQYRaWV0UJp5K3yrazdNNOlm7aya+nryEzwzipT0Fdc9rEAZ3pmK0ZBqTtSCvBuHs1cLaZnQ+cDGQAb7v7S3EGJ3I8a5eZwcn9O3Ny/858ecpQ9h44xNvvbq87w5m/bgfvRI9flawiOzODCf0LOXNINzpUH+L0A4c0pY20aukOU/408Gd3f4Ujo7WI5gX7Z3d/KKb4RE4YHdplcuaQbpw5pBs3ATX7DjKnbFvdGU7phmpmrdnGrDXbALjj7Rc4uX8hpw/uymmDujKhf6ESjrQq6TaR/ZYwz1dyh35eVKYEI9LEcttnMaW4O1OKuwOwY/d+Zq7eypurtvLywndZX3OYmau3MXP1NmAF2VkZjO8XEs7pg7pw8gBN2iktK90EY4TrUZL1573TtohIDAo7Zdfd3XNKQSVjTzmT2Wu2MXP1Vmat2caSjdXMXrON2Wu2cTeQnRkSzmmDu3D64K6c3F99ONK8GkwwZraQkFgcmGZmBxOKM4EBhM5/EWlmXXKyuXhMTy4e0xMIZzgh4YSks2RTNbPLtjG7bBu/eGUl7TKNcX2jJrXBXZg4oLPu8imxauzd9Wj0cwzwDGGOsFr7gTI0TFmkVSjslM3U0T2ZOjoknKrdB5hdto1Zq7cyc81WFm+o5q2123lr7Xb+51XIyjDG9i0ITWqDuzJxQGdy2ivhSNNp8N3k7v8FYGZlhE7+vc0RlIh8cAWd2nHhqB5cOCrMMlC15wBvlYVBAjNXb2VReVXdPGq/KllFVoYxpk8Bpw3qwikDuzBpYGcKO+lOn/L+pTtM+XdxByIi8Sro2O6oaW2q9x5gbtl2Zq7eysw121hUXlU3LPq+11YDUNwjj1MGdeaUgV04dVAXehV0bMldkDYm3WHK2cC3gCsJHftHXV7s7uo5FGlj8ju0Y8qI7kwZEUap1ew7yNy125mzJvTbvLNuB8s272TZ5p38YWaYaaBv546cOrALp0RnOUOKcjR5p9Qr3QbX7wGfBH4E/Bz4GjAQ+Gfg1lgiE5Fmlds+i/OGF3He8CIA9h08xIL1Vcxes405ZduYW7ad9dv3sH57OY/NKwega042kwYeOcMZ1SufLN0PRyLpJphPAP/q7s+Z2c+AJ9x9lZktIdwT5r7YIhSRFtE+K5NTBoYzFYBDh52lm6qZs2Ybc8q2M7tsGxU79/F86WaeL90MQE52JicPOJJwxvcrbMldkBaWboLpASyOfq8Bat81zxEmvBSR41xmwlxq15w1CHdn7dbdzC7bFiWdbZRt3c3rK8LdPwHaZRoD8oyZe5Zy6qDOTOzfhYJOmsDzRJFugnkX6B39XEm4hflc4AzCVP4icoIxMwZ2y2Fgtxw+MakfAFuq9zKnbDtzysIFn0s2VbNyh7Ny2irunRbWG94jl4kDOkc3buvMwK66CdvxKt0E8zhwATATuAv4k5l9AehDuH2yiAjd8zvw4bG9+PDYXkAYqfbgU9PYm9eXOWXbmL++iuWba1i+uYY/zV4HhH6ckwd0ZtKAzkwa2JkxfQpon6VxQ8eDdIcpfyPh90fNbB1wFrDc3Z+OKzgRadvyO7RjbFEWkyePAMLAgUXl1cxdu425a7czd+12Kmv28+Lizby4OPTjZGdmcFLfgugsJzy66UZsbdL7umzX3WcBswDMLMfddzVpVCJyXGqflVmXNIC6fpy50QwDb6/dzvItO+uST62BXTvVNalNGtiZoUW5ZGSoWa21e9/zQphZB+DfCEOWuzdZRCJywkjsx7liYl8gzDjw9rsh2bxVtp131u2gbOtuyrbu5q9vh1tN53fIqmtWO3lAZ8b3K9S8aq1QY5NdZgPfAaYCB4CfuPvfovvD/JgwCebPY49SRE4YBR3bHXWbggOHDrN0407eWruNt9ZuZ27ZdjZV76VkWQUlyyqA2hFu+Zzcvz/wKL8AABpaSURBVDMT+hdycv/O9O3cUYMHWlhjKf824MvAi4Q+l7+Y2f8SOvy/ATzs7gdijVBETmjtoj6Zk/oW8NmzBgFQvmNPaEYrC0lnycZqFqyvYsH6Kh58I6zXLTeb8f06c/KAQib068zYvgWazLOZNXa0PwFc4+6Pm9k4YB7QGRjt7gcbXlVEJB59CjvSp7AjHxnXG4Bd+w7yzrodzHt3O/Pe3cHb74bBAy8t2cxLS8LggQyD4p75dWc4E/oXMqhrjvpyYtRYgukHzAFw9/lmth+4XclFRFqTnPZZnDW0G2cN7QYcGTwwb92RhLNk406WbKxmycZqHp4V5lYr6NiO8f1CwsnccZAJew5Q0FEXgjaVxhJMO2BfwvMD6A6WItLKJQ4e+NiEMHhgz/5DLCyvOuosZ8vOfUxbXsG05aEv52dvvcDQ7rlM6FfIyQPCWc6w7nlk6iznfUmnQfJHZrY7+j0buM3Mjkoy7v7VJo9MRKQJdczO5NRBYY40CGc5G6r21iWckoVrWVcDK7fUsHJLDX+ZG0as5WRnMq5fIRP6h76ccf0KKcrTdTnpaCzBvAYMSXj+BmG6/kTepBGJiDQDM6vry/mHsb05J3cLZ5x9Dos3VNed4cx7dwflO/bwxqqtvLFqa926fQo7Mr5fIeP6FTCubyFj+mgAQSqN3dFycjPFISLS4tpnZTKhf2cm9O/M5wgj1rZU72Xeuh11SWdReRXlO/ZQvmMPzyzcCIQBBMO654WE06+QcX0LKe6ZR7sT/NYFSrkiIg3ont+Bi0b35KLRPYFw24KVW2qYv34H89ftYP76HSzduLPu5myPvBWa1tpnZTCmTzjDqT3TGXCCTeypBCMicgwyM4zinnkU98yrm0V674FDlG6oZv66HSxYv4P566tYU7nrPVPeFHZqx9i+hYzvG850xvY9vvtzmjXBmFkX4NeEmQEqgW+4+8Mp6hlhpoBro0UPAF93d4/Kx0fbGQksAT7v7u8krH8ycCdwMrAL+KG73xXXfonIia1Du6PnWAPYsXs/C9ZX1Z3lvLOuisqafby2vILXolFrEPpzas9wxvUr5KTjqD+nuffil8B+wg3MxgPPmNl8dy9NqncdcDkwjjCI4EVgDXBvNH3NE4QE8ivgeuAJMxvm7vvNrBvhRmg3AI8SRr71jX3PREQSFHbK5tzhRZwb3YLa3dlYtZf563bwTtS8tnD9kf6cZxduAsAMhhblclKfMHvBSX0KGNU7v03OtdZsEZtZDnAFMMbda4DpZvYk8Cng60nVPwPc4e7ro3XvAL4A3AtMjuK+MzqjudvMbgbOJySWG4Hn3f2P0bb2Ec5yRERajJnRu7AjvQs7cslJ4X45hw47qypq6s5y5q+rYsnGalZsqWHFlhoem1cOHBlEMKZPAWP7FnBg+yFO23+Ijtmt+745FrU6NVzJLHloci0H9rp7RT3liduYAMxw904Jy24GznP3y5LqVgFTo9sCYGaTgFfdPc/MbojKLkmo/3RUfoeZvQIsBE4BhhJuK/Bld383RUzXEc6WKCoqmvjII480thutQk1NDbm5uS0dRloUazwUa9NrLXHuP+SsrzlMWdVhyqoPs6bqMOU1hzmc9FWdYdA7xxhUkMnAggwG5mfQLy+D7MzmHUQwZcqUue4+KVVZumcwZTRwvYuZVQO/BW5pYBqZXKA6aVkVkFdP3aqkerlR30xyWfJ2+hL6Xi4kJJqfAH8iTNZ5FHe/H7gfoLi42CdPnlxP6K1LSUkJirXpKdZ4tJVYW3Ocew8cYsnGahaWV7FwfRVvLitnwy5nfY2zvuYgr4cTHbIyjGE98hib0Lw2oldei90hNN0EcyXhi/peohuNAacR/vu/DSgEvg3sJEzvn0oNkJ+0LD9ap7G6+UCNu7uZNbadPcDj7j4HwMz+C6g0swJ31zQ3ItLmdGh35PocgJKS7Zx25jks3ljNwvU7WFBexaLyKlZuqambb+3Pb4VbUrfLNIb3yGNs34LQxNYnXKOTnRX/NTrpJpgvAje4+2MJy14xs2XAv7v7eWa2Bfgv6k8wy4GsqDN+RbRsHJDcwU+0bBwwO0W9UuAmMzM/0r43ljCAAGABR59taaYBETnudMx+78i13fsPsnhDuHXBwvLwWFVRQ+mGako3VANHkk5xzzzG9C5gdJ8CxvTOZ0TP/Cbv00k3wZxGaG5KtojQ1wHwJg2M1nL3XWb2GPBdM7uWMIrso8CZKao/BNxoZs8SEsRNwC+ishLgEPBVM7uX0PkP8Er087fAX83sbkIyuhWYrrMXETnedcrOYtLALkwa2KVuWc2+g5SWH0k4C9dXsbpyF4vKq1lUXg1zQtLJMBjaPZcxvcOotTHR6LX8Du9/dul0E8xaQnPY15KWfwGo7TwvArY1sp0vAb8BtgBbgS+6e6mZnQP83d1re9juAwZzJKk9EC0jGop8ebTsx4QRYpe7+/6o/BUz+ybwDNAJmA5cleZ+iogcV3LbZ3Ha4K6cNrhr3bKdew+weEM1izZUU7qhitLyalZW1LB8c3jUjl4DGNC1U3Smkx9+9s6na256F4emm2BuIpwVXEp0fxhgEmEizCui56cADQ7DcvdthOtbkpe/Tui8r33uwC3RI9V25gETG3ide4B7GopFROREldeh3XuSzt4Dh1i6aSeLyquiJrUqlm7cydqtu1m7dXfdvGsAvQo6MLp3AWP6JHeHHy2tBOPuz5jZMMIZSHG0+Eng3trhv+7+q2PaQxERaTU6tMtkfL9CxvcrrFt24NBhVm6pOSrplG6oZmPVXjZW7a27W2h90r7Q0t3XAd9439GLiEib0i4zg5G98hnZK59/ipYdPuys2borJJzyKr55e/3rp51gzKwToWO+O3DU+Lak0WUiInKcysgwhhTlMqQol4+M6803G6ibVoIxsw8RLlbsmqLYgdY9X4GIiDS7dK+0uYswKquvu2ckPZRcRETkPdJtIhsIfMTdN8QYi4iIHEfSPYOZwZHRYyIiIo1K9wzmXuBnZtabcPHjgcRCd3+7qQMTEZG2Ld0E82j08/4UZerkFxGR90g3wQyKNQoRETnupHsl/9q4AxERkeNLvQnGzP4ReMrdD0S/10sXWoqISLKGzmAeBXoSZj5+tIF66oMREZH3qDfBuHtGqt9FRETSocQhIiKxOJbJLvsC55J6ssv/buK4RESkjUt3ssurCXeiPAhU8N573ivBiIjIUdI9g/kucAdwq7sfijEeERE5TqTbB9MDeEDJRURE0pVugnkWOC3OQERE5PiSbhPZi8DtZjaa1JNd6kJLERE5SroJ5r7oZ6q7Y+pCSxEReY905yLT9TIiInJMGk0cZtbOzGaZmW44JiIiaWs0wbj7AcJ0/d5YXRERkVrpNn39DvhCnIGIiMjxJd1O/hzgajO7EJgL7EosdPevNnVgIiLStqWbYEYCb0e/D04qU9OZiIi8R7qjyKbEHYiIiBxfNPxYRERikXaCMbMpZna/mT1nZq8kPo5hG13M7HEz22Vma83sqnrqmZndbmZbo8ftZmYJ5ePNbK6Z7Y5+jk+xjWwzW2Jm69ONT0REmk5aCcbMrgH+DuQBkwlT9ncGTgYWH8Pr/RLYT5g882rgnmj6mWTXAZcD44CxwGXA9VEs2cATwB+iGH4HPBEtT/S1KE4REWkB6Z7B3Ax8xd2vJMxD9g13n0D4kq9JZwNmlgNcQZjyv8bdpwNPAp9KUf0zwB3uvt7dywm3CrgmKptM6Du60933ufvdgAHnJ7zWIOBfgB+luX8iItLEzL3xQWBmthsY5e5lZlYJnO/uC8xsBFDi7j3T2MYEYIa7d0pYdjNwnrtfllS3Cpjq7rOi55OAV909z8xuiMouSaj/dFR+R8LzXwPbgT+4e996YrqOcLZEUVHRxEceeaTRY9Ea1NTUkJub29JhpEWxxkOxNr22Eie0rlinTJky190npSpLd5jyVkLzGEA5MAZYAHQFOqa5jVygOmlZVcJ2k+tWJdXLjfphksuO2o6ZfQzIdPfHzWxyQwG5+/3A/QDFxcU+eXKD1VuNkpISFGvTU6zxaCuxtpU4oe3Emm6CeR2YSpiq/xHg7uiiywsIU/mnowbIT1qWD+xMo24+UOPubmb1bidqhvsJcGmaMYmISEzS7YP5CvCn6PcfAT8lnL08Alyb5jaWA1lmNixh2TigNEXd0qgsVb1SYGziqDLCQIBSYBgwEHjdzDYBjwG9zGyTmQ1MM04REWkC6V5ouS3h98PA7cf6Qu6+y8weA75rZtcC44GPAmemqP4QcKOZPUuYKeAm4BdRWQlwCPiqmd3LkTnSXgEOA/0StnMm8D+E0W4aUSYi0oyO5TqYHmZ2s5ndY2bdomVnRSO20vUlQp/NFsIZ0RfdvdTMzomavmrdBzxFaJJbBDwTLcPd9xOGMH8a2AF8Drjc3fe7+0F331T7ALYBh6Pnh44hThER+YDSOoMxs4nAy8AaYDShiawSuBAYDqS8YDJZdCZ0eYrlrxM672ufO3BL9Ei1nXnAxDRerwRIOYJMRETile4ZzM+Au6JrX/YlLH8eOKvJoxIRkTYv3QQzkXDFfLKNhKvyRUREjpJugtlDmJYl2QhCf4qIiMhR0k0wTwDfMbP20XOPhv3eDvw1hrhERKSNO5a5yLoQhvp2AqYDKwlX0H87ntBERKQtS/c6mGrgbDM7n3BNSQbwtru/FGdwIiLSdqU7VQwA7v4K4YJGAMxsAPBTd/9EUwcmIiJt2we9o2UhYQp+ERGRo+iWySIiEgslGBERiYUSjIiIxKLBTn4ze7KR9ZPvyyIiIgI0Popsaxrla5ooFhEROY40mGDc/bPNFYiIiBxf1AcjIiKxUIIREZFYKMGIiEgslGBERCQWSjAiIhILJRgREYmFEoyIiMRCCUZERGKhBCMiIrFQghERkVgowYiISCyUYEREJBZKMCIiEgslGBERiYUSjIiIxKJZE4yZdTGzx81sl5mtNbOr6qlnZna7mW2NHrebmSWUjzezuWa2O/o5PqHsa2a2yMx2mtkaM/tac+ybiIgcrbnPYH4J7Ad6AFcD95jZ6BT1rgMuB8YBY4HLgOsBzCwbeAL4A9AZ+B3wRLQcwIBPR2UXA18xs3+Oa4dERCS1ZkswZpYDXAHc6u417j4deBL4VIrqnwHucPf17l4O3AFcE5VNJtyJ80533+fudxOSyvkA7v4Td3/b3Q+6+zJCMjorxl0TEZEUzN2b54XMJgAz3L1TwrKbgfPc/bKkulXAVHefFT2fBLzq7nlmdkNUdklC/aej8juStmPA28B97n5vipiuI5wtUVRUNPGRRx5por2NV01NDbm5uS0dRloUazwUa9NrK3FC64p1ypQpc919UqqyrGaMIxeoTlpWBeTVU7cqqV5ulDCSyxrazm2Es7TfpgrI3e8H7gcoLi72yZMnN7gDrUVJSQmKtekp1ni0lVjbSpzQdmJtzgRTA+QnLcsHdqZRNx+ocXc3s7S2Y2ZfIfTFnOPu+z5I4CIicuyas5N/OZBlZsMSlo0DSlPULY3KUtUrBcYmjiojDASo246ZfQ74OnCBu69vgthFROQYNVuCcfddwGPAd80sx8zOAj4K/D5F9YeAG82sj5n1Bm4CHozKSoBDwFfNrH10pgLwCoCZXQ38ELjQ3VfHtT8iItKw5h6m/CWgI7AF+BPwRXcvNbNzoqavWvcBTwELgUXAM9Ey3H0/YQjzp4EdwOeAy6PlAN8HugJzzKwmeryng19EROLVnH0wuPs2QnJIXv46ofO+9rkDt0SPVNuZB0ysp2xQkwQrIiIfiKaKERGRWCjBiIhILJRgREQkFkowIiISCyUYERGJhRKMiIjEQglGRERioQQjIiKxUIIREZFYKMGIiEgslGBERCQWSjAiIhILJRgREYmFEoyIiMRCCUZERGKhBCMiIrFQghERkVgowYiISCyUYEREJBZKMCIiEgslGBERiYUSjIiIxEIJRkREYqEEIyIisVCCERGRWCjBiIhILJRgREQkFkowIiISCyUYERGJhRKMiIjEolkTjJl1MbPHzWyXma01s6vqqWdmdruZbY0et5uZJZSPN7O5ZrY7+jk+3XVFRKR5NPcZzC+B/UAP4GrgHjMbnaLedcDlwDhgLHAZcD2AmWUDTwB/ADoDvwOeiJY3uK6IiDSfZkswZpYDXAHc6u417j4deBL4VIrqnwHucPf17l4O3AFcE5VNBrKAO919n7vfDRhwfhrriohIM8lqxtcaDhx09+UJy+YD56WoOzoqS6w3OqFsgbt7QvmCaPlzjax7FDO7jnDGA7DPzBaltystrhtQ2dJBpEmxxkOxNr22Eie0rlgH1FfQnAkmF6hOWlYF5NVTtyqpXm7Ul5JclrydetdNSkq4+/3A/QBm9pa7T0p/d1qOYo2HYo1HW4m1rcQJbSfW5uyDqQHyk5blAzvTqJsP1EQJorHtNLSuiIg0k+ZMMMuBLDMblrBsHFCaom5pVJaqXikwNmlk2Nik8vrWFRGRZtJsCcbddwGPAd81sxwzOwv4KPD7FNUfAm40sz5m1hu4CXgwKisBDgFfNbP2ZvaVaPkraazbkPuPfa9ajGKNh2KNR1uJta3ECW0kVmvOliMz6wL8BrgQ2Ap83d0fNrNzgL+7e25Uz4DbgWujVR8A/qO2mcvMJkTLRgFLgM+7+7x01hURkebRrAlGREROHJoqRkREYqEEIyIisTjhE0y686M1YzwlZrbXzGqix7KEsquiGHeZ2d+iPq3aslj3w8y+YmZvmdk+M3swqewCM1sazQ33qpkNSChrb2a/MbNqM9tkZjemu25Tx2pmA83ME45tjZnd2lKxRq/36+jvtdPM3jGzS9J5vdYUa2s7rtE2/2BmG6PXXG5m1yaUtZrj2lCsrfG4HjN3P6EfwJ+APxMu0DybcGHm6BaMpwS4NsXy0YRrfc6NYn0Y+L/m2g/gHwlzvN0DPJiwvFv0Wv8EdAB+CsxMKP8R8Dph3riRwCbg4nTWjSHWgYADWfWs16yxAjnAbVFcGcA/RH/jga3tuDYSa6s6rgmfl/bR7yOi15zY2o5rI7G2uuN6zPvW3C/Ymh7Rh2Y/MDxh2e+BH7dgTCWkTjA/BB5OeD4kij2vOfcD+D5Hf2lfB7yRdEz3ACOi5xuAqQnl3yNKjI2tG0OsjX1gWyzWhO0uIMzZ12qPa4pYW/VxBYqBjcAnWvtxTYq1VR/XdB4nehNZffOjpZy7rBn9yMwqzWyGmU2Olh01x5q7ryJKKrTsfiTHtQtYBYw2s85ALxqeVy7lujHHvNbM1pvZb82sG0BriNXMehD+lqUNvV4rjLVWqzquZvYrM9sNLCV8aT/b0Ou1wlhrtarjeixO9ARzLPOjNZf/AAYDfQgXUz1lZkNoeA62ltyPxuKC984NV9+8ccnlTa0SOIUwOd/E6HX+mBBL7euniiXWWM2sXRTL79x9aSOv19pibZXH1d2/FG3nHMJF3vsaeb3WFmurPK7H4kRPMMcyP1qzcPdZ7r7Tw60IfgfMAC6l4Vhbcj8aiwveOzdcffPGJZc3KQ+3iXjL3Q+6+2bgK8BUM8tryVjNLIPQpLk/iqmx12tVsbbW4xrFdsjDrUH6Al9s5PVaVayt+bim60RPMMcyP1pLccL9bo6aY83MBgPtCfvQkvuRHFcOoX+o1N23E073G5pXLuW6Mcdcq/Yq44yWitXMDPg14SZ8V7j7gcZerxXGmqzFj2sKWQnbbVXHtYFYk7XG49qw5uzwaY0P4P8II7BygLNowVFkQCFwEWHURxbhrp+7CG3downNYOdEsf6Bo0eRxbofUTwdCCNXfp8QY1H0WldEy27n6FE5PwamEUa6jCB8KGpHujS4bgyxnkboRM0AuhJG3b3awrHeC8wEcpOWt8bjWl+sreq4At2BfyY0E2VGn6ldwEda23FtJNZWdVzf1/419wu2tgfQBfhb9Ed9F7iqBWMpAuYQTmN3RB/mCxPKr4pi3EW4bXSX5toPwhBVT3rcFpV9iNA5uYcwCm5gwnrtCfPPVQObgRuTtlvvuk0dK3AlsCY6RhsJE6P2bKlYCW3rDuwlNGnUPq5ubce1oVhb4XEtInzx7ohecyHwhXRerzXF2tqO6/t5aC4yERGJxYneByMiIjFRghERkVgowYiISCyUYEREJBZKMCIiEgslGBERiYUSjMhxxsyuMbOaxmuKxEsJRiQmZvZgdMOo2kelmT1tZiOOYRu3mdmiOOMUiYsSjEi8XiJMq94LmAp0BB5v0YhEmokSjEi89rn7pujxNvBzYISZdQQwsx+b2TIz22NmZWb2EzPrEJVdA3yHcK+S2rOga6KyAjO7J7rV7l4zW2Jmn0x84eiWuYss3Eb7VTMb1Jw7LpLV0gGInCiiadY/CSx09z3R4l3A54ByYBRhQsl9wK2EyQ3HEG5PPDmqXxXNavwsYZLDzxJm0y4mTGpYqz3wjWjbe4HfRdu+KJ69E3kvJRiReF2c0OGeA6wj3N8HAHf/XkLdMjP7IXAzcKu774nWPejum2ormdmFwBmE2bKXRItXJ71uFvBld18WrfMz4DdmZq4JCKWZqIlMJF6vAeOjx6nAy8ALZtYPwMw+bmbTzWxTlEx+DvRvZJsTgI0JySWVfbXJJbIByCac9Yg0CyUYkXjtdveV0WMOcC3hzoLXmdnphPv4PA9cRkgc3wbaNcHrHkx6XnezqibYtkha1EQm0rwcOAx0ItwYrjyxmczMBiTV30+4EVWieUAvMxvZyFmMSItSghGJV3sz6xn93plwX/Vc4CkgD+hjZlcDbxI64K9MWr8MGGBmJxNuJLeT0Mw2C/irmd1A6OQfCuS4+9/i3R2R9Ol0WSReHyLcjXAjISmcAvyTu5e4+1PAT4E7gQXAhcB/Jq3/V8KIsZeBCuBKdz8MXALMINw6ewlwF6GPRaTV0B0tRUQkFjqDERGRWCjBiIhILJRgREQkFkowIiISCyUYERGJhRKMiIjEQglGRERioQQjIiKx+P9XCgqGIqEkdAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plotting the learning rate by epoch for exponential scheduling\n",
    "plt.plot(steps, lrs, \"-\", linewidth=2)\n",
    "plt.axis([0, n_steps - 1, 0, lr0 * 1.1])\n",
    "plt.xlabel(\"Batch\")\n",
    "plt.ylabel(\"Learning Rate\")\n",
    "plt.title(\"Exponential Scheduling (per batch)\", fontsize=14)\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Piecewise Constant Scheduling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "# defining the epoch for piecewise constant scheduling\n",
    "def piecewise_constant_fn(epoch):\n",
    "    if epoch < 5:\n",
    "        return 0.01\n",
    "    elif epoch < 15:\n",
    "        return 0.005\n",
    "    else:\n",
    "        return 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# defining piecewise constant scheduling\n",
    "def piecewise_constant(boundaries, values):\n",
    "    boundaries = np.array([0] + boundaries)\n",
    "    values = np.array(values)\n",
    "    def piecewise_constant_fn(epoch):\n",
    "        return values[np.argmax(boundaries > epoch) - 1]\n",
    "    return piecewise_constant_fn\n",
    "\n",
    "# calling the function\n",
    "piecewise_constant_fn = piecewise_constant([5, 15], [0.01, 0.005, 0.001])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 25000 samples, validate on 5000 samples\n",
      "Epoch 1/5\n",
      "25000/25000 [==============================] - 3s 139us/sample - loss: 0.9226 - accuracy: 0.7452 - val_loss: 1.4468 - val_accuracy: 0.6448\n",
      "Epoch 2/5\n",
      "25000/25000 [==============================] - 3s 108us/sample - loss: 0.7608 - accuracy: 0.7668 - val_loss: 1.0174 - val_accuracy: 0.7112\n",
      "Epoch 3/5\n",
      "25000/25000 [==============================] - 3s 114us/sample - loss: 0.7754 - accuracy: 0.7734 - val_loss: 0.7917 - val_accuracy: 0.7908\n",
      "Epoch 4/5\n",
      "25000/25000 [==============================] - 3s 117us/sample - loss: 0.8074 - accuracy: 0.7744 - val_loss: 0.7958 - val_accuracy: 0.7884\n",
      "Epoch 5/5\n",
      "25000/25000 [==============================] - 3s 139us/sample - loss: 0.8200 - accuracy: 0.7686 - val_loss: 1.0198 - val_accuracy: 0.7006\n"
     ]
    }
   ],
   "source": [
    "# creating an instance of the LearningRateScheduler\n",
    "lr_scheduler = keras.callbacks.LearningRateScheduler(piecewise_constant_fn)\n",
    "\n",
    "# defining the model\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.Dense(300, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n",
    "    keras.layers.Dense(100, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n",
    "    keras.layers.Dense(10, activation=\"softmax\")\n",
    "])\n",
    "\n",
    "# compiling the model\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"nadam\", metrics=[\"accuracy\"])\n",
    "n_epochs = 5\n",
    "\n",
    "# training the model\n",
    "history = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n",
    "                    validation_data=(X_valid_scaled, y_valid),\n",
    "                    callbacks=[lr_scheduler])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAEeCAYAAAAjNKpiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deZxcVZn/8c8XAghpYgRjkDAICgSIkwQT14B0WMWRIYLjKChElCDID2VVHEHAcQkSBxiZQEQNEEUyCIZFZdTQCsogm0AiEh0WJexglk4ggfD8/jinoSiqu253qup2p7/v16teXXXOubeeezqpp8+5p+5VRGBmZlaG9coOwMzMBi8nITMzK42TkJmZlcZJyMzMSuMkZGZmpXESMjOz0jgJ2atI2kZSSJpYdizdkdQh6dtlx2HFSJoqqbNJ+14g6fRebvOgpBO7e22t4yQ0CEmanZNMSHpe0v2SzpY0NDf5G/BG4A8lhlnPgcApzXwDJZ+SdLOk5ZKWSbpD0smShjXzvaviaNoHZNF9S9pW0hxJD0taJekRSddJ2qUZcZXg7cB/lR3EYDSk7ACsNL8EPg5sAOwGXAQMBY6KiDXAYyXGVldEPNOCt7kUOAj4GvBZ4AlgDHBMfj67BTGUTtIGwC+A/wM+DCwGtgT2ATYrMbSGiYgny45h0IoIPwbZg/TheW1V2XeAR/PzbYAAJlbU7wxcBywnfQBfBmxRtY/DgHuAVcDjwMUVda8FZuVtlwO/rtr/o8BHKl7flNsNya+3yzFtlV93AN+uaH8gcDfwLPBM3v/Iivr9gduB54AHgK8CG/bQRx/O73dgN/XD88/1gFNJo8dV+fgPqGjX1ZcHkT7IVwJ/BPauaLMBcB7wSN7H34BvVBxnVD5y+eb5d/BwPuaFwCeqYuwg/XX/NeCp3PdnA+v1tO8axzo+129X59/Va4GZ+Xf5HHAv8K+5birQCewJLABWADcA21bto8ffE/AGYF4+5oeAw/P+Tq9oE8CHqvb7IHBiL14HMA347xzr/cDHqvb5TuCOHOudwPvzdu1l/x8fSA9Px1mXZ0kfhq8i6Y3Ab0j/2d8B7AW0AfMkrZfbHAlcCHwfGEv6D7kg14mUwEYBHwB2yfubn/cNKWm05/abkKZHVgFd56Xagf+LiIdrxLcF8CPgYmAn4L2kUUxX/b7AD4Bvk0YyhwMfIn04d+cQYFFEXFmrMiKW5KefBU4CPg/8I3AVcKWk8VWbfJWUaMYBtwI/ktSW644FPgh8BNge+Ffgvlx3ICnRnEmaIu3qr9eQPgA/kI/pXOBCSXvWOI4XgPeQRnCfy/vvad/VngReBA6SVHP2JP+OfwrsDnyC9EfL8cDqimYbkaZQDwfeDQwHLqjYR5Hf02zSHyR7AVOAQ0mJvhlOIyW8ccDlwPckbZ1jbQOuBf4ETABOBr7ZpDjWbWVnQT9a/6BqJERKLE8Bl+fX21AxEiJ9SP2qah+vy23ekV8/TP7rvcb77UH6K3jjqvI/ACfn558G7svP9yL9FT0bOCWXzQEuqti2gzwSAt6WY3lTN+//G+DUqrIpOSZ1s80fgXkF+nIxcFpVWQcwp6ovj6yoH5XLds2vzwN+1UMsD1LxV3oPsfyoRh/dXNXmF1Vtiu77M6QRQSfpD4avAGMq6vcmJaqdutl+aj7m0RVlh5D+0FCR3xOwQ97HpIr6NwFraM5I6OsVr4eQRrEfy6+PJI24N65oczAeCfX64ZHQ4PU+SZ2SngNuJn0A/L9u2k4A3pvbd+ZVTn/LdW+R9AbSB+uveth+E+DJqn28FXhLbtMB7JBHRu2kqZqO/BzSX9gd3ez/LtI5rgWSfizpKEkjqt7/36re+4ekc2BbdLNPdVP+coO0OGFL4LdVVTeRRgKV7q54/kj++Yb8czZpymuRpPMl/VPXCLOH915f0r9JulvS0/mYDgS27uF9u977DfRSRJxP6quDScd3APAHSR/PTXYhTefe28NuVkXEfRWvHwE2JP1BA/V/TzuREt3vK+J6iJf7s9Fe6ruIeIE0Iuzqux2BBRHxbEX7W5oUxzrNCxMGr9+Q5ryfBx6JiOd7aLseaTqt1iqqx4GN67zXerndbjXqlgFExJ8kPQZMJiWec0nTVt+WtBOwFd0koYhYI2kf4F2kk+WfBL4uafeIuCu//xmk+f1q3Z2QXkT60Our6svTv9S/ERFp9ir9ERgRd0jaBtiXdM7kYuAuSXtHxIvd7P9E4ATSdOA9pNHC13h1gqn+vQZ9XBUbEcuBq4GrJX0JuJ40Irq0xw1f9kKNWKiIp+jvqd6l/4NX/xFRc6q5job1nXXPSWjwWhkRfynY9g7SifqHuklWyyUtJn2A/qKb7UcCL0bE/T28z6+BfyKdB+qIiCclPUWab695PqhLpPmQm4GbJZ1JOlH/r6RR0h3Ajr04Xkh/gf9I0oFR47yQpOERsUTSI8AkXjkK3JU0nVdY/oC/ArhC0mzgf0nnPhaRzqusX7XJrsA1EXFpjqdrumoJvVNr30XiDUl/Ik2FQjox/0ZJO9UZDfWkx99Tfr/1SNPHv8tlW5NGo5WepOL8lqSRdH++q6/+BBwmaeOK0dA7Gvweg4KzuhVxPmnl0+WS3inpzZL2kjRL0qa5zVeBz0k6TtIOksZLOiHX/ZI0ZTVP0n75OyfvlnSGpMrRUQcp2f0lXl4y2wF8jO6n4pD0LklfkvT2/KH0z8A/8HIiOBM4WNKZkt4qaUdJH5J0Vg/HPJd0MvoHkk7N+36TpPdJuo50rgLSyegTJX00H/eZpBHf2T3suzr+4/P2O0najjTltYx0ng3S+YrdJI2S9PpctgjYU9KuknYknczftuh7Vqi17+r4xkual/tsZ0nbSfokaeHAVbnZr0jTUT+WtG/+He8taUqtfXajx99Tnsr7OWkBxrvz4o/ZpEU1leYDn5E0Uel7TLNJK9ga6Yekc1HfyX2yF/DFXOebtPWCk5DVFRFdf+2/SPoQWEhKTKvyg4iYSTp5fQRpVdzPSSucukYp7yd9OHyHtPJrLjCaV87nd5BG5x11yqotzfFdC/wZmAF8JSLm5Pe/njTCmkw6n/B74AvAX3s45gA+Spru+gDpHNU9wNdJI7Yf56bnkRLRWfm4PwgclKcBi1pOWmH3e9JoYDywX0SszPWnkZLq//HytNS/5/Y/I02triCtLOutWvuu9jBpifJppBHaH0hTgWeTzyPmacP9SH9szCEtLDmXdM6nkIK/p6mkpdvzgWtIyeDBql2dkOPtII0uLyItT2+YPHLdn/Rv/E7Sv4HTc3WjE946rWtVipmZrQVJB5BGhm+IiKfKjmeg8DkhM7M+kHQYacT1N9JKz3NI5+mcgHrBScjMrG9GklbzvZF0mavrSF9atl7wdJyZmZXGCxPMzKw0no7Lhg8fHtttt13ZYdS1YsUKhg4dWr9hyRxnYw2EOAdCjOA4G+32229/KiJG1G9Zm5NQNnLkSG677bayw6iro6OD9vb2ssOoy3E21kCIcyDECI6z0SQ9tDbbezrOzMxK4yRkZmalcRIyM7PSOAmZmVlpnITMzKw0TkJmZlYaJyEzMyuNk5CZmZXGScjMzErjJGRmZqVxEjIzs9I4CZmZWWmchMzMrDROQmZmVhonITMzK01Lk5CkzSRdJWmFpIckHdxNO0maLunp/JguSRX1syTdJ+lFSVNrbH+cpMckLZP0PUkb1YvtwWUvMukb8/nJnYvX6hgHu5/cuZhJ35jP1J+vcH82gPuzcdyXjdXVnxtusd2EtdlPq0dC5wOrgZHAIcBMSWNqtJsGTAHGAWOB/YEjK+rvAo4G7qjeUNK+wBeAPYE3AW8GzigS3OIlz3LKlff4H2cf/eTOxZxy5T0sXvIs4P5cW+7PxnFfNlZ1f64NRUQDQirwRtJQ4O/AWyNiUS67FFgcEV+oavs7YHZEzMqvPwkcERHvqmp3E3BRRMyuKPsh8GBEfDG/3hP4QURs0VN8G71x+3jjYecAsOH667HL1sPX5nCbZsmSJQwf3j9ju/OvS1i95sVXlbs/+2ag9af7srEGSn8+evHnWPXon1Vnk2618vbeOwAvdCWg7C5g9xptx+S6yna1Rky1jAHmVW07UtLmEfF0ZUNJ00ijLjbcYruXyleveZElS5YUfLvWWrNmTb+NrdZ/8q7y/hqz+7Nx3JeNNRD7sy9amYTagGVVZUuBTbtpu7SqXZskRf2hW61tye/ziiSUR1qzII2EuspHDd+Y6z+/R523KUd/vu/8pG/Mrzk8d3/2zUDrT/dlYw3E/uyLVp4T6gSGVZUNA5YXaDsM6CyQgLrblm7e51U23mB9Ttp3dJGmVuWkfUez8Qbrv6LM/dl37s/GcV82Vq3+7KtWjoQWAUMkbR8Rf85l44CFNdouzHW/r9Oulq5t51Zs+3j1VFwto4ZvzEn7jmbKLqMKvpVV6uq3b15/H4uXPOv+XEvuz8ZxXzZWZX8+urY7i4iWPYAfAZcBQ4FJpKmyMTXafRq4FxgFbElKLJ+uqN8QeA3wW+CI/Hy9XPc+4DFgZ2A4MB/4Rr3YdthhhxgIbrjhhrJDKMRxNtZAiHMgxBjhOBsNuC3WIi+0eon20cDGwBOkZHRURCyUtJukzop2FwLXAPcAC4DrclmX/wGeBd5DOqfzLPBegIj4OXAWcAPwV+Ah4MtNPCYzM+ujVk7HERHPkL7/U11+I2lBQdfrAE7Oj1r7aa/zPt8CvrU2sZqZWfP5sj1mZlYaJyEzMyuNk5CZmZXGScjMzErjJGRmZqVxEjIzs9I4CZmZWWmchMzMrDROQmZmVhonITMzK42TkJmZlcZJyMzMSuMkZGZmpXESMjOz0jgJmZlZaZyEzMysNE5CZmZWGichMzMrjZOQmZmVxknIzMxK4yRkZmalcRIyM7PSOAmZmVlpnITMzKw0TkJmZlYaJyEzMyuNk5CZmZXGScjMzErjJGRmZqVxEjIzs9K0NAlJ2kzSVZJWSHpI0sHdtJOk6ZKezo/pklRRP17S7ZJW5p/jK+o2knSBpMclPSPpGkmjWnF8ZmbWO60eCZ0PrAZGAocAMyWNqdFuGjAFGAeMBfYHjgSQtCEwD5gDvA64GJiXywE+C7w7b7cl8HfgP5t0PGZmthZaloQkDQUOAk6NiM6IuAm4Gvh4jeaHATMi4uGIWAzMAKbmunZgCHBORKyKiPMAAXvk+m2B6yPi8Yh4DrgcqJXozMysZIqI1ryRtAvw24jYpKLsRGD3iNi/qu1SYJ+IuCW/ngjcEBGbSjou1+1X0f7aXD8jtz0X+BdgCXAR8EREfK5GTNNIoy5GjBgxYe7cuY096Cbo7Oykra2t7DDqcpyNNRDiHAgxguNstMmTJ98eERP7uv2QRgZTRxuwrKpsKbBpN22XVrVry+eFquuq9/Nn4G/AYmANcA9wTK2AImIWMAtg9OjR0d7eXvBQytPR0YHjbBzH2TgDIUZwnP1N4ek4SSMlnShppqTX57JJkrYtuItOYFhV2TBgeYG2w4DOSMO2evs5H9gI2BwYClwJ/KxgjGZm1kKFkpCkCcB9pMUEn+TlJLA38NWC77UIGCJp+4qyccDCGm0X5rpa7RYCYytXy5EWIXTVjwdmR8QzEbGKtCjhHV2J08zM+o+iI6GzgXMjYhdgVUX59cCkIjuIiBWkUcmZkoZKmgQcAFxao/klwPGSRknaEjgBmJ3rOkjTbMfm5dhdU23z889bgUMlvVbSBsDRwCMR8VSxQzUzs1YpmoQmkJZCV3uUtNy6qKOBjYEngMuAoyJioaTdJHVWtLsQuIZ0PmcBcF0uIyJWk5ZvH0paeHA4MCWXA5wIPEc6N/Qk8H7gg72I0czMWqTowoRnSd/JqbYjKaEUEhHPkBJIdfmNpAUHXa8DODk/au3nTlJirFX3NGna0MzM+rmiI6F5wJclbZRfh6RtgOnAj5sQl5mZDQJFk9CJwGak6a1NgJuAv5Cmw77UnNDMzGxdV2g6LiKWAbtK2gN4Gyl53RERv2xmcGZmtm4rlIQkHQpcHhHzeXkVWtd13D4SEZc0KT4zM1uHFZ2O+z7w2hrlm+Y6MzOzXiuahATUusjc1rz6EjpmZmaF9DgdJ+keUvIJ4NeSXqioXh94E/DT5oVnZmbrsnrnhK7IP99K+sJo5RdKVwMP4iXaZmbWRz0moYg4A0DSg6SFCc+1IigzMxscii7RrnXJHjMzs7VS9CraG0o6Q9IiSc9JWlP5aHaQZma2biq6Ou4r5FtuAy8CJ5Hu2/M06aKkZmZmvVY0CX0Y+HREXEi6jcK8iDgW+DLpnkJmZma9VjQJjQT+mJ93AsPz858D+zQ6KDMzGxyKJqG/Alvm538B9s3P3026zYOZmVmvFU1CVwF75ufnAmdIeoB0t9OLmhCXmZkNAkWXaJ9S8fwKSX8j3dZ7UURc26zgzMxs3Vb0zqqvEBG3ALcASBoaESsaGpWZmQ0KRafjXkXSaySdBDzQwHjMzGwQ6TEJ5S+pflXSrZJ+J2lKLj8UuB/4HPAfLYjTzMzWQfWm404HPgP8gnQO6L8lfYe0SOEU4IcR8XxTIzQzs3VWvST0YWBqRFwlaRxwJ/A6YExEvNDzpmZmZj2rd07oH4BbASLiLtLtG6Y7AZmZWSPUS0IbAKsqXj+P76RqZmYNUmSJ9tclrczPNwROl/SKRJSvI2dmZtYr9ZLQb4C3VLz+HbB1VZtoaERmZjZo1LuzanuL4jAzs0Goz19WNTMzW1tOQmZmVpqWJiFJm0m6StIKSQ9JOribdpI0XdLT+TFdkirqx0u6XdLK/HN81fZvk/QbSZ2SHpf02WYfm5mZ9V6rR0Lnk75rNBI4BJgpaUyNdtOAKcA4YCywP3AkpEsJAfOAOaQvzl4MzMvlSHo96WZ7FwKbA9sB/9O8QzIzs75qWRKSNBQ4CDg1Ijoj4ibgauDjNZofBsyIiIcjYjEwA5ia69pJCyrOiYhVEXEeIGCPXH88cH1E/CDXL4+Ie5t2YGZm1meKqL/CWlL1suwuATwXEU8W2McuwG8jYpOKshOB3SNi/6q2S4F98i0jkDQRuCEiNpV0XK7br6L9tbl+hqT5wD3A20mjoFuAz0TEX2vENI006mLEiBET5s6dW+8wStfZ2UlbW1vZYdTlOBtrIMQ5EGIEx9lokydPvj0iJvZ1+6L3E3qQHr4PJGkZ8H3g5B4u6dMGLKsqWwps2k3bpVXt2vJ5oeq66v1sBbwN2JuUjM4CLiNdgPUVImIWMAtg9OjR0d7e3k3o/UdHRweOs3EcZ+MMhBjBcfY3RZPQR0kf5heQb2YHvJM0ijgdGA58CVgOfLmbfXQCw6rKhuVt6rUdBnRGREiqt59ngasi4lYASWcAT0l6bUT4kkNmZv1I0SR0FHBcRFxZUTZf0n3AZyNid0lPAGfQfRJaBAyRtH1E/DmXjQMW1mi7MNf9vka7hcAJkhQvzyWOJS16ALibV47afEUHM7N+qujChHeSpraqLSCdewG4mTQVVlO+BfiVwJmShkqaBBwAXFqj+SXA8ZJGSdoSOAGYnes6gDXAsZI2knRMLp+ff34f+GBexr0BcCpwk0dBZmb9T9Ek9BD5BH6VI4CuE/4jgGfq7OdoYGPgCdJ5mqMiYqGk3fI0W5cLgWtIiW8BcF0uIyJWk5ZvHwosAQ4HpuRyImI+8MW8zROkxQk1v49kZmblKjoddwLwY0nvJ99fCJhIurjpQfn124Eel5dFxDOkBFJdfiNpwUHX6wBOzo9a+7kTmNDD+8wEZvYUi5mZla9QEoqI6yRtTxrJjM7FVwMXdC19joj/ak6IZma2rio6EiIi/gac0sRYzMxskCmchCRtAowH3kDVuaSqVXNmZmaFFEpCkvYiLSTYvEZ1AOs3MigzMxsciq6OO5e02myriFiv6uEEZGZmfVJ0Om4b4J8j4pEmxmJmZoNM0ZHQb3l5VZyZmVlDFB0JXQCcna9ecA/wfGVlRNzR6MDMzGzdVzQJXZF/zqpR54UJZmbWJ0WT0LZNjcLMzAaloldMeKjZgZiZ2eDTbRKSdCBwTUQ8n593y19WNTOzvuhpJHQFsAXpStRX9NDO54TMzKxPuk1CEbFeredmZmaN4uRiZmal6c0FTLcC3kvtC5h+q8FxmZnZIFD0AqaHAN8DXgCeJJ0H6hKAk5CZmfVa0ZHQmcAM4NSIWNPEeMzMbBApek5oJHCRE5CZmTVS0ST0U+CdzQzEzMwGn6LTcb8ApksaQ+0LmPrLqmZm1mtFk9CF+ecXa9T5y6pmZtYnRa8d5+8TmZlZw9VNLpI2kHSLJN/UzszMGqpuEoqI50m3coh6bc3MzHqj6DTbxcARzQzEzMwGn6ILE4YCh0jaG7gdWFFZGRHHNjowMzNb9xVNQjsBd+Tnb66q8zSdmZn1SdHVcZObHYiZmQ0+XnptZmalKZyEJE2WNEvSzyXNr3z0Yh+bSbpK0gpJD0k6uJt2kjRd0tP5MV2SKurHS7pd0sr8c3yNfWwo6V5JDxeNz8zMWqtQEpI0FfgZsCnQTrqdw+uAtwF/7MX7nQ+sJl0Q9RBgZr4UULVpwBRgHDAW2B84MseyITAPmJNjuBiYl8srnZTjNDOzfqroSOhE4JiI+CjpunGnRMQupETQWWQHkoYCB5FuB9EZETcBVwMfr9H8MGBGRDwcEYtJt5GYmuvaSeeyzomIVRFxHiBgj4r32hb4GPD1gsdnZmYlUET9xW2SVgI7R8SDkp4C9oiIuyXtCHRExBYF9rEL8NuI2KSi7ERg94jYv6rtUmCfiLglv54I3BARm0o6LtftV9H+2lw/o+L1d4G/A3MiYqtuYppGGnUxYsSICXPnzq3bF2Xr7Oykra2t7DDqcpyNNRDiHAgxguNstMmTJ98eERP7un3RJdpPk6biABYDbwXuBjYHNi64jzZgWVXZ0or9VrddWtWuLZ8Xqq57xX4kfRBYPyKuktTeU0ARMQuYBTB69Ohob++xeb/Q0dGB42wcx9k4AyFGcJz9TdEkdCOwD+k2DnOB8/IXV/ck3eahiE5gWFXZMGB5gbbDgM6ICEnd7idP+Z0FvL9gTGZmVqKi54SOAS7Lz78OfJM0CpoLfKrgPhYBQyRtX1E2DlhYo+3CXFer3UJgbOVqOdLihYXA9sA2wI2SHgOuBN4o6TFJ2xSM08zMWqTol1WfqXj+IjC9t28UESskXQmcKelTwHjgAOA9NZpfAhwv6aekKzKcAPxnrusA1gDHSrqAl69pNx94EfiHiv28B/g2aRWfV8qZmfUzvfme0EhJJ0qaKen1uWxSXolW1NGkc0hPkEZWR0XEQkm75Wm2LhcC15Cm/xYA1+UyImI1afn2ocAS4HBgSkSsjogXIuKxrgfwDPBifr2mF3GamVkLFBoJSZoA/Ap4ABhDmo57Ctgb2AGo+aXTanlENaVG+Y2kBQddrwM4OT9q7edOYEKB9+sAaq6MMzOz8hUdCZ0NnJu/G7Sqovx6YFLDozIzs0GhaBKaQLoyQbVHSVc/MDMz67WiSehZ0iVyqu1IOr9jZmbWa0WT0Dzgy5I2yq8jL3meDvy4CXGZmdkg0Jtrx21GWua8CXAT8BfSlQq+1JzQzMxsXVf0e0LLgF0l7UH6zs16wB0R8ctmBmdmZuu2opftASAi5pO+FAqApDcB34yIDzc6MDMzW/et7Z1Vh5Nuz2BmZtZrvr23mZmVxknIzMxK4yRkZmal6XFhgqSr62xffV8fMzOzwuqtjnu6QP0DDYrFzMwGmR6TUER8olWBmJnZ4ONzQmZmVhonITMzK42TkJmZlcZJyMzMSuMkZGZmpXESMjOz0jgJmZlZaZyEzMysNE5CZmZWGichMzMrjZOQmZmVxknIzMxK4yRkZmalcRIyM7PSOAmZmVlpWpqEJG0m6SpJKyQ9JOngbtpJ0nRJT+fHdEmqqB8v6XZJK/PP8RV1J0laIGm5pAckndSKYzMzs95r9UjofGA1MBI4BJgpaUyNdtOAKcA4YCywP3AkgKQNgXnAHOB1wMXAvFwOIODQXPc+4BhJH2nWAZmZWd+1LAlJGgocBJwaEZ0RcRNwNfDxGs0PA2ZExMMRsRiYAUzNde2kO8KeExGrIuI8UuLZAyAizoqIOyLihYi4j5SwJjXx0MzMrI8UEa15I2kX4LcRsUlF2YnA7hGxf1XbpcA+EXFLfj0RuCEiNpV0XK7br6L9tbl+RtV+BNwBXBgRF9SIaRpp1MWIESMmzJ07t0FH2zydnZ20tbWVHUZdjrOxBkKcAyFGcJyNNnny5NsjYmJftx/SyGDqaAOWVZUtBTbtpu3SqnZtOalU1/W0n9NJo73v1wooImYBswBGjx4d7e3tPR5Af9DR0YHjbBzH2TgDIUZwnP1NK5NQJzCsqmwYsLxA22FAZ0SEpEL7kXQM6dzQbhGxam0CNzOz5mjlwoRFwBBJ21eUjQMW1mi7MNfVarcQGFu5Wo60eOGl/Ug6HPgCsGdEPNyA2M3MrAlaloQiYgVwJXCmpKGSJgEHAJfWaH4JcLykUZK2BE4AZue6DmANcKykjfKIB2A+gKRDgK8Be0fE/c06HjMzW3utXqJ9NLAx8ARwGXBURCyUtFueZutyIXANcA+wALgulxERq0nLtw8FlgCHA1NyOcC/A5sDt0rqzI9XLUowM7PytfKcEBHxDCmBVJffSFpw0PU6gJPzo9Z+7gQmdFO3bUOCNTOzpvNle8zMrDROQmZmVhonITMzK42TkJmZlcZJyMzMSuMkZGZmpXESMjOz0jgJmZlZaZyEzMysNE5CZmZWGichMzMrjZOQmZmVxknIzMxK4yRkZmalcRIyM7PSOAmZmVlpnITMzKw0TkJmZlYaJyEzMyuNk5CZmZXGScjMzErjJGRmZqVxEjIzs9I4CZmZWWmchMzMrDROQmZmVhonITMzK42TkJmZlcZJyMzMSuMkZGZmpWlpEpK0maSrJK2Q9JCkg7tpJ0nTJT2dH9MlqaJ+vKTbJa3MP8cX3dbMzPqPVo+EzgdWAyOBQ4CZksbUaDcNmAKMA8YC+wNHAkjaEJgHzAFeB1wMzMvlPW5rZmb9S8uSkKShwEHAqRHRGRE3AVcDH6/R/DBgRkQ8HBGLgY1ntjwAAAf3SURBVBnA1FzXDgwBzomIVRFxHiBgjwLbmplZPzKkhe+1A/BCRCyqKLsL2L1G2zG5rrLdmIq6uyMiKurvzuU/r7PtK0iaRho5AayStKDYoZTq9cBTZQdRgONsrIEQ50CIERxno41em41bmYTagGVVZUuBTbtpu7SqXVs+t1NdV72fbretSlxExCxgFoCk2yJiYvHDKYfjbCzH2TgDIUZwnI0m6ba12b6V54Q6gWFVZcOA5QXaDgM6cxKpt5+etjUzs36klUloETBE0vYVZeOAhTXaLsx1tdotBMZWrXgbW1Xf3bZmZtaPtCwJRcQK4ErgTElDJU0CDgAurdH8EuB4SaMkbQmcAMzOdR3AGuBYSRtJOiaXzy+wbU9m9f6oSuE4G8txNs5AiBEcZ6OtVZxq5SyVpM2A7wF7A08DX4iIH0raDfhZRLTldgKmA5/Km14EfL5rSk3SLrlsZ+Be4JMRcWeRbc3MrP9oaRIyMzOr5Mv2mJlZaZyEzMysNIMqCTXq2nX9JMbTJT0vqbPi8eZWxJjf/xhJt0laJWl2nbbHSXpM0jJJ35O0UYvCLBynpKmS1lT1Z3uLYtxI0nfz73u5pD9I2q+H9qX0Z2/iLLM/8/vPkfRo7qNFkj7VQ9sy/30WirPs/swxbC/pOUlzuqnv2+dmRAyaB3AZcDnpC627kr7IOqZGuyOB+4CtgFHAH4FP97MYTwfmlNiXB5Ku0TcTmN1Du32Bx0lXrXgdaXXjN/phnFOBm0rqy6H597kN6Q/DD5C+97ZNf+rPXsZZWn/m9x8DbJSf7wg8BkzoT/3ZyzhL7c8cw/8AN3b3udPXz81BMxJS465d119iLFVEXBkRPyGtcuzJYcB3I2JhRPwd+AotvJZfL+IsTUSsiIjTI+LBiHgxIq4FHgAm1GheWn/2Ms5S5f5Z1fUyP95So2nZ/z6LxlkqSR8BlgC/6qFZnz43B00Sovtr19W6rlzh6881WG9iBNhf0jOSFko6qvnh9UmtvhwpafOS4unJLpKeytMip0pq5WWtXiJpJOnfQq0vWfeb/qwTJ5Tcn5L+S9JK4E/Ao8BPazQrvT8Lxgkl9aekYcCZwPF1mvbpc3MwJaFGXbuumXoT41xgJ2AEcARwmqSPNje8PqnVl1D7mMr0G+CtwBtIo9GPAie1OghJGwA/AC6OiD/VaNIv+rNAnKX3Z0QcTeqX3UhflF9Vo1np/VkwzjL78yuk0eLDddr16XNzMCWhRl27rpkKxxgRf4yIRyJiTUT8DjgX+FCT4+uLWn0Jtfu9NBFxf0Q8kKeZ7iH95dfS/pS0HukKIquBY7ppVnp/FomzP/RnjmNNntbeCqg1W1B6f0L9OMvqT6Ubhu4F/EeB5n363BxMSahR165rpt7EWC1I91Xqb2r15eMR0W/P0WQt7c/81+J3STd8PCginu+maan92Ys4q5X973MItc+19Ld/n93FWa1V/dlOWojyV0mPAScCB0m6o0bbvn1ulrnaotUP4Eek1WdDgUl0v/Ls06TLAY0Ctswd2arVcUVjPIC0mkfAO4DFwGEt7MshwGuAr5P+Kn4NMKRGu/eRVvzsDAwnXeOvlauPisa5HzAyP98RWAB8uYVxXgD8L9BWp13Z/Vk0ztL6kzRl9RHS9ND6pBVwK4B/7k/92cs4S+lPYBNgi4rH2cAVwIgabfv0udmSf7j95QFsBvwk/6L/Chycy3cjDRu72gk4C3gmP84iX+KoH8V4GWnFVyfphOaxLe7L03l5NU/X43Rg6xzT1hVtjyctg10GfJ+8JLU/xZn/cz2e+/1+0nTHBi2K8U05rudyTF2PQ/pTf/YmzpL7cwTwa9JqrmXAPcARua4/9WfhOMvsz6qYTycv0a7xmdSnz01fO87MzEozmM4JmZlZP+MkZGZmpXESMjOz0jgJmZlZaZyEzMysNE5CZmZWGichs3WUpJDUHy/lZPYSJyGzJpA0OyeB6sf/lh2bWX9SyqXqzQaJX/Lqe0GtLiMQs/7KIyGz5lkVEY9VPZ6Bl6bKjpF0naSV+ZbZH6vcWNI/SvqlpGfzfaNmS3ptVZvDJN2jdPvyxyVdXBXDZpL+W+l28fdXv4dZ2ZyEzMpzBunOueOBWcAlkibCS3fZvZ50/bB3AB8E3gN8r2tjSUcCF5KudzYWeD/pwpaVTgPmka5ofDnwPUlbN++QzHrH144zawJJs4GPkS72Wen8iPi8pAAuiogjKrb5JfBYRHxM0hGki1ZuFRHLc307cAOwfUT8RdLDpItJfqGbGIJ0RehT8ushpAtlTouIOQ08XLM+8zkhs+b5DTCtqmxJxfObq+puBv4pP98JuLsrAWW/A14Edpa0jHTJ/F/VieHuricR8YKkJ0m3EDDrF5yEzJpnZUT8pQn77c30RfVN5wJPw1s/4n+MZuV5V43X9+bn9wL/KGnTivr3kP7P3hsRT5BuZLhn06M0ayKPhMyaZyNJW1SVrYmIJ/PzAyXdCnQAHyIllHfmuh+QFi5cIuk00l10LwSurBhdfRX4D0mPA9eR7oK5Z0TMaNYBmTWak5BZ8+wFPFpVthjYKj8/HTgIOA94EvhERNwKEBErJe0LnAP8nrTAYR7w2a4dRcRMSauBE4DppLtZ/rRZB2PWDF4dZ1aCvHLtXyLiirJjMSuTzwmZmVlpnITMzKw0no4zM7PSeCRkZmalcRIyM7PSOAmZmVlpnITMzKw0TkJmZlaa/w92dAZgZHew2wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plotting the learning rate by epoch for piecewise scheduling\n",
    "plt.plot(history.epoch, [piecewise_constant_fn(epoch) for epoch in history.epoch], \"o-\")\n",
    "plt.axis([0, n_epochs - 1, 0, 0.011])\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Learning Rate\")\n",
    "plt.title(\"Piecewise Constant Scheduling\", fontsize=14)\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Performance Scheduling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.random.set_seed(42)\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 25000 samples, validate on 5000 samples\n",
      "Epoch 1/5\n",
      "25000/25000 [==============================] - 3s 109us/sample - loss: 0.6731 - accuracy: 0.7848 - val_loss: 0.5307 - val_accuracy: 0.8226\n",
      "Epoch 2/5\n",
      "25000/25000 [==============================] - 2s 96us/sample - loss: 0.5291 - accuracy: 0.8264 - val_loss: 0.5382 - val_accuracy: 0.8398\n",
      "Epoch 3/5\n",
      "25000/25000 [==============================] - 2s 95us/sample - loss: 0.4859 - accuracy: 0.8400 - val_loss: 0.4934 - val_accuracy: 0.8408\n",
      "Epoch 4/5\n",
      "25000/25000 [==============================] - 2s 95us/sample - loss: 0.4593 - accuracy: 0.8492 - val_loss: 0.5844 - val_accuracy: 0.8150\n",
      "Epoch 5/5\n",
      "25000/25000 [==============================] - 2s 93us/sample - loss: 0.4765 - accuracy: 0.8508 - val_loss: 0.5583 - val_accuracy: 0.8328\n"
     ]
    }
   ],
   "source": [
    "# using ReduceLROnPlateau callback for performance scheduling\n",
    "lr_scheduler = keras.callbacks.ReduceLROnPlateau(factor=0.5, patience=5)\n",
    "\n",
    "# defining the model\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.Dense(300, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n",
    "    keras.layers.Dense(100, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n",
    "    keras.layers.Dense(10, activation=\"softmax\")\n",
    "])\n",
    "optimizer = keras.optimizers.SGD(lr=0.02, momentum=0.9) # using momentum optimizer\n",
    "\n",
    "# compiling the model\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])\n",
    "n_epochs = 5\n",
    "\n",
    "# training the model\n",
    "history = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n",
    "                    validation_data=(X_valid_scaled, y_valid),\n",
    "                    callbacks=[lr_scheduler])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdoAAAEeCAYAAAAtndQNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOydd3hVVdaH3x8tNJGmYAPEAoqK2BuKvYzYu6OABQR1ZqxjHSvjx8xYKSI2sGNBAVHQcUDF7oggWBhFsVBUEDB0yPr+2CfkcHOTewIhNzdZ7/PsJ/fsdtY5hKy7915FZobjOI7jOBuGGtkWwHEcx3GqMq5oHcdxHGcD4orWcRzHcTYgrmgdx3EcZwPiitZxHMdxNiCuaB3HcRxnA+KK1nFiSDpFkvu8lTOSukgySc2zLYvjVDSuaJ2cQ9LQ6I+2SVol6XtJ90tqkm3ZyovoGV8upf272DtYKulLSVdJUkXKGZNnQkye5ZKmS7pOUs31nHNAecrpONnAFa2Tq/wb2AxoA1wAdAUGZVOgLHAr4R3sAPwL+DvQM4vyPBrJ0w64D7gduDKL8jhOpcAVrZOrLDezOWb2o5m9BgwHjoh3kLSxpCGSfpb0u6Q3Je2R0udcSTMlLYlWkC1S2m+WNDWlrruk/JS6YyR9EK0u50kaLalu1FZHUj9JP0b3+UjSkeXwDn6P3sF3ZvYQMCX1HaQiKU/SPZLmSlom6X1JB8TaC7d4D42eZ4mkjyXtlkCeJTF5BgBvACeUIEczSU9H72SppGmSesTahwIHARfHVsptorYdJY2J/k1/juZpGRu7p6TXJP0qaZGkiZL2Tbm/STolpe47Sf7FwCl3XNE6OY+ktsBRwMpYnYAxwBbAsUAn4C3gP5I2i/rsDQwFhgC7AqMJq8Sy3v8oYBTwOrA7cDDwJkX/vx4lKI2zgJ2AYcBoSR3Leq8S7i9JXQgr25UZuv8DOB04j/BOPgPGFr6TGHcA1wC7AfOAJ9dhW3opULuEtrrAJ4R/mw7AvcADkg6N2v8MvEfRKnkz4IdIzreAqcBewGFAQ2CkpML3vRHwONA56vMp8IqkZmWU33HKBzPz4iWnCkE5rgLyCX/MLSqXxfocErXXSxn7KXB19Pkp4PWU9ofCf4s11zcDU1P6dAfyY9fvAM+UIOs2QAHQKqX+JWBQhmd8uZT274Dl0TOuiJ5/KbBfKWMaRH3PjdXVBL4Bbo+uu0RzHRnrs39Ut2Upc08ABkSfaxC++CwH+qXM27yUOZ4BHko3Z6zuVuCNlLom0dx7lTCvgNnAH2N1BpyS5p1eme3fby9Vr/iK1slV3iKsQvcC+gOvEM4FC9kdqA/8Iim/sBBWlNtEfXYgrJripF4noRNhmzQduxH+0H+eIscfYnKsK3cR3sFBwHjgFjN7t5T+2xBWmO8UVpjZasIz75jSd0rs86zo56YZ5OkZPdsywgr/CeCWdB0l1ZR0vaQp0VZ7PnAS0CrDPXYHDkx5lz/Eng9Jm0p6IDLIWgj8HsmeaW7H2SDUyrYAjrOOLDGzr6PPf5I0HriRsAKFsKqaS9g+TGVRGe5TQFCUcUraDk1HDcLqaU+Kb+suLcM86ZgXvYOvJZ0M/E/SB2Y2fh3mSnVpWpmmLdMX8+EExbocmBUp8ZK4EriCsEX8GWFl/ncyK/MahCOBdGepc6Ofwwhn7ZdRtPJ/A6gT62us37+r4yTGFa1TVbgFeFXSEDObRTj/awEUmNmMEsZ8AeyTUpd6/QvQQpLMrFDh7JrSZxJwKPBgmntMIvxBb7mOCjARZvZb5Apzt6ROMVnjfEPYOt4/+kzkfrMvYRt9fVkY+/KTiQOA0Wb2eCSHgO2BBbE+Kwhb23E+AU4DZppZSefRBwB/MrMx0dwtCGe8cX6J15XQx3HKBd86dqoEZjYB+By4Iar6N2GLdKSkoyVtLWlfSbdIKlzl3gccJulaSdtJuhA4MWXqCUBT4DpJ20g6HzglpU9f4FRJt0cWsR0kXSapvplNB54EhioEw2graQ9JV0o6KcNjNZK0a0ppU0r/QQTXmlNLeEeLgfuBfgpW0jtE1y2oeNeo6cChkg6Q1B4YAGyd0uc7YC9JbSQ1j4ydBgIbA8Ml7R29z8MUrMs3is39x+jfYk/C2e+KlLn/Q7Bo3kNSJ8KZ+LIN8aCO44rWqUrcCZwvqXW0ojuG8Af1QeAr4FmCIpoFYGbvA+cDvQlnkidRtPVM1OeLqL1n1OdwwhZnvM8rBAV9NGEF+ybB8rgg6tKDYD37D+BL4GXgQGBmhufpHM0XL/8qqbOZ/Uywtr05ZoGbyl8JW7yPEgzDdgGOMrPZGWQpb24HPgReJZy3LyZ8IYnzL4KC/JywAm0V7VbsT3i3Y4FpBOW7PCoQLKobAv8lKNlHCEo7zhXADMIXqecJRnA/l9OzOc5aKP0Ok+M4juM45YGvaB3HcRxnA+KK1nEcx3E2IK5oHcdxHGcD4orWcRzHcTYg7kdbDtSoUcPq1auXbTEyUlBQQI0alf+7lctZvric5UcuyAi5I+eSJUvMzCq/oOuJK9pyoE6dOixevDjbYmRkwoQJdOnSJdtiZMTlLF9czvIjF2SE3JFT0vpGR8sJqvw3CcdxHMfJJq5oHcdxHGcD4orWcRzHcTYgrmgdx3EcZwPiitZxHMdxNiAVqmglmkq8KLFYYqbEWSX0k0Q/iXlR6SeF3JES20uMlPhFYr7EOIl2sbE7RXW/SsVybGaUQeKsqH6xxEsSTcv7PTiOk4PMns2uf/4zzJmTbUmcHKOiV7QDCdk4WgBnA/dLdEjTrydwAtCRkF2kK9AramsMjCJkYWlByAAyMjZ2JSFLy/lllSH6+QBwTtS+hIpPH+Y4TmXkttvY+LPP4Lbbsi2Jk2NUmKKVaACcDNxoRr4ZEwkK85w03bsBd5rxoxk/EdKfdQcw40MzHjZjvhkrgbuBdhLNovavzHiYkD6rrDKcDYw24y0z8oEbgZMkNkqdy3GcasTs2fDoo8gMHnnEV7VOmajIgBXbA6vMmB6rmwwclKZvh6gt3i/dyhdCXs85ZswrBxk6AO8WNpjxjcSKaNx/4xNJ9CSsvKlVS0yYMCHB7bNLfn6+y1mOuJzlS2WWc7u772bzFSsQYMuWMe/kk5nat2+2xSqRyvwuqyMVqWgbAotS6hZC2tViw6gt3q+hhMyKzl0ltiRsBV9eTjKk3rdEGc0YAgwBqFvXLBeisORKtBiXs3xxOdeT2bPhtdegoAAAAc3ffZcuw4bBgAHQoEF25UtDpX2X1ZSKPKPNBxql1DUCfk/QtxGQn6JkNwFeAwaZ8XQ5yVAWGR3HqQ7cdhusWrV2XY0aMHQodOwIEydmRSwnd6hIRTsdqCWxXayuI2nOUqO6jiX1k2hCULKjzCjL/k0mGda6r0RbIC8a5zhOdeS994or2oIC2Hbb8PPAA+HKK2HZsuzI51R6KkzRmrEYGAHcKtFAYn/geODxNN0fAy6X2EJic+AKYCiARCNgHPCOGdekDoxcg+oCdaLruhJ5CWV4Eugq0TkynLoVGGHmK1rHqbY8/HD4ec89TBg/HsxC+d//YPJk6NkT7rwTdtsNPvoou7I6lZKKdu/pA9QDfgaeBnqbMS1SbPmxfg8Ao4HPgKnAmKgO4ERgT6CHRH6stIraWwNLKVqlLgW+yiQDQPTzIoLC/ZlwNtunvB7ecZwcpPActnv34m0bbQSDB8PYsbBoEey7L/ztb7BiRYWL6VReKjRNnhnzCf6xqfVvEwyRCq8NuDoqqX2HAcNKucd3BHuFMskQa38KeKqkdsdxqhHz5sFTT8F558HGG5fc78gjYepU+POfw5nu6NEwbBjsskvFyepUWjwEo+M4Tkk89BAsXw4XX5y5b+PGQbm+9BLMmgV77AF33FH8fNcpG1JTpBeRFiPNREobURDpZqSVSPmx0jbWfgjSJ0iLkGYg9ayoR3BF6ziOk47Vq2HQIDjkEOhQkht/Go4/HqZNgxNOgOuug/33hy+/3HByVn2KRfNDKukfZDhmDWNlBgBSbeBFwhHkxsDpwF1IHUuYp1xxRes4jpOO0aPh++/hkkvKPrZ5c3j2WXjmGfj6a+jUCe6+e40vrpMQaU00P8zyMSstomBpNCW4aj6OmWH2EfAFsGO5ylsCrmgdx3HSMWAAbLUVdO267nOcfno4uz3sMLj8cjj4YJgxo/xkzHGaQy2kj2MldTt3e2AVZqnR/Epa0XZFmo80Dan3mlqzuQTj1x5INZH2JRjOVogTtCtax3GcVL74At54A/r0gVrraTO62WYwalSIkfzpp8FA6oEHgotQNefXoET3iJUhKV3KElHwWWAHYBPgQuBvSGfG2p8G/gYsB94Grsfsh/J4jky4onUcx0llwADIy4MLLiif+STo0QM++yy4AF10ERx1FPz4Y/nMX3VJHq3P7HPMZmG2GrN3gXuBUwCQ2gPPAOcSYix0AK5G+sOGE70IV7SO4zhxFi4M1sNnnhnOWsuTVq1g3DgYODCEbtxpJ3jsMV/dlsx0wvZykoiCqRhFrp47AdMxG4dZAWZfEeIzHF2u0paAK1rHcZw4w4bB4sXrZgSVhBo1wpb05Mmw887QrRuceCLMnbth7pfLmK2J5ofUAKnkiILS8UhNkIS0F/AninKVTwK2i1x8hLQNcCwwpSIewxWt4zhOIQUFYbW5zz6w++4b9l7bbgsTJoTwjWPHBhei557bsPfMTYpF88NsGlJnpHhEwTOArwnbyo8B/TALwY3MvgHOA+4jnPm+CbwAPFQRD1ChkaEcx3EqNf/+N0yfDk8+WTH3q1kzWCMffTScey6cdhqccUY4I27WrGJkqOyYpY/mZ7ZWREHMzizWZ+3+zxIMpiocX9E6juMU0r8/tGgBp5xSsffdYYeQJei22+CFF8LZ7ejRFSuDs8FwRes4jgPBv3XMGOjVC+rUqfj716oFN9wAH34Im24Kxx0XYiwvXFjxsjjliitax3EcCOEWa9YMijab7LprULbXXRcMs3beOWxpOzmLK1rHcZwlS0Le2ZNOgs03z7Y0wYe3b194912oXx8OPzwkNsjPzzzWqXS4onUcx3nqKViwAC69NNuSrM3ee8OkSXDZZXD//WG1O7FCogY65UiFKlqJphIvSiyWmCmRNt2RhCT6ScyLSj8pOB5LbC8xUuIXifkS4yTapYy/TGKOxCKJRyTyovpWKcni8yVM4oqovYtEQUp7tw39XhzHySJmwQiqY8eQaaeyUa8e3HVXcAUqKIADD4Qrr4SlS7MtmZOQil7RFkt3JKUNDt2TYM7dEdgF6AoUHpw0JmRvaBfN8yFFTslIHAlcAxxKCBrdFrgFwIzvzWhYWICdgQKCP1Uhs+J9okTzjuNUVSZOhClTwmpWytw/Wxx4YJCzV6/ge7vbbvDRR9mWyklAhSlaiTXpjszIN6O0dEfdgDvN+NGMn4A7ge4AZnxoxsNmzDdjJXA30E6iWWzsw2ZMM+M34LbCsWk4F3jLjO/K5SEdx8k9+veHJk1CyMXKTsOGYQt53LhwXrvvvnDjjbBiRbYlc0pBVkExNiU6Ae+YUT9WdyVwkBldU/ouBI4w44Poeg9gvFnxjA0SJwD3m7FZdD0Z+LsZw6Pr5sAvQHMz5sXGiRBF5DYzhkZ1XYDXgN+AJcBLwA1mLE5z356ElTe1atXb/fXXX12X11Kh5Ofn07Bhw8wds4zLWb64nCVT55df2PeMM/jh1FOZcdFFGftXpndZKz+fbQcMoOW4ceRvsw1fXHsti7fZBqhccpbGwQcfvMTMGmRbjg1OyIG74QtYZ7A5KXUXgk1I03c1WPvY9XbhIMWU0m9LsJ/AzozVfQN2VOy6djS2TRp58sEaxupagu0IVgNsa7C3wB7I9Gx5eXmWC4wfPz7bIiTC5SxfXM5SuPFGM8lsxoxE3Svlu3zpJbNNNzWrXdusb1+zlSsrp5xpABZbBemgbJaKPKNNnu6oeN9GQL4Za5bfEpsQVp+DzHg6w1jS3Kcb8IIZa+zlzZhjxudmFJjxLXA1YbvbcZyqxvLlIS/sscfC1ltnW5p15/jjYdq0kJjg+uth//2p//332ZbKiVGRinY6UEsiSbqjaVFb2n4STQhKdpQZfROMnZuybVwPOBUyGjoZ7gLlOFWT55+Hn3+ufC4960Lz5jB8ODzzDHz9NbtfeCHcfXewUnayToUpkeiccwRwq0QDiZLTHYXMC5dLbCGxOXAFrDlHbQSMI5z3XlPC2PMldpRoDNxQODbGiYRz2PHxSomDJVpH7kVbAf9HzKLZcZwqRP/+0K4dHHpotiUpP04/HaZN47fddw/JCg4+OISWdLJKRa/WiqU7MmOaRGeJeMiTB4DRwGfAVEKC3geithOBPYEeKf6urQDMGAv8g6BEvwdmAjelyNENeDy+FR3RCXgXWBz9/IyQ09BxnKrERx/BBx+EaEs1qtimVcuWTO3bFx59FD79FHbZBQYP9uTyWaRC0+SZkTbdkRlrpTuKFODVUUntO4wMW75m3AXcVUr7kesyznGcKsLAgcFVplsVjUcjQffuYbV+3nnQuze8+CI89BBstVW2pat2VLGvco7jOBn45ZdwltmtGzRKtc+sYmy1Fbz2WkiYMHFiSFAwbJivbisYV7SO41QvHnooWBxfckm2JakYpLCinTIlbCN37w4nnABz5mRbsmqDK1rHcaoPq1aFyEqHHQbt22dbmoplm21g/PgQvnHcuJBc/rnnsi1VtcAVreM41YdRo+CHH6rPajaVmjWDNfKkSdC2LZx2GpxxBsybl3mss864onUcp/owYAC0bh2CVFRndtgh5Lq9/XYYMQI6dIDRo7MtVZXFFa3jONWDqVPD1mmfPmFlV92pVStEkvroI2jRAo47Dnr0gIULsy1ZlcMVreM41YOBA6FuXTj//GxLUrno2DEo2+uvh8ceC5bJr7+ebamqFK5oHcep+ixYEJTIWWdBs2aZ+1c36tQJ28jvvQcNGsARR4SVf35+5rFORlzROo5T9Rk6FJYsqb5GUEnZay/45JNgMDV4cFjtvv12tqXKeVzROo5TtSkoCNvG++8PnTplW5rKT716wQXozTfD9UEHwZVXwtKl2ZUrh3FF6zhO1WbcOPj6a1/NlpXOnWHyZLjooqB4d9sNPvww21LlJK5oHcep2gwYAJttBiedlG1Jco+GDUP4xtdeC+e1++0HN9wAK1ZkW7KcwhWt4zhVl6+/hldfhV69gsGPs24cfjh89hmccw707RvOcidPzrZUOYMrWsdxqi6DBgWf2Z49sy1J7tO4cUi9N3JkiJO8555B6a5alW3JKj2uaB3HqZosXgyPPAKnnhq2jp3y4bjjYNq0sBV/ww1hO/nLL7MtVaWmQhWtRFOJFyUWS8yUOKuEfpLoJzEvKv0kFLVtLzFS4heJ+RLjJNqljL9MYo7EIolHJPJibd9JLI0ljH8t6VjHcXKIJ54IUY7cCKr8adYspBp85hn45ptgzX333cHC2ylGRa9oBwIrgBbA2cD9Eh3S9OtJSBDfEdgF6Ar0itoaA6OAdtE8HwIjCwdKHAlcAxwKtAbaArekzN/VjIZROaKMYx3HqeyYBSOo3XaDfffNtjRVl9NPD6vbI44IvrddugTFW1WRNkXqjrR3WYZVmKKVaACcDNxoRr4ZEwkK85w03bsBd5rxoxk/AXcC3QHM+NCMh82Yb8ZK4G6gnUSz2NiHzZhmxm/AbYVjE7A+Yx3HqSy8+WaIbXzJJSEfq7PhaNkSXnopBAWZPDkEuRg8uGokl5deQfpL9LkB8DFwHzAR6ezE01gFvQyJTsA7ZtSP1V0JHGRG15S+C4EjzPggut4DGG/GRmnmPQG434zNouvJwN/NGB5dNwd+AZqbMU/iO6Ae4UvGJOAqMyYnGZty356ElTe1atXb/fXXX12v91MR5Ofn07Bhw2yLkRGXs3ypjnJ2uOkmGn/6Ke89+ywFeeV3+lMd32VZyPv5Z9r98580/fhj5u+xB19ddRXLN920xP4HH3zwEjNrUOqkUlPgYeAI4FfgWsyeStPvZuB6YHmsdhfMZkTtNQk7lOcBGwFfAwdjtqCUe/8MHIbZFKRzgOuAXQk7sn/GrGOpshdiZhVSwDqDzUmpuxBsQpq+q8Hax663C1+PTCn9tgT7CezMWN03YEfFrmtHY9tE1/uD1QOrD3Yt2BywxknGllTy8vIsFxg/fny2RUiEy1m+VDs5v//erGZNs7/+tXzmi1Ht3uW6UFBgdv/9Zg0amDVqZPboo6EuDcBiy6Q/4GmD4QYNDQ4wWGjQIU2/mw2eKGWe2w3+Y9DaQAY7GdTNcO9lBltFnx83+Hv0uZUlkT0qibeOJVpIXClxf7TSQ2J/ia0TTpEPNEqpawT8nqBvIyA/PN8aeTYBXgMGmfF0hrEU3seMd8xYasYSM+4AFgCdk4x1HCcHKNy2vOiibEtSPZHCu58yJWwj9+gBxx8fXILKPteaI0fM8jEr7cixtHmaAH8BLsRsZqQBp2K2LMPI74F9keoDRwKFaY2aAEuS3j6RopXYHfiKsFw+nyIFdDjQN+G9pgO1JLaL1XUEpqXpOy1qS9tPoglByY4yK3b/dGPnpm79xjCg8BCnrGMdx6lMLFsGQ4ZA167Qpk22panetG0LEybAXXeFyFIdOsCzz67VpTnUQvo4VlIdnrcHVmE2PVY3GdIa0QJ0RZqPNA2pd6x+Z2AVcArSHKTpSBcneIp7gCcICncuEAWA5kBgaoLxQHJjqH8B95rRibX3v8cB+yeZwIzFwAjgVokGEvsDxwOPp+n+GHC5xBYSmwNXAEMBJBpF933HjGtKGHu+xI4SjYEbYmNbRavwOhJ1Ja4CmgPvZBrrOE4O8Oyz8OuvcOml2ZbEAahRAy67DD79FLbdNlgpn3EGzJsHs2fTDGphtkesDEmZoSGwKKVuIRS31wGeBXYANgEuBP6GdGbUtiWwMUFxbw2cAtyMdHip8psNIux49gb2w6zQf2km8LcEbwBIrmh3B4alqZ9NcLFJSh+CIdLPwNNAbzOmSXSWiCc+fAAYDXxG+NYwJqoDOBHYE+gR84XNl2gFYMZY4B/AeMK3kJnATdHYjYD7gd+An4CjgKMLV6wZxjqOU9kZMAB22AEOOSTbkjhx2reHd94JkaRGjAir2/POo2FmHZT8yNHsc8xmYbYas3eBewkKFaAw9dCtmC3FbArwDHBMRtnNPsDsOczCPaWamI3CLHH+wKSKdilhTzqV9gSlmQgLLjknmNHAjFZmPBXVv21Gw1g/M+NqM5pG5erC81kzhpmhaI6GsfJ9bPxdZrQwo5EZPczCKtyC284u0dhmZhxqxscpMqYd6zhOJefDD+Gjj9ylp7JSqxZcd134N2rWDMaOTTJqOmF7OcmRYyrxY8EpsTrSfE6P1AfppNj1A8CyaGt6u5IHrk1SRTsSuCkWJckk2gD9gBeS3sxxHGeD0b8/bLRRCHzvVF46doQDDggxqDNhtubIEakBUslHjtLxSE2QhLQX8CcKgxmZfQO8DVyPlIe0A3AG8HIGCS6DyEZH6kywU+oGfE6I75CIpIr2SqApwae0PjCR4IO0gHCO6TiOkz3mzg3ns927B2XrVF5mz4bHHoPVq5OOKHbkiNk0pM5I8SPHMwh66XeCvU0/zOJHnmcSIv7NIxxH3ojZGxnuvSUwI/rcFXg+8uG9Cdgv6QPUStLJjEXAARKHALsRFPQnZvw76Y0cx3E2GA8+GHKkXpzEkNTJKrfdVraYyGbzCSF5U+vfhqIjR8zOLNZn7f6Fdjll4XeCcdUPBC+bwlXsCqBu0kkSKVqJc4HhZvwH+E+svg5whhmPJb2h4zhOubJyZfCdPeIIaNcuc38nu7z3Xi4ljn8deADpE4LFcmEIwB2B75JOknTr+FGCaXQqG0VtjuM42WHkSPjpJ3fpyRUmTYoC7hn/LUPQhyxxMSG+8ZbAaZgVxlTYE0Ko3iQkWtESLLfSWWi1Ivg0OY7jZIf+/WHrreHoo7MtiVPVCHGQe6epv7Es05SqaCU+IyhYA96UWBVrrkk4WH6lLDd0HMcpN6ZMgbfegn/+M5kVq+OUFakOwdBqR4IunAY8i1ni/e9MK9rno587Eay04hZeKwh71O7e4zhOdhgwAOrVg/POy7YkTlVEag+MJXjdFPruXgzchnQUZl8lmaZURWsWkp5HqeWGm5EpALPjOE7F8Ntv8MQTcPbZ0LRptqVxqib3EqIT/pHCdHpSY+DJqC2RFXNS95504Rcdx3Gyx6OPwtKlIRKU42wYDgD2Jp6z1mwB0rXAu0knSZq9p47ELRLTJZZJrI6XMovuOI6zPqxeDQMHQufOIdKQ42wYllM81jIEj5vEZ7RJ3XtuI4SduhMoAK4CBhIibPRJejPHcZxyYexYmDHDV7POhmYMMARp7yi0o5D2AQYTEt8kIqmiPQ24yIwHgNXASDP+RAhDVXqaIcdxnPKmf3/YfHM48cRsS+JUbf5EyOL2HrAsKu8QDIH/knSSpH60LQhBlCFYHjeOPo8lJBZwHMepGKZPh3Hj4NZboXbtbEvjVGXMfgP+EFkf7xDVfoHZl2WZJumK9ntg8+jz18CR0ed9KcrzlxGJphIvSiyWmClxVgn9JNFPYl5U+kkh3ZHE9hIjJX6RmC8xTqJdyvjLJOZILJJ4pDDrkMSmEk9LzJJYKPGOxN6xcV0kClLy3HZL+nyO41QAgwYFBduzZ7YlcaoLZl9i9mJUvkTaFql8jaGAF4FDo8/3ArdIfAsMBR4qg7gDCQfILQjphu6X6JCmX09CEOmOwC6ErAm9orbGwCigXTTPhxSmQgIkjgSuieRtDbSF4KZECED9ESGRfVNCMvsxUiwwNcxKyXPrFteOU1nIzw/WxqedBi1aZFsap/rSAIoWaZlIpGjNuNaMvtHn5wkmz/2Bk8y4PskcEg2Ak4Ebzcg3YyJBYaZLHtkNuNOMH834iWCE1T26/4dmPBwlkV8J3A20k2gWG/twlOT9N4IhV+HYGVFi99lmrDZjCFAH8EjkjpMLPP44LFrkRlBOTiGzzEnmS51ANDBjcYJ+nYB3zKgfq7sSOMiMril9FwJHmPFBdL0HMN6MYokmJVCDAUEAACAASURBVE4A7jdjs+h6MvB3sxDwWaI5IY9uczPmpYzdFXgfaGHGQokuwGvAb4Rg1y8BN6R7PomehJU3tWrV2/31119N7VLpyM/Pp2HDhpk7ZhmXs3ypMnKasWePHqyuW5dP7r8fpIoTLqLKvMtKwsEHH7zEzBpkW44yI3UEPsEsWdxPM1unAlYX7CqwnxP27ww2J6XuQrAJafquBmsfu94uSveglH5bgv0Edmas7huwo2LXtaOxbVLGNgL7DOzaWF1LsB3BaoBtDfYW2AOZni0vL89ygfHjx2dbhES4nOVLlZHzjTdCzpehQytEnnRUmXdZSQAW2zrqoKwW6GiwOmn/TEkF6hBceI4AVgL/MOOlKD/t/xECLN+d8DtAPsUdfxsREutm6tsIyA/Pt0a2TQirz0FmPJ1hLPH7SNQj+EC9b8YdhfVmzAHmRJffSlwNvEzR+bDjONliwABo3hxOPz3bkjhVHWkS6TPWFVK/lLZiZHLvuZkQQPl1YH/gOYkHCYZG1wJPWTgnTcJ0oJbEdmb8L6rrSFGg5jjTorYP0/WTaEJQsqMsOjtOM/bZ2Ni5hdvGkQXyS8CPZFagRnKDMcdxNhQzZ4a8s3/9K9Stm21pnKrPy+U5WSZFexrQ3YwXJToCk4AmQAeztVLmZcSMxRIjgFslLgB2BY4H9kvT/THgcolXCMruCoLxFRKNgHGE895rShg7VOJJYBZwA8E6GonahIxES4FuZhTEB0ocDMwguDNtSVi1j8RxnOwyeHD4edFF2ZXDqR6UMd9sJjKt1rYiuMNgxmSCa06/sirZGH2AesDPwNNAbzOmSXSW1krB9wBha/czQuaEMVEdwImE7PY9UvxdW0VyjgX+AYwnKMyZhO1vCEr9WMJW+ILY2M5ReydCoOjF0c/PCJFBHMfJFkuXwoMPwgknQKtW2ZbGccpMphVtbUJQ5UJWAgvX9WZmzCf4x6bWvw1FvqzRWezVUUntOwxK92014y7grjT1bwIlmiqWNM5xnCwyfDjMm+cuPU7OkiQE4x0SS6LPdYCbI/ebNZj5qs9xnA2AWYhr3KEDdOmSbWkcZ53IpGjfAraJXb8LpO7drJ8jruM4Tkm8/z588glkyW/WccqDUhWtGV0qSA7HcZziDBgAG28Mf/xjtiVxnHUmafYex3GcimXOHHjuObj4YsiBKEdOFUXanBB2eFNSDYjN7ksyhStax3EqJ0OGwMqV0KdPtiVxqivSGUTuocCvrH1UaoArWsdxcpSVK4Pv7NFHw3bbZVsap/pyO0GZXo9Z0uBMxXBF6zhO5WPECJg9Gx4qSxZOxyl3WgKD10fJgocXdBynMjJgAGyzDRx1VLYlcao3YwkBktaLRCvawqhLaTBgmRm/rK8gjuM4AHz6KUycCHfdBTV8LeBklVeBfyDtQIgUuPbK1mxUkkmSbh1/Ryn+shKLgEeBq9cjPKPjOE5YzdavDz16ZFsSx3kw+vm3NG0GJMpHm1TRnkmIHzwYQjJ2YG9C4vObgcaE4P2/UxRX2HEcp2zMnw9PPgndukHjxtmWxnFql8ckSRVtb+AyM0bE6v4j8RXwZzMOkvgZuAVXtI7jrCsPPwzLlgXfWcfJNmary2OapAcgexP2p1OZStFB8XuE1HKO4zhlZ/VqGDQIDjoIdt4529I4TkA6Euk/SHOQZiO9gXREWaZIqmhnEraJU7mQkIoOYBNgfllu7jiOU0iz99+H776DSy/NtihOZUJqivQi0mKkmUhnldDvZqSVSPmx0jZNv3ORDOmCBPfuQUgC/xNht/ZmYDbwMlL3pI+QdOv4CuAFiWOI8tMCexASDpwcXe8JPJv0xo7jOHG2eOkl2HJLOP74bIviVC4GEnKhtwB2BcYgTcZsWpq+wzErOTC21AS4Dkg3Nh3XAldidm+s7gGkj6O2oUkmSbSiNWMMsB0wCmgUlVFAOzNeifoMMuPy0uaRaCrxosRiiZkSab+ZSEiin8S8qPSTQh5Zie0lRkr8IjFfYpxEu5Txl0nMkVgk8YhEXqytjcR4iSUSX0oclnRsSSxfXpM2bYINh7PuPPkktGkDhxxykL/PciCn3ueXX9L044+hd2+oVfni6OTUu8wBCt9nMC8vBakBYTF3I2b5mE0k6J5z1vHWdxAiPf2asH9rYEya+pejtmSYWYUVsKfBhoM1BDsAbCFYhzT9eoF9BbYl2BZgn4NdFLXtBXY+WFOw2mC3gX0ZG3sk2FywDmBNwCaA/V+s/T2wu8DqgZ0MtgBskyRjS36u+gZm9eubPfGEVVrGjx+fbRFK5IknwvsLCUjN3+d6knPv85JLbHXt2mZz52ZbkmLk3Lu03PndbE4dM/g4Vnpa/O8rdDJYklJ3pcFoS/1bDDcbLDSYbzDNoHdK+17RPWoYTDC4oNgcxef82qBXmvqLDP6XcXxUZJYsnaxEfcKyvVgGgxRr5JLGNwB+A3YyY3pU9zjwkxnXpPR9FxhqxpDo+nzgQjP2STNvU2Ae0NyMeRJPAd+ZcV3UfijwpBktJbYnGHU1N+P3qP3tqH1waWNLf7YGBosByMuDfYpJWTlYsGABjSupy8T778Py5dCS2TzDGZzOcObS0t/nOlL4PlOpjO+z/qpFPP/+FrzR6A/cucsz2RanGLn0LgvJnd/NBpgtLjnRsNQZeA6zlrG6C4GzMeuS0ndHYAEwl2DA+wJwOWZPI9UkuKZegtn7SBOAJzArPcan1Ae4G3iEkI8dYH+gO3AZZvdnfGCSR4Y6DHgaaJamOanT7vbAqkIlGzEZOChN3w5RW7xfhxLmPRCYY8a82NiRKWNbSDSL2mYUKtk0c5c4NjY/ABI9SWMgtny5sWDBwhJEzS6rV69mwYIF2RYjLcuXbwyIG7mVA5jIjdzGJQz097mOFL7P4vWV730e+esQ6q/O58km51fK95lL77KQXPzdLIF8wlFlnEaw1t/wgNnnsat3ke4FTiHorj7AFMzeL5OwZoOQfiHYKRUedX5BUPQvlGGeRFu+08CGgm2edKmcZo7OYHNS6i4Em5Cm72qw9rHr7aItG6X02xLsJ7AzY3XfgB0Vu64djW0Ddg7Y+ylz9AUbmmls6c9Wf82WUuvWiXdQKpxKt520cqXZ1Klmjz9uD250mU1kXyuIXuRi6lkLZvv7XEdat157q7PS/n4WFJi1a2e2116V9n3mzLuMUVnfpVnq+6xvVvrWbQODFQbbxeoeM8h4pGfwV4MR0eeXDH4zmBOVFdE284CM85RDSWp10AY4zoxZiTV4cZJ/MynetxGQH95XQGIT4DVgkBlPZxhLdJ9MMpQ2NiP160Pfvkl6VkOWLYPPPoNJk0L55BOYMiXUA91r12UejSigBjUpII/l3FrzVhr0HZRlwXOTvn2hZ09YsqSorlL+fr7xBnz1FTz+eLYlKZGceZc5Qrr3WSJmi5FGALdG7ji7AscD+xXrKx0PvEXYPt4T+BPBwhjCVm/dWO8RwPPAw+v4GGUiqR/tO7C2Ze86MB2oJRFPLtmR9GbW06K2tP0kmhCU7CgzUn/d042dG239TgPaSmxUwtyljS2V1q1Dnuqzz87UsxqwaBG89Rbcc08IpbfLLtCwIey1F/TqBU8/Hf5S9e4Njz0GU6dS65vpNKu9iJoUAFCTAi6wBzn70DlZfpjc5Oyzw+9j69YgWeX9/ezfHzbZBE49NduSlEjOvMscIf4+E9IHqAf8TNgG7o3ZNKTOSPmxfmcAXxMWRo8B/TAbBoDZAszmrCnBXWgRZsX3/qX5SM2jz79F1+lLQpKuaAcD/5LYnDQZDMz4JNMEZiyWGAHcKlH6N5Pwki6XeIVwBnwF0B9AohEwDnjHUoyoYmOHSjwJzCLEYB4ayTBd4lPgJokbgKOBXSjyBS5xbGnk5a3mu+8y9aqizJ279ip10iT45pui9pYtYbfd4LjjoFOnULbeGpRyRtOnD7VUsFZVjYJVcMYZMGHChn+OKsjZZ4cyYcKbdOnSJdviFOe772D0aLjuumBZVImp9O8yxyh8n1KCda3ZfOCENPVvAw1j12cmFiDVkGptrqJoF/MqSkmok5Skivb56OeQNG2JMxgQvpk8QvhmMg/obcY0ic7Aq2ZrXtoDQFuKwj4+FNUBnEjYFugg0T02945mfG/GWIl/AOMJ34JeYO34y2cQlOdvhKhWp1iU5i/B2OqLWfjDWKhUCxXr7NlFfdq2DYr0vPOKlGrLUg22i3jvPVixonj9m2/C2LGel7QqMmhQSIN30UXZlsRxijB7OPa5dKvkhCRVtFuXx83MSPvNxIy1vplEZ7FXRyW17zBgWIb73AXcVULbd0CXdRlbbVi9OpybFa5QC0uhFWPNmrDDDnDYYUGZ7rYb7LorbLzxut9z0qQ1HydMmBBWDb//Dp07w2mnhfyku+yyfs/lVB6WLIGHHoITTwzRoBynMiJNB/aJVtXx+sbAh5htn2SaRIrWjJllFtDJDZYtg6lT1976nTIFli4N7XXrhgDvp59etErdeWeoV2/Dy7bRRvDyy7D33vCHP8AHH8Dmm2/4+zobnmeegd9+87jGTmVnW9LryTzKEBmqREUrcRIw2oyV0ecSsQQBK5zsU3Px4mCkFN/6/eILWLUqdNh447Ay7dUrrFI7dYL27bMbEm/LLWHMGDjgADj22CB/w4aZxzmVF7NgBLXzzmHHwnEqG9JxsasjkeJGUzWBQ4Hvkk5X2l/Q54GWhPPU50vpV5YzWqei+PnntVepkybR+euvi9pbtAjKtGvXou3fdEZKlYFdd4Vnnw2ynnkmvPRS2L52cpN334VPP4UHHqicv2+OAy9FP43iR5WrCfY9lyWdrERFa1bk+hP/7FQyzGDmzOKWv7NiLs9bbw2dOjHjoINoe9JJQbFutln2ZF4XjjkmrIIuvhguuwzuuy/bEjnrSv/+0Lix+8c4lZnahPBV3xKMb39Z07IOyeArX5oMp2QKjZTiBkqTJoWzLggWnDvsAIccsraRUhTz9PsJE2iby64JffrA11/D3XfDttvCn/6UbYmcsjJrFrzwQvi3a9Ag29I4TnqKlOlW5TFdYkUrsSUhrnC6pALV20p3Q7B8eTBSilv+TplSFE4lLy9Y4Z566tpGShmyTuU8//wnfPst/OUvIc/WccdlHOJUIoYMCV8Y+/TJtiSOkwxpY+BIoBVQZ602s78nmSJpUoGzCf6vqwhL6LgDr1Hd3WHWl0WLYPLktbd+P/+8yEipUaOwMu3Zs0iptm8PtWtnV+5sULMmPPEEdOkSzmvfegt23z3bUjlJWLEinMsecwxss022pXGczEh7Aq8CBUBTYDbBdmkZ8ANQfooWuBW4E7jRjDLvTzsxCo2U4meqqUZKnToFd5ZCy9+ttw7bwk6gQYMQUWjvvYMl8gcfQKtW2ZbKycQLL8CcOXDJJdmWxHGScifwDHApsIiQbW4pIRTk4KSTJFW0LYCHXMmmZ+sVK8IfkHgUJDP4/vtilr/89FNRnzZtgjLt1q3oTDXXjJSyRcuW8MorsN9+QdlOnBhW/k7lpX9/2G47OOKIbEviOEnpCFyImSGtBvIwm4F0NfAEQQlnJKmifYWQSHfGOolaxWlgFixhu3YtUqyffgrzo2AiNWqErd4uXYpWqbvuCk2aZFXunKdDh7BKOvrocFb98svVczs9F/jvf0OYzXvu8d0ZJ5dYQdFR6VzCOe0XhNVt4pBmSRXt60A/iQ6kTyrgASueeSaUvLxglHTyyUWr1OpgpJQtDjsMBg+GCy4IW5KDB7tvZmVk4MCw5d+9e7YlcZyyMAnYg5B97k1Cur7mwDnAlKSTJFW0hQH9r0vT5gErIBjpnHxyMNTxVVXFcv75IWPQHXcEt5+rrsq2RE6cX3+Fp54KySbWJx6241Q8N8CatKo3AE8CDxIUb7ekkySNdex7PZlYvToY6MyblzxjjVN+3H57ULZXXx2Mx045JdsSOYU8/HBwV7v44mxL4jhlw+zD2OefgcPXZZqMClSitsQH0nonfq/6rF4Nt92WbSmqJzVqwNChsO++cM458P772ZbIgeCiNmhQCKLSoUO2pXGcrJBxRRslFdiackh+W+VZsSLEcXWyQ716MHIk7LNPCGTx/vshR66TPV5+OVjf33NPtiVxnGRI/yOpvkuYJi/plvAw4MKEfUtEoqnEixKLJWZKnFVCP0n0k5gXlX4SirUPkfhKoiAl+TsSeRJ3S8yS+E1ikETtWHt+Slkt0T9qayNhKe03ZnquqXl5wZ3HbK28qk4W2GST4PazalXwRS4MT+lkhwEDYKutgkW+4+QGDwEPR+VpgnvrT4TkOs8DP0Z1TyedMKkxVAPgbInDgf8Ci+ONZiQNOjuQYC7dAtgVGCMx2YxpKf16EhLEdyR8s3idENy50EF4MjAc6JfmHtcQrMR2IhhpjSYcYt8Uybomx5pEQ2AO8FzKHI3NWJXwmZzKRrt28OKLcPjhwUBt7FioUyfzOKd8+fxzeOONYKSWzVSLjlMWzIr0ivQo8C/M1j4TlG4Atks6ZdIV7Q7AJ8BvQFtg51jZKckEEg2AkwnRpfLNmAiMIphJp9INuNOMH834iRCdo3thoxkDzXiDEAYrla7AfWbMN+MX4D7gvBLEOpmQBvDtJM/g5BAHHQSPPALjx4fQleYnHxXOwIHB3e2CC7ItieOsKyeRfuU6HDgx6SRJrY4PTjphKWwPrDJjeqxuMiGkVSodorZ4v7JYUijl85YSG5uxMKVfN+Axs2L78TOlNSvpq8z4tdgNRE/CyptatcSECRPKIF52yM/Pr15ybrklrbt3Z+uhQ/m2Rg1mnnvu+s8Zo9q9zzJQMz+f/R55hJ8PPpivpk5NNCYX3mcuyAi5I2cOsJSQTOfrlPrOwJLEs5hZhRSwzmBzUuouBJuQpu9qsPax6+2iQ1Cl9JsI1j2l7nawd8A2AWsJ9kE0drOUfq2j+2wdq2sItgdYLbAWYM+Djcv0bHl5eZYLjB8/PtsiJKJc5SwoMDvnnHCC/sQT5TevVdP3mZR77w3v/OOPEw/JhfeZCzKa5Y6cwGKrIB20TgWuNVhmMMDgj1EZYLDE4Lqk85QlTd7BwJmkSRVkxiEJpsgHUoPRNgJ+T9C3EZCfZuWZjr5AY+BTYDnBubgTIXxWnHOAiWZ8W1hhRj7wcXQ5V+ISYLbERmZp5XQqOxI8+GCwfD3vvJB8oHPnbEtVtSkoCNvG++zjmZWc3MbsDqSZwJ+Bwi2xL4ALMHsq6TSJzmgjy95XCREyuhBS5TUBdgM+T3iv6UAtaa0D5I5QzBCKqK5jgn7FMGOpGZeYsYUZbYF5wH/NKEjpei7BmrrU6aKfHrAjl8nLgxEjQiCLE06A6dMzj3HWnddfD+/40kuzLYnjrD9mT2G2N2aNorJ3WZQsJFcgVwKXmHEmIc7xtWZ0ImQvyE8mK4uBEcCtEg0k9geOBx5P0/0x4HKJLSQ2B64AhhY2StSRqEs4f60tUVcKz1I4JnIR2ge4kcjiODZ+P2ALUqyNJfaWaCdRQ6IZwZBqQpqzXSfXaNoUxowJgS3+8IcQFtDZMAwYENI9enQuxwGSK9q2wL+jz8thjYvMAFjbjzUDfYB6BEvfp4HeZkyT6CytpbAfILjlfAZMBcZQFG8Z4DXCIfV+wBCKDqwBtgHeJbggDQOuMeO1FDm6ASPSbAe3BcYStrOnRs96Zhmez6nMbLNNCGjxww9hZbssndG6s17MmBG+0PTq5S5VTm4izY8SB4D0W3SdviQk6RntPIoCK/9EcOmZAjQjKM5EmDGf4B+bWv82Rcqb6Cz26qikm6dLKfd4C2iTQY5eJdQ/TRmckJ0cZL/94LHH4PTToUcPePJJT9tWngwaFBJs9Er7X8xxcoGrKLIdurI8JkyqaN8GjiCsMJ8F7ouCVxxKcIFxnNzhtNPCyuvaa8Mq9/bbsy1R1WDJkpBA4KSTYPPNsy2N46wbZg+n/bweJFW0lwB1o893AKuA/QlK1/9KObnHX/8asv307RuUbY8e2ZYo93nySViwwI2gHCeFpAEr5sc+F5A+9KHj5A5S2OacOTNEjmrVCg49NNtS5S5mwQiqY0fYf/9sS+M46470G8mTCjRN0q0sfrQtCL6n2xDCKP4aWQ7PivuiOk7OULs2PPdcUAwnnxwyL+24Y7alyk3efhumTIGHHgpfYhwndymXc9k4iRStxO7AG4TA/h2AfwK/EpLgbg/ps/A4TqVn442Dlew++wS3n/ffD64pTtkYMACaNIEz3UjfyXHK6Vw2TlJzy38B90a+s8tj9eMIZ7WOk7u0bg2jR8PcuSGP7ZLkIUwd4KefQkCQ88+H+vWzLY3jVDqSKtrdSR9FaTYh5Z3j5DZ77AFPPQUffQTnnBPCCDrJGDw4vK8+fbItiVMVkZoivYi0GGkmUvodVOlmpJVI+bHSNmrbHmkk0i+RD+w4pHYJ7l0b6Uakz6P5VqxVEpJU0S4lhFxMpT0h+ITj5D4nnAB33hlWZ3/9a7alyQ2WL4chQ+DYY0OIS8cpf+J5zM8G7kcqKZvbcMwaxsqMqL4xIS1ru2ieD4GRCe59K3BhJENN4HpCYviFhPjHiUiqaEcCN0nkRdcm0YZgffxC0ps5TqXnL3+Biy+Gf/0rrNSc0nnuOfj5Z3fpcTYM0po85pjlY1ZaHvOSMfsQs4cxm4/ZSuBuoB1SswwjTwd6YTaQ4NY6ArM+wC2QPH1sWWIdNyUkE6gPTCTk51sI3JD0Zo5T6ZHgnnvgmGOCwn311WxLVLkZMADatXPXKGedaA61kD6OlZ4pXbYHVmGWmse8pBVt12hreBpS71JufSAwB7N5GURsSVFCm3zCyhjgFeDIDGPXkNSPdhFwgMQhhIw9NYBPzNbEP3acqkOtWjB8eEind9ppMHFi8A911uajj+CDD+C++zyMpbNO/BqU6B6ldGkILEqpW0hRSOA4zxJi388F9gZeQFqA2dphdaUtCVvBlycQ8QdgM+B74BuCp81/gb2AxMHSy/S/w4z/mPEvM/5hxr8lWks8W5Y5HCcnaNgQXn45uP/84Q/BstZZmwEDwnvq1i3bkjhVl+R5zM0+x2wWZqsxexe4F1g7hZS0CSEpzaBiCjg9owjKFaA/cBvS/wjGwY8mfYjEAStKoDFh/9xxqh5bbBF8bA84ALp2hbfeCorFgV9+gWeegQsvhEapfwcdp9yYTthe3g6z/0V1SfOTGyGVakBqQlCyozDrm+juZlfFPg9H+omQNW46Zi8lmgNPaO44pdOxY9hGnjw5BGNYvTrbElUOHnwQVqyASy7JtiROVcZsTR5zpAZIJecxl45HaoIkpL2AP1FoWSw1IsR9eAezazLeVzqsBHkmYvaPsihZqGBFK9FU4kWJxRIzpfQRpaKk7f0k5kWln1T0zURiiMRXEgXS2vlwJfIk7paYJfGbxCCJ2rH2CRLLJPKj8lXK+LMi2RZLvCSRKJalU4U55piwTfryy3DZZdmWJvusWgX33w+HHQbt22dbGqfqUyyPOWbTkDojxfOYn0Ew0v0deAzoh1lh/IcTgT2BHil+tq1KuOdrSDOQrkfaYn0foKJXtMX8oaS01mM9CXlrOwK7AF1hrRyykwkv/5M0Y68B9iDkzN2eYLyVahl9iRkNo7LGaTmS5QGC6XgLYAkwqIzP6FRFeveGyy+H/v3h3nuzLU12GTUKfvzRV7NOxRBcck7ArAFmrTB7Kqp/G7OGsX5nYtYs8p9tj9l9sbZhmCmaI+5n+30Jd+1AWElfCnyHNAbpRKSa6/IIpZ7RSozKMD7x4YxEoT/UTmbkAxOj+c8hKMc43YA7zfgxGnsnwWl4MIAZA6P6dFZfXYF+hRmHJO4j+PvelEDMs4HRUfJ4JG4EvpDYyCzN4btTvfjnP+Hbb8Oqtk2bYChVHenfP4StPPbYbEviOBsGsy+AK5GuAY4DziNYNc9DGgY8gtlXpU0RJ5MxVCYfo3mQOHPP9sAqM1L9oQ5K07dD1BbvV5LfVDqU8nlLiY3NWBjV3SHxf8BXwPVmTIjd993CgWZ8I7Eikv2/a91A9CSsvKlVS0yYMIHKTn5+vsu5ntTo2ZNdv/iCBmecQc077ljzi1OZKc/32eDbb9lzwgS+6dmTH95+u1zmLKQy/7sXkgsyQu7IWekxC0EqYATS5kB3oAdBCb+D2YEJ57EKKWCdweak1F0INiFN39Vg7WPX24WEl6aUfhPBuqfU3Q72DtgmYC3BPojGbha17w22EVgeWDew38G2idreALsoZb6fwLqU9mx5eXmWC4wfPz7bIiSi0ss5e7ZZ69a2rGlTs5kzsy1NRsr1ffbqZVa3rtmvv5bfnBGV/t/dckNGs9yRE1hsFaSDyqVAY4NLDeYZrE46riLPaJP7QxXv2wjID8+Zkb7AJOBTwur0JWAlwYkZMz4w43czlpsxDHgHOGYdZHSqKy1bwpgx1Fy+PPjYLlyYeUxVYMECePxxOOssaJYpcp3jVCGkw5CeAmYRwi8+Q7AFSkRFKtrpQC2J7WJ1JflDTYvaMvUrhhlLzbjEjC3MaEvY3v6vGSWlY4n7Wq11X4m2QF4ku+MU0aED0265Bb78Ek49FVauzLZEG56hQ0MKQTeCcqoDUiukm5C+JfjfbkY4Ltwcs4sxm5R0qgpTtGas8YeSaCBRsj9UMM2+XGILic2BK4ChhY0SdSTqEhRkbYm6UniWwjGRi9A+wI1EhlASjSWOjPrXkjibEPNybDT1k0BXif9v796jpazrPY6/P7gRVMQkCRUiF17SKLE0S4vCbFVmaGmWeTlqSxFYZh09Ka0lyhHtZCuWpeAFk1TU1FOImpqXYmsdrKN5yUjjIEbeIAVvG1Fu3/PH79k4DrP3nol55pmBz2utZzH7eX4z85mf2/nt53l+l5FZ561zgVnuCGWVvLz33nD55XDPPWle5KjmgkuLWrsWpk2DT3wCPvzhotOY5Uu6/y1e3AAAFR9JREFUF1hIGu1yA7AbEQcQcS0RVU+92GlDZ4aq1XhgBmk81FJgXATzJEYCd0bQ2VX7cmAY8Hj280+zfZ3u5u1OVPuT5rc8AGgHdiY11O8hzVM5IYK7s7K9gfNIy/utAZ4EvhxZB60sy1hSg/tu4F7SjW+zyr75TXjqKfj+92GXXeCMM4pOlI+77oIFC2Dy5KKTmDXCcuAw4HYiNniWmoY2tJGG3Hy5wv7fwbpGluxe7BnZVul1RnXzHvcDO3Vx7EXSoOXuMl4PXN9dGbN3mDw5NbZnnpnWZD3iiKIT1d/FF8MOO8BhhxWdxCx/EYfW8+U8BaPZhurVK92/3H9/OPZYeOCBohPV14IFabnAk0+GzTcvOo1Zy3FDa1YPffvC7NkwZAgceigsXFh0ovq55JK0dOCY8qVCzawabmjN6mXgwLTaz+rVaX7kl18uOtGG6+iAGTPS5fAddig6jVlLckNrVk/vf386s124MN3PXLmy6EQb5rrr0jhhD+kx+5e5oTWrt099Kp0Ftren9VpbddhPROoE9ZGPwH77FZ3GrGU1eniP2abhmGPSWe0556RhPxMnFp2odvfdB/PmpT8apJ7Lm1lFbmjN8jJxYhr2c/bZMGwYHH100YlqM3VqmmrxyCOLTmLW0nzp2CwvElxxBYwalSa2uP/+ohNV75ln0r3mE0+ELbYoOo1ZS3NDa5anzTeHWbPSRBZf+QrMb5Fpsy+7LN2jHTeu6CRmLc8NrVnett0W7rgDNtssDft56aWiE3XvzTdh+nQ45JC0wLuZbRA3tGaNMGwY3HILPPtsmtDizZrnJW+cm25Kfwx4SI9ZXbihNWuU/fZL67nOnQvHH59WxGlGU6fCHnvAZz5TdBKzjYIbWrNGOuII+MEP4MYbm3PIzx//CA8+mM5mPaTHrC48vMes0c444+2l9XbeOfVIbhZTp8LWW6fFEcysLtzQmjWalBZRX7QorYgzdCh89rNFp4IlS9KZ9rhxqbE1s7po6KVjiQESN0ssl1gkcVQX5SRxgcTSbLtAQiXHp0v8TWKtxPFlz+0jcaHE8xIvS1wi0bvk2JXZe78u8ajEQSXP3UkiJDpKtia8vmctr3fv1Olo993h8MPTDExFu+IKWLUKxo8vOonZRqXR92inASuBQcDRwKUSwyuUG0NaIH4EsCcwGji55PhjwHjg4QrPnQDsA3wQ2A34CHBWdqwNeAb4NLBNtv8mab2F4t8VQb9sm1zjZzSrzjbbpNV+ttwSDj4YFi8uLsuqVWns7Oc+lxZGMLO6aVhDK7EVcDgwMYKOCH4P3ApUuhl0HDAlgmcjeA6YAm+fuUYwLYLfAJXGSIwGLopgWQQvAhcB38yetzyCSRH8PYK1EfwKeBrYu36f1KwGQ4fCr34FL76Yxq2+8UYxOWbPhueeg299q5j3N9uINfIe7W7A6ghKp8Z5jHR2WW54dqy0XKUz366o7PEQiW0iePUdhcSgLFf5dbtFEgHcA3w3gvVmGJAYQzrzpq1NtLe31xCvGB0dHc5ZR/XM+e7vfY8Pnn02L33hC8ybNAl61e9v4Gpy7nXeefTZYQf+uMUWadWhArTCf/dWyAitk3OTEREN2SBGQiwu23cSRHuFsmsgdi/5edc0H1yorNzvIY4v23cexP9ADITYHuKP2XN3KCvXG+JeiMtL9vWD2AeiDWIQxC8g7urps/Xp0ydawZw5c4qOUJVNNuePfxwBEaefXteX7THnY4+l9/3Rj+r6vrVqhf/urZAxonVyAsujQW1QkVsjz2g7gP5l+/oDr1dRtj/QEUE1C3ueD7wLeBR4C7gC+DCwpLOARC9gJul+8brpbyLoAB7KflwicQrwgsTWERVzmtXPqafCggUwZUoa9tOoeYanTk0LB5xwQmPez2wT08jOUPOBNoldS/aNYP3LtmT7RlRRbj0RrIjglAgGRzAMWAr8KYK1kHo0A1eSOmQdHsGq7l4u+9cTe1j+JLjwwtQx6pRT4M4783/Pl1+Ga69NS/gNGJD/+5ltghrWgESwHJgFnCuxlcQngENJZ5blrgFOkxgssSNwOnBV50GJzSX6ku6/9pbom52l0vmcbIjQx4GJwDklr30psAcwOoIVpW8q8TGJ90v0kng3qSNVe5Td2zXLTVsb3HADjBgBX/saPPZYz8/ZEDNmwIoVntfYLEeNPlMbD2wB/BP4OTAugnkSIyU6SspdDtwGPA78Bbg929fpbmAFsD8wPXv8qezYzsBcYDlwNTAhgrsBJN5HGia0F7C4ZKxs54rcw4Bfky5n/4V06fkb9fv4ZlXo1w9uuy0N/zn44NQbOA9r1sAll8DIkalhN7NcNHRmqAiWkcbHlu//HdCv5OcAzsi2Sq8zqpv3uB/WGxfbeWwR7+yRXH7856Q/AMyKNXhwGmP7yU/Cl74Ev/tdaoDr6c47YeHCNPeymeXG9x7NmtWIEWn2qMcfhyOPhNWr6/v6U6fCjjvCl9f729eseUgDkG5GWo60CKnijIJIk5BWIXWUbMNKju+F9CekN7J/92rUR3BDa9bMDjoILr44nd1+5ztppFo9zJ8Pd90FY8em6SDNmtd6MwoidTWvwo1E9CvZFgIgbQ7cAlwLbEu6rXhLtj93bmjNmt24cXD66Wkhgp/8pD6vOW1aamDHjKnP65nlQVo3oyARHUR0N6Ngd0aRbpX+mIi3iLiIdBuxIYsuu6E1awU//CEcdhicdhrccsuGvdbrr8NVV6VezYMG1SWeWU52A1YTUT6jYFdntKORliHNQyodiD4c+HM2M1GnP3fzOnXlhtasFfTqBTNnwkc/CkcdBQ891PNzujJzJrz2muc1tsJtB21ID5Vs5ZdY+gGvle17Fai0juNNpKGbA4GTgLOROkeN9MueV83r1J0bWrNWseWWcOutMHAgjB6d1rOtVUTqBLXPPrDvvvXPaFaDl9LZ6j4l2/SyItXPKBjxVyKeJ2INEXOBnwBfrfl1cuCG1qyVDBoEd9yRJpn40pfg1RrnUpkzB554Ik1QoS5Hupk1i/mks95qZhQsF7w9nHMesCd6xy/9nlW+zgZzQ2vWaj7wAfjlL+HJJ+GII9JastW6+GLYbjv4+tfzy2dWLxHrZhRE2gqp6xkFpUORtkUS0r7AqaSexgDtwBrgVKQ+SJ1Tof02748AbmjNWtOBB8L06XDPPTB+fHXDfhYtSpeeTzoJ+vbNP6NZfaw3oyAR85BGIpXOKHgksIB0Ofga4AIirgYgYiVpsqR/A14hrVH+5Wx/7ho6M5SZ1dEJJ8BTT8H558Muu8CZZ3Zf/tJL079jx+afzaxeIirOKEjEO2YUJKL76XIjHgH2rm+46rihNWtl556bGtsJE2DYsHQpuZIVK+CnP02zQA0d2tiMZps4N7RmraxXL/jZz+CZZ+DYY2HIENhvv/XL3XgjLF3qVXrMCuB7tGatrm9fmD0b3vteOOSQtFBAqYjUCWr4cBg1qpCIZpsyN7RmG4PttkvDftauhS9+EZYtW3eo/1//Cg8/7CE9ZgVxQ2u2sdh113Rm+/TTabrGlalD5eBZs9LatsccU3BAs01TQxtaiQESN0ssl1gkUXG5IwlJXCCxNNsukN5eR1ZiusTfJNZKHF/23D4SF0o8L/GyxCUSvUuOd5tB4qhs/3KJ2RID6lwNZvkZOTLds73vPjjxRHj0Ud7z29+meY3rvZ6tmVWl0We06y13JFWc1HkMqTv3CNLsHaOBk0uOP0YaW/VwhedOAPYBPkiakPojwFnVZMj+vZy0MsQg4A3gkn/hc5oV56ijUm/kmTPhgAPSX6jLlxedymyT1bCGVmLdckcRdETQ3XJHxwFTIng2gueAKfD2mWsE0yL4DfBmheeOBi6KYFkELwIXkQYnV5PhaOC2CO6PoAOYCBwmNWbiabO6OeusNNTnlVfSzzffDIsXF5vJbBPVyOE9uwGrIyhf7ujTFcoOz46VlqtlOSOVPR4isQ0wrIcMw4G5nQcieEpiZZb9T+94AzGGdOZNW5tob2+vIV4xOjo6nLOOmj3nritWsGOvXmjtWtauWsULY8fyf9/5TtGxutTs9QmtkRFaJ+emopENbS3LHZUvafQq0E9CEfQ019yvgW9LzAE2I813CbBlFRmqXkopgunAdIC+fSNGtcCwifb2dpyzfpo65wsvwL33pl7IQK/Vqxl8990Mvuwy2H77gsNV1tT1mWmFjNA6OTcVjbxHW8syReVl+wMdVTSyAOcDjwCPks5OZwOrgCVVZCh0KSWzupk8eV0ju86aNWm/mTVUIxva+UCbRDXLHc3LjvVUbj0RrIjglAgGRzAMWAr8KYK1VWR4x/tKDAP6ZM8zax0PPLBueM86K1fC3LmVy5tZbhrW0Eawbrkjia0kul7uKK28cJrEYIkdgdOBqzoPSmwu0Zd0/7W3RF8pfZbO52RDhD5O6tB0TpUZrgNGS4zMOk6dC8yK8BmttZhHHkkzQkXQPmfOusc88kjRycw2OY0e3rPeckcRzMsattLlji4HbgMeB/4C3J7t63Q3sALYn3SfdAXwqezYzqRLxsuBq4EJEdzdUwaA7N+xpAb3n6R7s+Pr8snNzGyT1NBFBSKouNxRBO9Y7ii7F3tGtlV6nVHdvMf9wE61Zig5fj1wfVfHzczMauEpGM3MzHLkhtbMzCxHbmjNzMxypIhqhqZadyStJXXIanZtwOqiQ1TBOevLOeunFTJC6+TcIiI2+hO+hnaG2og9HBH7FB2iJ5Iecs76cc76aoWcrZARWitn0RkaYaP/S8LMzKxIbmjNzMxy5Ia2PqYXHaBKzllfzllfrZCzFTKCczYVd4YyMzPLkc9ozczMcuSG1szMLEduaM3MzHLkhrZKkgZIulnSckmLJB3VRTlJukDS0my7QJKaMOckSaskdZRswxqU8RRJD0l6S9JVPZT9d0mLJb0maYakPo3ImL13VTklHS9pTVldjmpQxj6Srsz+W78u6VFJB3VTvpD6rCVnkfWZvf+1kl7I6mi+pBO7KVvk72dVOYuuzyzDrpLelHRtF8cL/d7Mmxva6k0DVgKDgKOBSyUNr1BuDGl1oBHAnsBo4ORGhaT6nAA3RkS/km1hgzI+D5wHzOiukKTPAxOAA4H3AcOA/8w93duqypl5oKwu2/ONtk4b8AzwaWAb4CzgJkk7lRcsuD6rzpkpqj4B/gvYKSL6A4cA50nau7xQE/x+VpUzU2R9QvpeerCb40V/b+bKDW0VJG0FHA5MjIiOiPg9cCtwbIXixwFTIuLZiHgOmAIc34Q5CxMRsyJiNrC0h6LHAVdGxLyIeBmYTIPqEmrKWZiIWB4RkyLi7xGxNiJ+BTwNVPrCLaw+a8xZqKx+3ur8Mdt2rlC06N/PanMWStKRwCvAb7opVtj3ZiO4oa3ObsDqiJhfsu8xoNKZ4vDsWE/l8lBLToDRkpZJmidpXP7xalapLgdJendBebrzYUkvZZfwJkoqZHpTSYNIvwfzKhxumvrsIScUXJ+SLpH0BvAk8AJwR4VihddnlTmhoPqU1B84Fzith6JFfm/mzg1tdfoBr5XtexXYuouyr5aV69eg+w215LwJ2AMYCJwEnC3pG/nGq1mluoTKn6dI9wMfBN5DuqLwDeC7jQ4hqTdwHXB1RDxZoUhT1GcVOQuvz4gYT6qXkcAs4K0KxQqvzypzFlmfk0ln/c/2UK7I783cuaGtTgfQv2xff+D1Ksr2BzqiMTODVJ0zIv4aEc9HxJqImAv8BPhqAzLWolJdQuV6L0xELIyIp7NLoo+T/oJvaF1K6gXMJN2fP6WLYoXXZzU5m6E+sxxrstsvQ4BKV3wKr0/oOWdR9SlpL+CzwIVVFC/yezN3bmirMx9ok7Rryb4RVL7sNS871lO5PNSSs1wAzfbXY6W6XBIRTXvPNNPQusz+6r+S1AHu8IhY1UXRQuuzhpzliv7dbKPyvc9m+/3sKme5RtXnKGAn4B+SFgP/ARwu6eEKZYv83sxfRHirYgNuAH4ObAV8gnRpY3iFcmOBJ4DBwI6kX5axTZjzUGBb0v9w+wLPAcc1KGMb0JfUa3Jm9ritQrkvAIuBDwDvAn4L/KCBdVltzoOAQdnj3YG/AOc0MOdlwB+Afj2UK7o+q81ZWH2SLq8eSbqUuRnweWA5cEgz1WeNOQupT2BLYPuS7UfAL4CBFcoW+r2Ze10UHaBVNmAAMDv7Zf4HcFS2fyTpEkdnOQE/BJZl2w/J5pRuspw/J/Wm7SB1pDi1gRkn8XYvyc5tEjA0yzO0pOxpwBLSveefAX2aLWf2BbIkq/OFpEtzvRuU8X1ZrjezTJ3b0c1Un7XkLLg+BwL3kXrJvgY8DpyUHWum+qw6Z5H1WZZ5EnBt9ripvjfz3ryogJmZWY58j9bMzCxHbmjNzMxy5IbWzMwsR25ozczMcuSG1szMLEduaM3MzHLkhtbM1iMpJDXblJxmLckNrVmTkXRV1tCVb38oOpuZ1a6QpbzMrEf3sv46wiuLCGJmG8ZntGbN6a2IWFy2LYN1l3VPkXS7pDckLZJ0TOmTJX1I0r2SVmRrDl8laZuyMsdJelzSW5KWSLq6LMMASf8tabmkheXvYWbVcUNr1pr+E7gV2AuYDlwjaR8ASVsBd5Hmu90X+AqwPzCj88mSTgYuJ83PuyfwRdJk86XOBm4hraRyIzBD0tD8PpLZxslzHZs1GUlXAceQJuAvNS0izpQUwE8j4qSS59wLLI6IYySdRJpIfkhEvJ4dHwXMAXaNiAWSniVN8D6hiwxBWonme9nPbaTJ68dExLV1/LhmGz3fozVrTvcDY8r2vVLy+IGyYw8AB2eP9wD+3NnIZuYCa4EPSHqNtBzZb3rI8OfOBxGxWtKLpOXZzKwGbmjNmtMbEbEgh9et5RJW+cLsgW83mdXM/9OYtaaPV/j5iezxE8CHJG1dcnx/0v/vT0TEP4HngANzT2lmPqM1a1J9JG1ftm9NRLyYPT5M0oNAO/BVUqP5sezYdaTOUtdIOhvYltTxaVbJWfL5wIWSlgC3A1sCB0bElLw+kNmmyg2tWXP6LPBC2b7ngCHZ40nA4cBFwIvACRHxIEBEvCHp88CPgf8ldaq6Bfh25wtFxKWSVgKnAxcAy4A78vowZpsy9zo2azFZj+AjIuIXRWcxs575Hq2ZmVmO3NCamZnlyJeOzczMcuQzWjMzsxy5oTUzM8uRG1ozM7McuaE1MzPLkRtaMzOzHP0/inueNryxjykAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plotting the learning rate by epoch for ReduceLROnPleatue\n",
    "plt.plot(history.epoch, history.history[\"lr\"], \"bo-\")\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Learning Rate\", color='b')\n",
    "plt.tick_params('y', colors='b')\n",
    "plt.gca().set_xlim(0, n_epochs - 1) # get current axes\n",
    "plt.grid(True)\n",
    "\n",
    "ax2 = plt.gca().twinx() # create twin axes\n",
    "ax2.plot(history.epoch, history.history[\"val_loss\"], \"r^-\")\n",
    "ax2.set_ylabel('Validation Loss', color='r')\n",
    "ax2.tick_params('y', colors='r')\n",
    "\n",
    "plt.title(\"Reduce LR on Plateau\", fontsize=14)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### tf.keras schedulers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 25000 samples, validate on 5000 samples\n",
      "Epoch 1/5\n",
      "25000/25000 [==============================] - 2s 98us/sample - loss: 0.5468 - accuracy: 0.8044 - val_loss: 0.4855 - val_accuracy: 0.8278\n",
      "Epoch 2/5\n",
      "25000/25000 [==============================] - 2s 84us/sample - loss: 0.4135 - accuracy: 0.8519 - val_loss: 0.4266 - val_accuracy: 0.8510\n",
      "Epoch 3/5\n",
      "25000/25000 [==============================] - 2s 84us/sample - loss: 0.3739 - accuracy: 0.8670 - val_loss: 0.3985 - val_accuracy: 0.8630\n",
      "Epoch 4/5\n",
      "25000/25000 [==============================] - 2s 85us/sample - loss: 0.3491 - accuracy: 0.8751 - val_loss: 0.4018 - val_accuracy: 0.8574\n",
      "Epoch 5/5\n",
      "22848/25000 [==========================>...] - ETA: 0s - loss: 0.3329 - accuracy: 0.8807"
     ]
    }
   ],
   "source": [
    "# defining the model\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.Dense(300, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n",
    "    keras.layers.Dense(100, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n",
    "    keras.layers.Dense(10, activation=\"softmax\")\n",
    "])\n",
    "s = 20 * len(X_train) // 32 # number of steps in 20 epochs (batch size = 32)\n",
    "learning_rate = keras.optimizers.schedules.ExponentialDecay(0.01, s, 0.1)\n",
    "optimizer = keras.optimizers.SGD(learning_rate)\n",
    "\n",
    "# compiling the model\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])\n",
    "n_epochs = 5\n",
    "\n",
    "# training the model\n",
    "history = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n",
    "                    validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For piecewise constant scheduling, try this:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = keras.optimizers.schedules.PiecewiseConstantDecay(\n",
    "    boundaries=[5. * n_steps_per_epoch, 15. * n_steps_per_epoch],\n",
    "    values=[0.01, 0.005, 0.001])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1Cycle scheduling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read the following for more information on the 1Cycle policy by Leslie Smith:\n",
    "\n",
    "https://arxiv.org/pdf/1803.09820.pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "K = keras.backend\n",
    "\n",
    "class ExponentialLearningRate(keras.callbacks.Callback): # function to define learning rate\n",
    "    def __init__(self, factor):\n",
    "        self.factor = factor\n",
    "        self.rates = []\n",
    "        self.losses = []\n",
    "    def on_batch_end(self, batch, logs):\n",
    "        self.rates.append(K.get_value(self.model.optimizer.lr))\n",
    "        self.losses.append(logs[\"loss\"])\n",
    "        K.set_value(self.model.optimizer.lr, self.model.optimizer.lr * self.factor)\n",
    "\n",
    "def find_learning_rate(model, X, y, epochs=1, batch_size=32, min_rate=10**-5, max_rate=10): # function to find learning rate\n",
    "    init_weights = model.get_weights()\n",
    "    iterations = len(X) // batch_size * epochs\n",
    "    factor = np.exp(np.log(max_rate / min_rate) / iterations)\n",
    "    init_lr = K.get_value(model.optimizer.lr)\n",
    "    K.set_value(model.optimizer.lr, min_rate)\n",
    "    exp_lr = ExponentialLearningRate(factor)\n",
    "    history = model.fit(X, y, epochs=epochs, batch_size=batch_size,\n",
    "                        callbacks=[exp_lr])\n",
    "    K.set_value(model.optimizer.lr, init_lr)\n",
    "    model.set_weights(init_weights)\n",
    "    return exp_lr.rates, exp_lr.losses\n",
    "\n",
    "def plot_lr_vs_loss(rates, losses): # function to plot learning rate against loss\n",
    "    plt.plot(rates, losses)\n",
    "    plt.gca().set_xscale('log')\n",
    "    plt.hlines(min(losses), min(rates), max(rates))\n",
    "    plt.axis([min(rates), max(rates), min(losses), (losses[0] + min(losses)) / 2])\n",
    "    plt.xlabel(\"Learning rate\")\n",
    "    plt.ylabel(\"Loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.random.set_seed(42)\n",
    "np.random.seed(42)\n",
    "\n",
    "# defining the model\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.Dense(300, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n",
    "    keras.layers.Dense(100, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n",
    "    keras.layers.Dense(10, activation=\"softmax\")\n",
    "])\n",
    "\n",
    "# compiling the model\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
    "              optimizer=keras.optimizers.SGD(lr=1e-3),\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 25000 samples\n",
      "25000/25000 [==============================] - 1s 40us/sample - loss: nan - accuracy: 0.3392\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAERCAYAAACO6FuTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO2deZgcd3nnP2/fPd1zz0jWYVmSJR8xxAYEONgmjiELJBuOOJvYiTchyRMnIQ7HLrtLWLMYhw2BALuB9TrrbLxOSAIsGxNwiCGEK2BCjAzIxmBLtg5bkiWNNJJmpmemz9/+UUdX9/QxPdNHVff7eZ5+pru6uvSrKU19+73FGIOiKIqi1CLU6wUoiqIo/kVFQlEURamLioSiKIpSFxUJRVEUpS4qEoqiKEpdVCQURVGUukR6vYB2MjU1ZbZv397rZShdZDlf4sCpeQAmUzFikRDPnV/m8k0jRELS49UNBjPzWU7MLbNrOk0yFm7ps/tPzpOIhtk2MeRuW8wVeHomA8ClG4eJRfS7bKd55JFHThtjpmu911cisX37dvbu3dvrZShdZGY+y4v/6z8C8OZX7GbXhjRv/vh3+dTbXs7ujcM9Xt1g8N++uJ8//tIB/vJNL+MF28Zb+uwNH/wqV2wZ5aM3v8Ddtu/Zc7zurocA+MLv3cCm0WRb16usRESO1HtPJVoJNONDUcQ2GMaSUaZSMQBOL+R6uKrBIlsoAVBaQ2FuyRiqDT6v5RAJ6S2q1+gVUAJNJBxiLBkFYGwoymQ6DsCZTLaXyxoosoUiAIXiWkQCQlKpEtFwyPNcXYa9RkVCCTwTtvUwPhRzn59RS6JrOJZEcY2WRJVGEPdaEmG9RfUavQJK4HGsh9GhKGNDllUxt5Tv5ZIGimzedjeVWv+sqWFJeN1Nakn0HhUJJfBM2tbDWDJKNBxiKBZmbllFolvkipY6FNagErViEhXuJo1J9By9Akrg8bqbAEYSUc6rJdE1snkrJrH2wHV9SyKkacw9R0VCCTwXTQ4xHI8wYgewR5IR5pYKPV7V4ODGJNbgbioZkGqR0DiEr+irOgllMPnlH9vOv/7RzYTtb50jiai6m7qIk91UXIO7ydR0N6n14CdUspXAk4iG2TxWLrgaSapIdJP1WxKV26otC6W3qEgofcdIQt1N3SS3jhRYUyMmofgLFQml7xhVS6KrlC2JtWQ3rUyBVfyFioTSd4wko8wt5dH57d2hHJNo/bO1iukUf6EiofQdI4koJQMLWXU5dQOnmG5tgWu1JPyOioTSd4wkraS9uWUViW6wvsD1yuwmxV+oSCh9x0hCW3N0k/UErmsV0yn+QkVC6TtGkyoS3cIYU45JrMGUqFVMp/gLFQml73Aqr9Xd1HkKJUPJNiDW0Cm8ZjGd4i+04lrpO9Td1D2ceARAqdSeeRIA/+FVl2p2mk9QkVD6jnLgWkWi0zjN/cCyKlqlXuD6d35i13qWpbQRdTcpfUc6bomEdoLtPBWWxJoqrlnZl0PxFV0VCRG5TUT2ikhWRO5rsu9OEfk7EZkXkdMi8oEuLVMJOJFwiHRcW3N0g5xHJIotWhKOO0ljEv6m25bEceC9wL2NdhKRGPBF4MvABcBW4C87vjqlb9DWHN3Ba0m06m5ydtcUWH/T1ZiEMeZ+ABHZg3Xjr8cbgePGmA97tj3awaUpfcZwIqKB6y7gpL9C64HrkloSgcCvMYmrgcMi8qDtavqqiDy/1o4icqvtwto7MzPT5WUqfkXbhXcHryXRajGdIxJaJ+Fv/CoSW4GbgI8Am4HPAZ+x3VAVGGPuMcbsMcbsmZ6e7vIyFb8ykogyt1Tg24dnuesrT3FqbrnXS+pLnL5NsJaYhPVT3U3+xq8isQR8wxjzoDEmB3wQmAQu7+2ylKAwkowwt5zn3Z95nD/6wpNc8/4v83ePHu/1svqOXLHsbmpVJNTdFAz8KhKPAlpJo6yZ0WSU4+eW+MFzc/zmj+9kfCjG579/otfL6jvWY0lo4DoYdDsFNiIiCSAMhEUkISK1gud/CVwtIq8UkTDwVuA08MMuLlcJME678HBI+I3rdrJjKsVJdTm1new6UmDLMYm2LklpM922JG7HciW9A7jFfn67iGwTkQUR2QZgjHnSfv9PgLPA64DX2q4nRWmK07/p5bunmErHuWA0wQkVibbjzW5qNXBtbH1RS8LfdDsF9g7gjjpvp6v2vR+4v8NLUvqUkYT1X/v1L9gCwAWjCU6ez2KM0WyaNuJYEpGQUGyxw59aEsHArzEJRVkX11+6gd++/mJedcUFAFwwkiBXLDGbUWO0nTgV18lYeM0psGpJ+BsVCaUvmR6O859efRmJaBiwRAJQl1ObcSyJVCzScjGds7dmN/kbFQllILhg1BaJ8yoS7SSbLyICiWhoDW05tJguCKhIKAOBKxJqSbSVbKFELBwiFJLWA9eaAhsIVCSUgWA6HSckcFItibaSLZSIR0JEQqK9m/oUFQllIIiEQ0wPx3lORaKtZAsl4tEwIRHtAtunqEgoA8MFI1or0W6yhSLxSIjwWiyJkqbABgEVCWVg2DiS0KrrNuN1N2lMoj9RkVAGhk2jCXU3tZlsvkQsErYC12uNSehdyNfo5VEGho2jCeaXCyzmdKxpu3DdTbIOkVBLwteoSCgDg1tQp9ZE23DcTeHQ2gPXir9RkVAGBi2oaz85O7tpLYFrp+ZaLQl/oyKhDAxT6TgAs4tW/6a3fuK7vP1T+3q5pMDjtSRa791k/VSR8Ddd7QKrKL1k2O4MO79sxSSeODHfsh9dqSRbKBKLhMgXSzqZrk9RkVAGhpGENWNibikPWGIxt5zv5ZICTzZvWRLLuTUEru15Etq7yd+ou0kZGIZilu/csSTmlvLMLxfIZDXbaa1Y7ibr96qWRH+iIqEMDCLCcCLC3HKeYskwb4uDVmGvnZyn4rpVkdBiumCgIqEMFMOJCHNLeRaWy9aDNv1bO1bvprV1gdViumCgl0cZKEYS0RWxCLUk1oYxxnU3racLrMYk/I2KhDJQOO6m80tlkdBWHWsjV7Qiz07FtXaB7U9UJJSBwrEk5r3uJrUk1oQzutQZOtTy+FINXAcCFQlloBhORJlbyrvuppBoBfZaKRStm3xsjV1gHU0RVCX8jIqEMlCMJCNWTMJ2N22fTGlMYo0UbHdTOCRr6gKrlkQwUJFQBorhRJT5bIFzi5ZI7N6YVktijTgxiGhY1tgF1vqpgWt/oyKhDBQjdmuO4+eXANi1Ic3MQpa8/a1YWT2OKIRDa+sCq5ZEMFCRUAYKpzXHsbNLDMcjbB5LYgzMzGd7vLLg4QhrJCRrG1/qZDepSviaroqEiNwmIntFJCsi963yM18SESMi2mdKWTcjSeu/0bFzSwwnImxy2odrXKJlHEsiEpY1Bq7VkggC3bYkjgPvBe5dzc4i8ktAtKMrUgaKYduSOHp2iZFklI32ICKtum6dvJ3dFFlj4FqL6YJBV0XCGHO/MeZvgTPN9hWRUeDdwH/s+MKUgcFxN51fyjOSiLJpNAnAn33jEI8cOdvLpQWOipjEGgLX2rspGPg5JvEHwN3AiV4vROkfnJkSYLmeJlIx3vPaKzh4OsMv/K9/5ti5pR6uLlgU7F7fkbAdkzDlYPRqUHdTMPClSIjIHuAa4KOr2PdWO86xd2ZmpvOLUwLNSLLsvXSsil952XY+evMLKJQMz84u9mppgcPJZnIC10BL1oS25QgGvhMJEQkB/xN4izGmaaN/Y8w9xpg9xpg909PTnV+gEmgqLYmyYEykYgDMZnJdX1NQcSquw16RWIMlofgb34kEMALsAT4pIieAb9vbj4rIdb1bltIPRMMhktEwUK6ZABWJtVB0i+lCrkiUWig3KddJqCXhZ7qaVmqnsUaAMBAWkQRQqLIYzgObPa8vBB4GXgSoP0lZN8OJCEv5YoUlMT6kItEq+VK5LUfYvtFbcYrwqj7vBq79+FVVcen25bkdWALeAdxiP79dRLaJyIKIbDMWJ5wHZWE4aYzRv2Bl3Tji4MQkwGpSN5yIqEi0QNF2N0VDIbcgrhVLQmMSwaCrloQx5g7gjjpvp+t85jBom0ilfThxCaewzmEiFVORaIFCqRyTiKwjJqHZTf5GDT1l4HAsiOFEZZ2mikRreFNgHUui0IIpocV0wUBFQhk4XEuiWiSGVCRaoehNgZW1BK6tn+pu8jcqEsrA4cYk1N20LsptOUKuu2ktloS6m/yNioQycNS1JFIxZhdzGGN46tSCCkYTik52k8fdpIHr/kNFQhk4dm8YZsNwvKKwDiyRyBVKZHJFbv7Tb/HhLz7ZoxUGA3fo0DoD16oR/kbbbysDx40v3MLPvmDLijkGTkHdgZPzzMxnOX5OO8M2wltxHXLbcqzelNBiumCgloQycIhIzUE3jkg8fGgWgNMLOoioEW7vpnDIDVwXS5ArlFbVw6k8vrRjS1TagIqEotisEAmdVtcQx2qobvD3urse4u6vPtX085rdFAxUJBTFxhWJw44lkWup9fWgka/V4M/upLualusakwgGKhKKYuOIxPyy1UosVywxt9y0EfHAUtngz95mDMv5IrlCc3HVmEQwUJFQFJt0PEI0bN2wnE6xGpeojxOTCIk1nQ6seEShZMgXmwewNQU2GKhIKIqNiLjWxJ7t4wCcWdBaiXoUiiUiIUGkXHG9mLMsr9WJhBbTBQEVCUXx4LQMv3rnJKCWRCOKJUPEtrycdt+LuSJQjlc0opzdpCrhZ1QkFMXDZNoRiQlARaIR+aIhYquD8zOTXb0lYdSSCAQqEoriYSIVJx4J8fwtY4hoGmwjiqWSm9UUXmFJtOJuUpXwM1pxrSgebnrxhVx14RixSIiJoRgzGpOoS6Fk3EC/c6PPtBSToOKzij9RS0JRPFyza4pfv3YHAFPpOKcXshw/t8THH35GayaqKBSNa0k47qbFrGVJ5FYVk9A6iSCgIqEodZgajnFmIcs9/3SQ37v/MZ6eyfR6Sb6iUCrHJFYErguriUlYP1Uk/I2KhKLUYTIV5/RCjoeeOg3AV5881eMV+YtiqeRmN7mWRAvuJi2mCwYqEopSh6l0nOPnljhwagGAr6hIVJAvmRWB60xLgWvrp4qEv1GRUJQ6TA3H3Kria3ZN8vChWTfFU4Fi0RB13E1OMZ2bArv6mISmwPobFQlFqcNUOg7A+FCUN12/i3zRuK4nxRpVWh24drKbci1YElpM529UJBSlDtO2SLxs1xQv3j5BOh7hK0/O9HhV/qHQsOJ6dTEJtSL8j4qEotRh40gCgGt3TRGLhLjqwjF+cPx8j1flH4ol444tdSwKRyQKq3Q3aTzC/2gxnaLU4fJNw/zJLS/khss2AjCVjnH4jKbBOuSLJdfN5IpEtjV3k4qE/1FLQlHqICK8+nmbiEWsP5OJVJzZjFWBnSuU+Oy+4wNdYFf0Zje5Fddld1Oz303JGK2RCAAqEoqySibTMRZzRZbzRb70w5O8+ePfZd/RwXU/eWMSZXeTZUkYQ9M510YtiUDQVZEQkdtEZK+IZEXkvgb7/YqIPCIicyJyVEQ+ICLqGlN6ijNr4kwmx3PnlwF4Znaxl0vqKYXiypiEN/W1WRpsqaSWRBBYl0iISFJEXikiF63yI8eB9wL3NtlvCHgrMAW8FHgF8PY1L1RR2oAjErMLOWbsFuLHzjaf5dyvWJZEZUzCS7O4hMYkgkFL387tb/8PG2P+p4jEgIeBK4CciLzBGPNgo88bY+63j7MH2Npgv7s9L4+JyF8BP9HKWhWl3Uy6lkSWU3O2SJwbXEuiWCqtsCS8NEuDNaglEQRatSReBXzLfv5aYBi4ALjDfnSKlwOPd/D4itIU15LI5Dg1b7mbBtqSKK4MXHtpKhJqSQSCVkViHHAa2Lwa+BtjzCngE8CPtHNhDiLya8Ae4IN13r/VjnPsnZnRQielc0ymrOK62UyOmXnHkhhgkSgZog3cTflC8+wmLabzP62KxAngeSISxrIq/tHengby7VwYgIi8Hngf8BpjTM1+CMaYe4wxe4wxe6anp9u9BEVxGUlGiISEM16ROLs0sGmwhWK5LYeIrLjhN49JaDFdEGhVJO4FPgl8HygCX7K3vxR4oo3rQkReDfwp8DPGmMfaeWxFWQsiwngqxqm5LLOLOUYSETK5IueX2v79KBAUPBXXsNKaKJSaB661b5P/aUkkjDF3Ar8G3ANca4xxZjsWgPc3+7yIREQkAYSBsIgkaqW2isgNwF8BNxpjHm5ljYrSSSZTMQ6cmscYuGrbOABHBzQuUfTUSUA5vuBoRTN3k/ZuCgYtp8AaY/7GGPPfjDFHPdv+3BjzmVV8/HZgCXgHcIv9/HYR2SYiCyKyzd7vXcAo8Pf29gURaZg5pSjdYCIV48kT8wBcdeEYMLhxCW9bDsC1KlJx63tfU3dTSQPXQaDVFNifB84ZY/7Bfv1fgFuxMo/eaIx5rtHnjTF3UD8LKu3ZT9NdFV8ykYqRtUdzvmCbLRIDbEl4XUwh+/lIIsr8cqFpdlNRLYlA0KolcYfzREReCLwT+AgQBT7UvmUpij9xaiUALt04TDIaHlhLolDlbnIEI21bEs1EolAsucV4in9ptdXFRcCT9vM3AH9rjPmAiPwD8IW2rkxRfMiEnQYL1lCiLePJgbUkqgPXzvPhxOpEIl8lMoo/aVXGl7EK6MBqleGkwJ73bFeUvmUibVkS40NRYpEQW8aSA2lJGGPseRLlW4gTX0jbIpFrErguFEvu+FPFv7RqSXwd+JCIfAOrwO3n7O2XAM+2c2GK4kccd9OGYWsg0eaxJI8P4CAip8NrrRTY4UQUWI27SS2JINCqjN8G5LDE4beMMcft7a9B3U3KAOC05tgwYrmdJlMxzi7mKTVpi91vFOzzDa8jJpH3NAhU/EtLloSd9vozNba/tW0rUhQf41gSzvzr8VSMYskwv1xgdCjay6V1FUckvO6icIsxCcvdpJaE31nTjAa72O1HAAP8wBjzlbauSlF8imNJTNuWxETKEobZxdwKkZjN5CgZw1Q6Tr9RsAUgXMvd5NZJNItJqLspCLRaJ7EF+DTwIqzZEACbRWQv8AaP+0lR+pLxoRived4FXH/JBvc1WIKwYypVse/v3f8oi7kiH/v1l3Z9nZ3GsSQqUmCrAteFpu6mEumozhLzO606BD+C1bNplzHmQmPMhcBue9tH2r04RfEboZBw9y0v4scungTKlsXZTG7FvifOL3Nybrmr6+sW5cD1SnfT6uskKlNoFX/Sqoz/JHC9MeaQs8EYc1BE3ky52Z+iDAyuJbG4UiTmlwtudXa/4QhA7ewmRyQau5vyWkwXCNZi69W68oOV2qEoNo0sibnlQtNOqEHFsSRqxSTScSs2k2sikNY8CrUk/E6rMv4l4KMicqGzwW7K99+BL7dzYYoSBIZiYWKRUB1LIk8mW+jBqjpPrZiEU0yXiIaIhmV1bTm0mM73tHqF3gykgIMickREjgBPA0PA77Z7cYrid0SEiaHYCksiVyiRLZTIFw3ZQrFHq+scheLKmITjekpEw0TDoeZ1EprdFAharZN41m7s90rgMnvzD4GngA8DP9/e5SmK/xlPxZjNVA4eml8uv17MFolHwt1eVkdx3GgVloQtEvFIyBaJJimwJW3LEQRajkkYa1bjF+0HACJyJXBjG9elKIFhIhXlbJW7aX657GZayBYY93SP7QdqtuWQSkui2TwJrZMIBirjirJOxmu4m7wikcmtjEs8sO84jxyZ7fjaOoVjJXgD184NPx4JEQsL+SaB61yxRFSzm3yPVrIoyjoZH4qtCFx73U21gtfv//wTPH/LKC+6aKLj6+sEjiXhvck7get4NEw00jwmUShqdlMQUBlXlHUynopxfinv3jjBSn91WMiuDFwv5YqcX8qv2B4UarXliFTHJJo0PSyUtE4iCKzKkhCRzzbZZaQNa1GUQDIxFMUYOL+Ud+smKgPXKy2JpXyRc4sBFokaMQlv4DoSauxuMsaQLxpt8BcAVutuOrOK9w812UdR+hInKD2byXlEojJw7cUYw1I+2JaEG7j2WAJhEeKRECJCrIm7qdbnFX+yKpEwxvxqpxeiKEHFrbr2xCUqAtdVIpEtlFzLI0h879lzXLl1FBGp3ZYjLCSiVqpvsxTYWsV4ij9RGVeUdeLtBOswv5x3g7KZXGVMYjlvvV7IFpoGd/3Cd545y+vveoiHD1kZWTXbctiWBEA0LA1TYJ3z1joJ/6NXSFHWiWNJPHb0PK+76yF++Nwc88sFJlIxIiFZYUks5cuiERRr4gfH5wA4Z6/XHTrksQQ2DMfZNJqwtzd2N7kV22pJ+B5NgVWUdeJYEnd/7WmKJcO/HDzDfDbPcCLKcr60UiQ8lsW5xXwghhI9dWoBKDftcyquwx5L4O2vutQVhlgTkci7Fdv6PdXvqEgoyjpJxsIko2HXQjh6don55QLDiQhLueKKFNhKS2JlY0A/4oiE0/q83LupbAkkouHKmEShQUyi6Iw/VUvC76hIKEobuHAiyWQqzsm5ZY6dW2JuucBoMkomW1hhSSwH0N104NQ8ULYkik0Cz82K6cruJrUk/I6KhKK0gY//xtWk4hFu/dgjHD27RCZXYOt4krmlyIq2HEu58s0zCLUS55fynJzLApCzO9rmawSuvTQNXNvuJq249j9dlXERuU1E9opIVkTua7Lv20TkhIjMici9IuJ/x60ysEym4ySiYbaOJzl2znI3jSQipOORFZbEokc0giASjqsJyu6mYpPspFg45FoLtajValzxJ92+QseB9wL3NtpJRF4FvAN4BXARsBN4T8dXpyjrZMtYktlMjrOZHMOJKKl4mEyDmMS5ALibnvaIRDlwbVsSdSyBSJOhQ26dhVoSvqerImGMud8Y87c0r+D+FeDPjDGPG2POAr8PvLHT61OU9bJ1PAlYN9HheIRUPLKi4roiJlFjop3fOHBqnngkREg8gesabTm8NGsVXiuFVvEnfrX1rgD2eV7vAzaKyGSP1qMoq8IRCYDhRIRUrFZMoui+H4TA9YFTC1w8nSYWKd/4y/Mk6rubGgeunZiEX29BioNfr1AaOO957Twfrt5RRG614xx7Z2ZmurI4RanH1vEh97nlboqwuMLdZN0gN40mAuFuOnBygV0b0sTCIdfdVKsth5dmbTnyGpMIDH69QgtUdpZ1ns9X72iMuccYs8cYs2d6erori1OUekyn48Tsb8fDiQjpeJhcseTeXMGKSYjAxpFEIALX5xZzTA/HiUfD7rzuYskgUu78Wk00HKJYMhXt070UNLspMPhVJB4HrvS8vhI4aYxpFstQlJ4SCgmbx6zWFI4lAZVN/pbzRZLRMKPJaCDcTfmSIRoOEQuHKmISjfouRSPWzb+ey6kcuPbrLUhx6HYKbEREEkAYCItIQkRq1Wr8BfDrIvIjIjIG3A7c18WlKsqa2WLHJYYTEVckvMHrpZwlEmNDUc4FIHCdL5aIha3mfW52U7FUt0YCcK2p+iLROPCt+Iduy/jtwBJWeust9vPbRWSbiCyIyDYAY8zngQ8AXwGeAY4A7+7yWhVlTWwds+ISI4koqZglEouefk1L+SKJaJixpDXRrlTDJfPMmUV+7u5vcnoh251F16FYMhhjfeOPRSotiUbpq1FXJOq4m5y2HGpJ+J5up8DeYYyRqscdxphnjDFpY8wznn0/bIzZaIwZMcb8qjGmt38tirJKnAynkWSEVNzqZVRhSeSLJGOWJVEysJBbObnuc489x94jZ/n+sfMr3usmeU8WkteSKJZMQysg2sSSKJS0TiIoaFsORWkzN71kG1vGk4wNxUjXiknkyjEJgPOLeUYS0YpjfPPp00DljIpeUBYJIR4Je7KbTEUH2GqcgHSuzgjTvNvgTy0Jv6NXSFHazPRwnJ994VaAmoHrRTcmYbUYr85wyhaKfPuwNdyn9yJRdgtZ7iYnu6nUMDPJsSQK9bKbtOI6MKhIKEoHcWIS1e6mhO1uAjhX1S78e8+cY9mupTi90FuR8N7MvcV0haJpGLhu5m7K6/jSwKAioSgdxIlJeAPXVgpsiDHb3VRtSTz09BlCYmVHzWZ6G4rLVcUksnlP4LqhSDR2NxV0fGlg0JiEonSQmimwdp3EhhGrnuLE+eWKz/zz06d5/pZRckXTc3dTOQtJVrTlaFTjEI00CVzr+NLAoDKuKB0kYddDHDu35G5bylnZTaPJKMOJCEfPLrrvZQtFvvfsOa6+eJLJVKzn7iZvdlN1W45GlkSsSQpseZ6E3oL8jl4hRekwO6dSHJrJuK+dOgmAC8eHePZsWUCePpUhXzQ8b/Mok+lYzy0Jb4+leLRcJ1EsrS8mUWv8qeJPVCQUpcPsmEpz6HRZJJy2HGDVVHgtiSdOzAFw+aZhJlKdF4mjZxd58LHn6r7v3ORjESEWDlfMk2joblplTKKR0Cj+QEVCUTrMzukUJ+aWyWQL5Isl8kXjEYkhjp5dwhjrm/WTJ+aJRUJsn0wxmYqxkC2wnC/y2X3H+dbB9rcu+8tvPcPv/PV3mjbii4SsFNiySDR2N20eswoK6xUD5kuGWDiEiIqE31GRUJQOs2MqBcCh0xl34FAyVrYkFnNFztoZTj88Mc/uDWki4RATKWti72wmx50PPM5dX3mq7WubW85bVd/LK6u+obJOIm4HrkslQ6HYOLtp40iCF28f54FHj9d8v1AsadA6IKhIKEqH2TlticTB0xl3dGnC424CXJfTE8/NcekF1tiUybRVbPfUqQVOL+QqXFbtwhGHueXa3Wi9FdcxO2MpVyzZ2U2Nb/KvvXIz+08uuC60yuM2FhnFP6hIKEqH2T5pWxIzGZZz1k3XcTddOGE1A3x2donZTI5T81kuv8AanzKZskTiXw5ZbqZj55bciud24aTm1mtZXk5VtSwJsEaY5kuN23IAvOb5mwiHhAf2rbQmCqWSZjYFBL1KitJhEtEwW8aSHDq94FoSQ7a7aYvHknC+cTuWxIQtEt86aLXoMMbqDttOHEuinkjkKno32ZZEocRitkDKPod6TKXjvOziSR7Y95wbc3EoFJtbIoo/UJFQlC6wYyrFwdMZFu2Orwn7BjuSiDKajHL07BiBbiQAABf6SURBVBJPnrAGL162yXY32TGJR4+ec49zsM0uJ8eSmGtiSTi9m8ASjky24BYKNuKaXVM8M7tIJldpAeWKJR1dGhD0KilKF9g5bdVKLNk3S8fdBOU02B8cn2MyFWM6bYnDSDJCJCTki8aNa7Q7LtHM3VTZKtxaczZfZD5bcDvcNmLCbmJ4tiqVt1A0Oro0IGhbDkXpAjumUsxnCxy1C+eqReLhQ7NkskVe8/wL3LRQEWEiFePUfJYXbhtnbinP4U5ZEk0C15FQOXCdLViWxGpEwm1iuJjnwony9kKppKNLA4JeJUXpArs3WC6kvUes+ELS48+/cHyIs4t5JlIx3v0zV1R8zolLXLIx7bqs2kmzmISTAhuLhNxWG+eXrLTZdKK5SIzb6z9bNaZVs5uCg4qEonSBF100Tiwc4ks/PAVUWhKXbBwmHBL++KarXFFwcNJgd28YZsdUqq3upmyh6Aam55Zq10mUi+mEeNS6XThV4KuJSYzblkS1SBSKmt0UFNTdpChdIBkLs2f7ON982kpnTXhE4sYXbeX6S6fdrrBenOD1rg1ptk+lmNl7lIVVunqa4S2gq5vdZFdYRz2WxBlbJIZX5W6qPVip2YxsxT+olCtKl7h295T73OtuCoekpkCAFa8YH4qyZSzJTrtyu11xCW/78noxCWeyXDRUzm6aXVi9JeHMzFjpbirpLImAoFdJUbrEdbum3eeJyOr+9N70E7v47G3XEgoJO6bSADw9s9D0cw8+9pw7J7seXpGoG5MoeOskLGFzbvirsWYi4RDDichKS0LrJAKDioSidIkrNo8wPhQlFg6tOrMnHY+4VdkXTQ4REnj6VHOR+NAX9/OHDz7RcB/H3TQ+FK0vErYlEfZkNznuptW6vMaHYistiSZdZBX/oFdJUbpEKCRcu3t6VVlBtUhEw2yfTLH/ZHORWMoV+f6x88wt5zm9kOV/f/3giqpnx5LYPJasG7jOF0tut1an4toZqbra8xgfiroNDB0KxRJRzW4KBBq4VpQu8s6fumxdrTV2b0yz/9R8xTbn5u9tu50tFCkZ2Ht4loeeOsOffeMQr7x8I9vtuAZUisSBkwsYY1a07vZ2a3VE4owbk2jclsNhrIYloe6m4KCWhKJ0kU2jSV66c3LNn79k4zBHziy6jf6MMdzwoa/x5988XLGfU9n9jQNn3AZ7Z2wLwGHedjdtGUuSK5bcqXNe8kXjpqq6gWs3uym6qjVblkS1u0lTYIOCXiVFCRC7Nw5TLBkO2uNQ57MFDp3O8LX9MxX7Lds3/E98+xlOzVvicKZqXrZjSWyxBwTVikvkiyW3fYY3cB0SSERXd/sYG4rVDFyrSAQDvUqKEiB2b7AynA7YwesZWwAeO3bedTvl7XkPyWiYxVzRrWw+U9U/aWG5QEhg46iVfluryV/eU/TmWBL5oiEdj6x6qtzYUJT55YI7shRsN5bGJAJBV0VCRCZE5NMikhGRIyLyi3X2i4vIn4jISRGZFZEHRGRLN9eqKH5k53SKcEg4cNKKSzgicXohx3PnlwHcduTX7LLcWj/9o5sAOLNQ6W5yivKcWoZaloQ3dhAOiTuTupVivnGnoM5zfM1uCg7dvkp3ATlgI/BLwN0ickWN/d4C/Bjwo8Bm4Czw0W4tUlH8SjwS5qLJIfbbIuG4kgAePWrNk152RWKKm158Ib97wy6G45GVloQtEiMNRCJfqnQLOcHrVjK0yk3+yv9+wePGUvxN10RCRFLAjcC7jDELxphvAJ8F/m2N3XcAXzDGnDTGLAOfBGqJiaIMHJdsGObAyUp3E8Bjx6y5E870u+FElD+88UfZtWGYiXRsZUxiuUA6EWHUFolaVdf5QmVltONyWk21tYNjSXjTYK0Z2WpJBIFuXqVLgIIxZr9n2z5q3/z/DLhGRDaLyBCW1fFgrYOKyK0isldE9s7MzNTaRVH6iks2pjl8JsNyvsip+WVi4RCXbxopWxIFZ452+c97MhVbkd3kWhK2VXB+sYa7qVQiGil/43f6N63F3eSdKZFTSyIwdFMk0kD1RPTzwHCNfQ8AzwLH7M9cDtxZ66DGmHuMMXuMMXump6dr7aIofcXFG9KUDDwzu8jMfJbp4ThXbh11g9eOu8nbaXYyHV9hScxnC6QTUdfdNLe8sqAuV/WN3+kE24pIeGdKOGiDv+DQTZFYAEaqto0A8zX2vQuIA5NACrifOpaEogwaO+0eTgdnMszMZ5kajvP8raOcW8xz9OySWyPh7TQ7lY7VyG7KMxyPEA2HSMXCdQLXld/412RJVM2UMMZQLKm7KSh08yrtByIistuz7Urg8Rr7XgXcZ4yZNcZksYLWLxGRqRr7KspAsX3K6uV06LQlEhuG41w8bQnHkTOLbo2E1900kYoxm8lRKpVbc3hbjo8ko01TYKFcK9FKTCIVCxMNixuTyLtzs9WSCAJdEwljTAbLIrhTRFIicg3wOuBjNXb/NvDLIjIqIlHgTcBxY0zjtpaKMgAMJ6JMD8c5dHrBdTc5N/tMruC6m7yWxGQqTrFkKqwFJ3ANMJqMVqSoOuSLlamqTuB6uIXsJhGxC+osS8IdZKQpsIGg21fpTUASOAV8HPhtY8zjInKdiHi7lr0dWMaKTcwAPwW8octrVRTfsmPKavR3JpNjOu0RiWwdkbAn3Dkup1LJkMkVXYtg81iSZ2dX9pSyGvx53E2R1t1NUNmaw7EktJguGHS1wZ8xZhZ4fY3tX8cKbDuvz2BlNCmKUoMdkynu/+5RADaMxBmym+1lckX3pl5tSYBVULdrQ5pMzgpSO9Pldkyl+Oenz6xo8ledqhpfQwoswFQ67tZ0OJXX2pYjGOhVUpQAsmM65X4jr7YknMB1soEl4fRtctxN26dSLOWLnJzLcmYhy//48gFKJWPFJCIrRaIVdxPARZMpd6KeM+1Os5uCgYqEogSQHZ6W3xtGEiSjYURsd1ONwPUKkbDTXR1x2TFpHe/g6QU+9chRPvgP+zkyu2h1aw2tdDelYq2JxM6pFGcX85xbzJF3LAnNbgoEepUUJYDs9IjE9HAcESEVi5DJFssxiUjZkpiwC9qc/k3z2UqRcDKmDp9e5DG7KC+TLZAvVLflsI7Z6uAkZ47FodMZCk52U0QtiSCgIqEoAWTb5BBO6GDKthJS8bDlbsoXiUVChDwWQCQcYmwo6hbUnZqzmgFOD1uxis2jSWKREIfPZNh31GrvsZgrUiiVKtxCa6mTgLLlc/hMppzdpJZEINCrpCgBJB4Js2UsydhQtFy7EIuQyRXI5kskIiv/tCftWgmAo2eXgPIsiVBI2D45xCNHzrrvZXIFcoXKOom1Zjdtm7Dmcx+ayWidRMBQkVCUgLJ7Q5pNo0n3dSoecVNgk7GVo0Un03FO2+6mo2eXSMXCbssMgO2TKR45ctZ9vZgtUiiZipv5WrObYpEQW8eHOHRm0XU3qSURDHTGtaIElDtf9zx3dgTY7qZckaV8sSL91WF6OM7jx6x4w7FzS2wZT1aku+6YTlXsn8kVVlRcr6WYzj3+VIpDpxfIu8V0akkEAZVyRQkoF04MccnGcn9MK3BtWRLeoLXDxVMpnpm15mMfPbvE1vGhivedDKcJu9dSJltYUXG9Z/s4N1y2wbUoWmHHVIpDM57AtdZJBAK9SorSJzjupqV8iUQNd9OujcOUjJVhdOzsohuPcHAykK7eOQHA3JKVAeWtuL7hso3c+8YXr3p0qZcdUykyuSLPnbdiHlpxHQxUJBSlT3DcTZYlsfJP25mP/Z0j55hbLrB1PLni/WhYuGbXFOGQuH2e2tVjyREhZ2CS9m4KBnqVFKVPcNxN2TqB653TKUICX33yFABbqkRiMh3ni2/7cW568TaGPK3D2+UWcmo7vnXwjH1ctSSCgIqEovQJqXiExVyRxVztmEQ8Emb7ZIqHnrKaKVfHJMD6th8OWYV5ZZFoz818y1iS63ZPsdfOoNLspmCgV0lR+oSU3eRvNpOraMnhxWruZ2VEVcckvAzFw+58iXZZEqGQ8H/e+GJu+4ldTKZibBiJt+W4SmdRkVCUPsGpXZhdzNV0NwHs3mjFJeKRkFupXfNYsQjnlqzCu3YGmCPhEG9/1aXsvf2VTKVVJIKAioSi9AlO0z1jyj2WqnFSZqtrJKpJemISsTWkuzZjLdlRSm9QkVCUPsFbBV2rmA4sdxPUjkdUHCsW5pw9blRjB4ONXn1F6ROcmARUzpLwcvF0GpHG8QiAoXiEbEEroxVty6EofYN3xkO9wHUiGuZ9b3g+L9g23uRYZZGJaT3DQKMioSh9gtfdVC9wDXDTS7Y1PdaQR3DUkhhs9CuCovQJXndTrTqJtR5LeywNNnr1FaVP8FoS8TruptXitSS0MnqwUZFQlD7BG5OoF7he/bHUklAs9OorSp8QDokbsK6XArtaKmISmgI70OjVV5Q+whkrum6R8MQkYhF1Nw0yKhKK0kc4FsD63U1qSSgWevUVpY9IuZbEegPXnphEB9pyKMFBr76i9BFp2020XneTN1MqqhPkBpquioSITIjIp0UkIyJHROQXG+z7QhH5JxFZEJGTIvKWbq5VUYKI425af+Bas5sUi25XXN8F5ICNwFXA50RknzHmce9OIjIFfB54G/D/gBiwtctrVZTAkW6Tu8lrSWjF9WDTta8IIpICbgTeZYxZMMZ8A/gs8G9r7P7vgC8YY/7KGJM1xswbY37YrbUqSlBJtcndpJaE4tDNq38JUDDG7Pds2wdcUWPfq4FZEfmmiJwSkQdEpGbDGRG5VUT2isjemZmZDixbUYLDUCxCJCTrvrFXVlyrSAwy3XQ3pYG5qm3ngeEa+24FXgj8JPAY8AHg48A11TsaY+4B7gHYs2ePaeN6FSVwvPp5F6zbigCrMC8eCZEvlghr4Hqg6aZILAAjVdtGgPka+y4BnzbGfBtARN4DnBaRUWPM+c4uU1GCy9U7J7l652RbjpWKR1jIFtpyLCW4dNOO3A9ERGS3Z9uVwOM19n0U8FoFaiEoSpcZioV1loTSPZEwxmSA+4E7RSQlItcArwM+VmP3/wO8QUSuEpEo8C7gG2pFKEr3SMUimtmkdL2Y7k1AEjiFFWP4bWPM4yJynYgsODsZY74MvBP4nL3vLqBuTYWiKO1nKB7WoLXS3ToJY8ws8Poa27+OFdj2brsbuLtLS1MUpYpULKLV1kp/jS998sknuf7663u9DEXpC05d8npyQ9P6NzXgiDH9ExMWkXngyXUeZhQrNXc9+9V6r9m26ved197tU8DpVaytEd06v0av6z3v1vm1em61tvfi/Dp17Wptb/X8gvR/s9a2fj6/1dxbLjLGTNf8F40xffMA9rbhGPesd79a7zXbVv2+87pqn8CcX6PXDZ535fxaPTe/nF+nrl07zi9I/zcH7fxWc29p9NCo1EoeaMN+td5rtq36/QfqbF8v3Tq/Rq8bnfd6Wc3xWj23Wtt7cX6duna1tvfT+bX6/7Xfzm9d95Z+czftNcbs6fU6OoWeX7Dp5/Pr53OD/j+/RvSbJXFPrxfQYfT8gk0/n18/nxv0//nVpa8sCUVRFKW99JsloSiKorQRFQlFURSlLgMnEiKyXURmROSr9qN2bnCAEZGbRaTvhmuIyEZ7xsjXROTLIrKp12tqJyLyEhH5Z3ts78ftvmV9g4iMisjD9kji5/V6Pe1ARN4vIl8XkY/12/VyGDiRsPmaMeZ6+9FXN1MRCQP/Bni212vpAKeBa40xPw78BfDrPV5Pu3kWuMEY83LgMFYDzH5iEfhprJHEgUdErgS2GGOuA54Afq7HS+oIgyoS19jq/wci0m/NaW4GPgWUer2QdmOMKRpjnPMapnab+cBijHnOGLNkv8zRZ9fQGJPvsy9lLwP+wX7+eWoMResHfC0SInKbPZo0KyL3Vb03ISKfFpGMiBwRkdV2iX0Oq6vsy4ENwM+2d9WroxPnZlsRPw98sgNLbokOXTvs9vH/AtwGfKfNy141nTo/+/MXAf+K9hcatrKGjp2f31jHuY5TnrZ5Hpjo0pK7it8b/B0H3gu8CqvFuJe7sL5tbQSuAj4nIvuM1Xr8AuATNY53kzHmBJAFEJH7seZp/02H1t+Itp+bfaz/a4wp+cBA6si1M8Z8D3ipiPw88HvAb3XsDBrTkfMTkRGsGStvNMbkO7f8pnTqb8+PrOlcgXOUp22OArPdWW6XWW8/km48sC7gfZ7XKawLd4ln28eAP1zFsYY9z98H/HIfndv7sczfz2N9s/lIn127mOf5q4AP99n5RYC/B17R6/PqxPl59r8PeF6vz22954olGn9hP38ncHOvz6ETD1+7mxpwCVAwxuz3bNsHXLGKz14rIo+IyNeBLcBfd2KB62DN52aM+U/GmH9ljHk1cMAY8+ZOLXIdrOfaXWVn/nwFeCvwR51Y4DpZz/ndDLwUeJedefcLnVjgOlnP+SEif4/lSvtTEXlj+5fXVhqeq7Gs2pP2veQKeuOR6Dh+dzfVI03ZF+hwHiuY2RBjzIPAg51YVJtY87l5Mf7tM7Oea/cwVizJz6zn/D5G7XG+fmJd/z+NMT/V9hV1jqbnaoz5D11dUQ8IqiWxQNkX6DACzPdgLe2mn88N9PyCTr+fn5dBOte6BFUk9gMREdnt2XYl/ZES2c/nBnp+Qaffz8/LIJ1rXXwtEiISEZEEEAbCIpIQkYgxJgPcD9wpIikRuQar8MjvprpLP58b6Pmh5xcYBulc10SvI+dNsg3uAEzV4w77vQngb4EM8Azwi71er56bnp+eX/Aeg3Sua3loq3BFURSlLr52NymKoii9RUVCURRFqYuKhKIoilIXFQlFURSlLioSiqIoSl1UJBRFUZS6qEgoiqIodVGRUJQ2IiJGRPpyjKUymKhIKIFCRO4Tkb/r9ToasIkeTpRrhojcISLf7/U6lOCgIqEoTRCR2Gr3Ndb0vGwn11OLVtaoKK2gIqH0FSIyKiL3iMgpEZkXka+JyB7P+5Mi8nEROSoiSyLyuIj8atUxvioid4vIB0VkBnjI3m5E5FYR+ZQ98/igiNxS9VnX3SQi2+3XN4rIF0VkUUR+ICI/WfWZnxaRJ0Vk2R6qdJP9ue0NzvOwbRXcKyLngL+yt/+hfawle58P2M3rsIf8vBu4wj6+cQb/NPu9KYOLioTSN4iIAJ/Dmjj4r4EXAP8EfFlENtm7JYDv2O9fAfwx8L9E5BVVh7sFEOA64Jc92/8L8BmsltGfBO4VkW1NlvZfgY/Yn/k28AkRSdtr3obVafRz9vsfAT6wylP+d8ATwB6s8ZlgNaL7NeBy4E1Ys8//s/3eJ4EPAU9iucU2AZ9c5e9NGVR63WFQH/po5YE1H/nv6rx3A9agmGTV9u8B/7HBMT8B/G/P668Cj9bYzwDv87yOAIvALVX7/Jz9fLv9+jc972+xt11rv34f8MOqf+ed9j7bG6z5MPDAKn5fvwU85Xl9B/D9dvze9DEYj6COL1WUWrwIGAJmrC/HLgngYgARCQPvAH4B64YdB2JYwuDlkTr/xqPOE2NMwXZHbWiyrkc9z4/bP53PXIZlXXj5lybHc9hbvcF2db0V2IU1fjNsPxrR9PemDC4qEko/EQJOYrmIqnFmFb8d+PfAW4DHsL5B/wErb/SZOv9Gvuq1obnb1v2MMcbYN+J2uHor1igiV2NZRe8B3gacA14LfLDJcVbze1MGFBUJpZ/4DrARKBljDtbZ51osN83HwI1jXIJ1Q+0FT2BNO/PykjUe6xrgmDHm950NInJR1T45VloWq/m9KQOKBq6VIDIiIldVPbYD/4iVifQZEXmNiOwQkR8TkfeIiPMteT/wChG5VkQuA/4HsKMnZ2HxJ8DFdibVpSLys8Bv2u+1OhFsP7BFRH5JRHaKyG8DN1ftcxi4SEReKCJTIhJndb83ZUBRkVCCyHXAd6seHzTGGOCngC8Df4qVxfN/gUspxwLeCzwMPIiVwZPBTh/tBcaYI8CNWG6hfVhuovfYby+3eKwHgD8C/jtWHOQnsbKxvPwN8PfAl4AZ4OZV/t6UAUXHlyqKzxCRtwB3AmNG/0CVHqMxCUXpMSLyO1gZTjPA1cC7gPtUIBQ/oCKhKL1nF1ZtxCRwFCtOcWdPV6QoNupuUhRFUeqigWtFURSlLioSiqIoSl1UJBRFUZS6qEgoiqIodVGRUBRFUeqiIqEoiqLU5f8Dk8a3KPXfp7sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "batch_size = 128\n",
    "rates, losses = find_learning_rate(model, X_train_scaled, y_train, epochs=1, batch_size=batch_size)\n",
    "plot_lr_vs_loss(rates, losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OneCycleScheduler(keras.callbacks.Callback): # 1Cycle scheduler function\n",
    "    def __init__(self, iterations, max_rate, start_rate=None,\n",
    "                 last_iterations=None, last_rate=None):\n",
    "        self.iterations = iterations\n",
    "        self.max_rate = max_rate\n",
    "        self.start_rate = start_rate or max_rate / 10\n",
    "        self.last_iterations = last_iterations or iterations // 10 + 1\n",
    "        self.half_iteration = (iterations - self.last_iterations) // 2\n",
    "        self.last_rate = last_rate or self.start_rate / 1000\n",
    "        self.iteration = 0\n",
    "    def _interpolate(self, iter1, iter2, rate1, rate2):\n",
    "        return ((rate2 - rate1) * (self.iteration - iter1)\n",
    "                / (iter2 - iter1) + rate1)\n",
    "    def on_batch_begin(self, batch, logs):\n",
    "        if self.iteration < self.half_iteration:\n",
    "            rate = self._interpolate(0, self.half_iteration, self.start_rate, self.max_rate)\n",
    "        elif self.iteration < 2 * self.half_iteration:\n",
    "            rate = self._interpolate(self.half_iteration, 2 * self.half_iteration,\n",
    "                                     self.max_rate, self.start_rate)\n",
    "        else:\n",
    "            rate = self._interpolate(2 * self.half_iteration, self.iterations,\n",
    "                                     self.start_rate, self.last_rate)\n",
    "            rate = max(rate, self.last_rate)\n",
    "        self.iteration += 1\n",
    "        K.set_value(self.model.optimizer.lr, rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 25000 samples, validate on 5000 samples\n",
      "Epoch 1/5\n",
      "25000/25000 [==============================] - 1s 36us/sample - loss: 0.7365 - accuracy: 0.7454 - val_loss: 0.5076 - val_accuracy: 0.8248\n",
      "Epoch 2/5\n",
      "25000/25000 [==============================] - 1s 32us/sample - loss: 0.4699 - accuracy: 0.8326 - val_loss: 0.4883 - val_accuracy: 0.8290\n",
      "Epoch 3/5\n",
      "25000/25000 [==============================] - 1s 33us/sample - loss: 0.4111 - accuracy: 0.8522 - val_loss: 0.4323 - val_accuracy: 0.8510\n",
      "Epoch 4/5\n",
      "25000/25000 [==============================] - 1s 33us/sample - loss: 0.3567 - accuracy: 0.8699 - val_loss: 0.3995 - val_accuracy: 0.8632\n",
      "Epoch 5/5\n",
      "25000/25000 [==============================] - 1s 33us/sample - loss: 0.3293 - accuracy: 0.8844 - val_loss: 0.3898 - val_accuracy: 0.8632\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 5\n",
    "onecycle = OneCycleScheduler(len(X_train) // batch_size * n_epochs, max_rate=0.05)\n",
    "\n",
    "# training the model\n",
    "history = model.fit(X_train_scaled, y_train, epochs=n_epochs, batch_size=batch_size,\n",
    "                    validation_data=(X_valid_scaled, y_valid),\n",
    "                    callbacks=[onecycle])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Avoiding Overfitting Through Regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loading the Fashion-MNIST dataset\n",
    "(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.fashion_mnist.load_data()\n",
    "\n",
    "# trimming the data since it takes lot of time\n",
    "X_train_full = X_train_full[:30000]\n",
    "y_train_full = y_train_full[:30000]\n",
    "\n",
    "X_test = X_test[:5000]\n",
    "y_test = y_test[:5000]\n",
    "\n",
    "# scaling the dataset\n",
    "X_train_full = X_train_full / 255.0\n",
    "X_test = X_test / 255.0\n",
    "\n",
    "# dividing the dataset into traingin and validation set\n",
    "X_valid, X_train = X_train_full[:5000], X_train_full[5000:]\n",
    "y_valid, y_train = y_train_full[:5000], y_train_full[5000:]\n",
    "\n",
    "# scaling the dataset\n",
    "pixel_means = X_train.mean(axis=0, keepdims=True)\n",
    "pixel_stds = X_train.std(axis=0, keepdims=True)\n",
    "X_train_scaled = (X_train - pixel_means) / pixel_stds\n",
    "X_valid_scaled = (X_valid - pixel_means) / pixel_stds\n",
    "X_test_scaled = (X_test - pixel_means) / pixel_stds\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " $\\ell_1$ and $\\ell_2$ regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "layer = keras.layers.Dense(100, activation=\"elu\",\n",
    "                           kernel_initializer=\"he_normal\",\n",
    "                           kernel_regularizer=keras.regularizers.l2(0.01))\n",
    "# or l1(0.1) for ℓ1 regularization with a factor or 0.1\n",
    "# or l1_l2(0.1, 0.01) for both ℓ1 and ℓ2 regularization, with factors 0.1 and 0.01 respectively"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "782/782 [==============================] - 5s 6ms/step - loss: 2.6742 - accuracy: 0.8070 - val_loss: 0.9272 - val_accuracy: 0.7796\n",
      "Epoch 2/10\n",
      "782/782 [==============================] - 4s 5ms/step - loss: 0.7767 - accuracy: 0.8189 - val_loss: 0.7519 - val_accuracy: 0.8088\n",
      "Epoch 3/10\n",
      "782/782 [==============================] - 4s 5ms/step - loss: 0.7335 - accuracy: 0.8212 - val_loss: 0.7821 - val_accuracy: 0.7900\n",
      "Epoch 4/10\n",
      "782/782 [==============================] - 4s 5ms/step - loss: 0.7111 - accuracy: 0.8294 - val_loss: 0.7242 - val_accuracy: 0.8286\n",
      "Epoch 5/10\n",
      "782/782 [==============================] - 4s 6ms/step - loss: 0.6975 - accuracy: 0.8332 - val_loss: 0.7375 - val_accuracy: 0.8234\n",
      "Epoch 6/10\n",
      "782/782 [==============================] - 5s 6ms/step - loss: 0.6964 - accuracy: 0.8360 - val_loss: 0.7218 - val_accuracy: 0.8300\n",
      "Epoch 7/10\n",
      "782/782 [==============================] - 4s 6ms/step - loss: 0.6920 - accuracy: 0.8371 - val_loss: 0.6885 - val_accuracy: 0.8420\n",
      "Epoch 8/10\n",
      "782/782 [==============================] - 4s 5ms/step - loss: 0.6897 - accuracy: 0.8388 - val_loss: 0.6851 - val_accuracy: 0.8404\n",
      "Epoch 9/10\n",
      "782/782 [==============================] - 5s 6ms/step - loss: 0.6862 - accuracy: 0.8391 - val_loss: 0.6932 - val_accuracy: 0.8444\n",
      "Epoch 10/10\n",
      "782/782 [==============================] - 5s 6ms/step - loss: 0.6871 - accuracy: 0.8413 - val_loss: 0.6825 - val_accuracy: 0.8498\n"
     ]
    }
   ],
   "source": [
    "# defining the model\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.Dense(300, activation=\"elu\",\n",
    "                       kernel_initializer=\"he_normal\",\n",
    "                       kernel_regularizer=keras.regularizers.l2(0.01)),\n",
    "    keras.layers.Dense(100, activation=\"elu\",\n",
    "                       kernel_initializer=\"he_normal\",\n",
    "                       kernel_regularizer=keras.regularizers.l2(0.01)),\n",
    "    keras.layers.Dense(10, activation=\"softmax\",\n",
    "                       kernel_regularizer=keras.regularizers.l2(0.01))\n",
    "])\n",
    "\n",
    "# compiling the model\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"nadam\", metrics=[\"accuracy\"])\n",
    "n_epochs = 10\n",
    "\n",
    "# training the model\n",
    "history = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n",
    "                    validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "RegularizedDense = partial(keras.layers.Dense,\n",
    "                           activation=\"elu\",\n",
    "                           kernel_initializer=\"he_normal\",\n",
    "                           kernel_regularizer=keras.regularizers.l2(0.01))\n",
    "\n",
    "def RegularizedDense(xyz):\n",
    "    keras.layers.Dense(activation=\"elu\",\n",
    "                           kernel_initializer=\"he_normal\",\n",
    "                           kernel_regularizer=keras.regularizers.l2(0.01))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "782/782 [==============================] - 5s 6ms/step - loss: 2.7121 - accuracy: 0.8054 - val_loss: 0.9632 - val_accuracy: 0.7818\n"
     ]
    }
   ],
   "source": [
    "# creates a thin wrapper for any callable, with some default argument values\n",
    "from functools import partial\n",
    "\n",
    "RegularizedDense = partial(keras.layers.Dense,\n",
    "                           activation=\"elu\",\n",
    "                           kernel_initializer=\"he_normal\",\n",
    "                           kernel_regularizer=keras.regularizers.l2(0.01))\n",
    "\n",
    "# defining the model\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    RegularizedDense(300),# keras.layers.Dense(300, activation=\"elu\", kernel_initializer=\"he_normal\",\n",
    "                           # kernel_regularizer=keras.regularizers.l2(0.01))\n",
    "    RegularizedDense(100),\n",
    "    RegularizedDense(10, activation=\"softmax\")\n",
    "])\n",
    "\n",
    "# compiling the model\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"nadam\", metrics=[\"accuracy\"])\n",
    "n_epochs = 20\n",
    "\n",
    "es = tf.keras.callbacks.EarlyStopping(\n",
    "    monitor='val_loss',\n",
    "    min_delta=0,\n",
    "    patience=0,\n",
    "    verbose=0,\n",
    "    mode='auto',\n",
    "    baseline=None,\n",
    "    restore_best_weights=False\n",
    ")\n",
    "\n",
    "# training the model\n",
    "history = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n",
    "                    validation_data=(X_valid_scaled, y_valid), callbacks=[es])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.54340494 0.27836939 0.42451759 0.84477613]]\n",
      "[[0.00471886 0.12156912 0.67074908 0.82585276]]\n",
      "[[0.13670659 0.57509333 0.89132195 0.20920212]]\n",
      "[[0.18532822 0.10837689 0.21969749 0.97862378]]\n",
      "[[0.81168315 0.17194101 0.81622475 0.27407375]]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.random.seed(100)\n",
    "print(np.random.random((1, 4)))\n",
    "print(np.random.random((1, 4)))\n",
    "print(np.random.random((1, 4)))\n",
    "print(np.random.random((1, 4)))\n",
    "print(np.random.random((1, 4)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.54340494 0.27836939 0.42451759 0.84477613]]\n",
      "[[0.00471886 0.12156912 0.67074908 0.82585276]]\n",
      "[[0.13670659 0.57509333 0.89132195 0.20920212]]\n",
      "[[0.18532822 0.10837689 0.21969749 0.97862378]]\n",
      "[[0.81168315 0.17194101 0.81622475 0.27407375]]\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(100)\n",
    "print(np.random.random((1, 4)))\n",
    "print(np.random.random((1, 4)))\n",
    "print(np.random.random((1, 4)))\n",
    "print(np.random.random((1, 4)))\n",
    "print(np.random.random((1, 4)))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "782/782 [==============================] - 5s 5ms/step - loss: 1.2469 - accuracy: 0.6491 - val_loss: 0.4763 - val_accuracy: 0.8296\n",
      "Epoch 2/20\n",
      "782/782 [==============================] - 4s 5ms/step - loss: 0.7221 - accuracy: 0.7368 - val_loss: 0.4439 - val_accuracy: 0.8378\n",
      "Epoch 3/20\n",
      "782/782 [==============================] - 4s 5ms/step - loss: 0.6426 - accuracy: 0.7656 - val_loss: 0.4217 - val_accuracy: 0.8472\n",
      "Epoch 4/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.6043 - accuracy: 0.7788 - val_loss: 0.4096 - val_accuracy: 0.8484\n",
      "Epoch 5/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.5855 - accuracy: 0.7829 - val_loss: 0.3978 - val_accuracy: 0.8532\n",
      "Epoch 6/20\n",
      "782/782 [==============================] - 4s 5ms/step - loss: 0.5671 - accuracy: 0.7905 - val_loss: 0.3969 - val_accuracy: 0.8546\n",
      "Epoch 7/20\n",
      "782/782 [==============================] - 4s 5ms/step - loss: 0.5602 - accuracy: 0.7923 - val_loss: 0.3988 - val_accuracy: 0.8480\n",
      "Epoch 8/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.5587 - accuracy: 0.7956 - val_loss: 0.3926 - val_accuracy: 0.8566\n",
      "Epoch 9/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.5483 - accuracy: 0.7956 - val_loss: 0.3890 - val_accuracy: 0.8570\n",
      "Epoch 10/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.5414 - accuracy: 0.8005 - val_loss: 0.3826 - val_accuracy: 0.8544\n",
      "Epoch 11/20\n",
      "782/782 [==============================] - 4s 5ms/step - loss: 0.5323 - accuracy: 0.8028 - val_loss: 0.3809 - val_accuracy: 0.8586\n",
      "Epoch 12/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.5314 - accuracy: 0.8061 - val_loss: 0.3666 - val_accuracy: 0.8612\n",
      "Epoch 13/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.5248 - accuracy: 0.8068 - val_loss: 0.3662 - val_accuracy: 0.8642\n",
      "Epoch 14/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.5233 - accuracy: 0.8057 - val_loss: 0.3650 - val_accuracy: 0.8604\n",
      "Epoch 15/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.5294 - accuracy: 0.8043 - val_loss: 0.3726 - val_accuracy: 0.8604\n",
      "Epoch 16/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.5195 - accuracy: 0.8082 - val_loss: 0.3622 - val_accuracy: 0.8632\n",
      "Epoch 17/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.5218 - accuracy: 0.8099 - val_loss: 0.3728 - val_accuracy: 0.8610\n",
      "Epoch 18/20\n",
      "782/782 [==============================] - 4s 5ms/step - loss: 0.5153 - accuracy: 0.8133 - val_loss: 0.3644 - val_accuracy: 0.8630\n",
      "Epoch 19/20\n",
      "782/782 [==============================] - 4s 5ms/step - loss: 0.5197 - accuracy: 0.8104 - val_loss: 0.3513 - val_accuracy: 0.8676\n",
      "Epoch 20/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.5055 - accuracy: 0.8158 - val_loss: 0.3522 - val_accuracy: 0.8642\n"
     ]
    }
   ],
   "source": [
    "# defining a model applying dropout regularization\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.Dropout(rate=0.5),\n",
    "    keras.layers.Dense(300, activation=\"elu\", kernel_initializer=\"he_normal\"),\n",
    "    keras.layers.Dropout(rate=0.5),\n",
    "    keras.layers.Dense(100, activation=\"elu\", kernel_initializer=\"he_normal\"),\n",
    "    keras.layers.Dropout(rate=0.5),\n",
    "    keras.layers.Dense(10, activation=\"softmax\")\n",
    "])\n",
    "\n",
    "# compiling the model\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"nadam\", metrics=[\"accuracy\"])\n",
    "n_epochs = 20\n",
    "\n",
    "# training the model\n",
    "history = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n",
    "                    validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.5106 - accuracy: 0.8106 - val_loss: 0.3606 - val_accuracy: 0.8634\n",
      "Epoch 2/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.5085 - accuracy: 0.8149 - val_loss: 0.3481 - val_accuracy: 0.8672\n",
      "Epoch 3/20\n",
      "782/782 [==============================] - 4s 5ms/step - loss: 0.5004 - accuracy: 0.8166 - val_loss: 0.3527 - val_accuracy: 0.8700\n",
      "Epoch 4/20\n",
      "782/782 [==============================] - 4s 5ms/step - loss: 0.5006 - accuracy: 0.8125 - val_loss: 0.3569 - val_accuracy: 0.8672\n",
      "Epoch 5/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.5002 - accuracy: 0.8165 - val_loss: 0.3471 - val_accuracy: 0.8692\n",
      "Epoch 6/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.5005 - accuracy: 0.8147 - val_loss: 0.3508 - val_accuracy: 0.8740\n",
      "Epoch 7/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.5122 - accuracy: 0.8152 - val_loss: 0.3549 - val_accuracy: 0.8660\n",
      "Epoch 8/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.4941 - accuracy: 0.8171 - val_loss: 0.3495 - val_accuracy: 0.8678\n",
      "Epoch 9/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.4916 - accuracy: 0.8184 - val_loss: 0.3554 - val_accuracy: 0.8668\n",
      "Epoch 10/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.4964 - accuracy: 0.8199 - val_loss: 0.3546 - val_accuracy: 0.8678\n",
      "Epoch 11/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.4994 - accuracy: 0.8197 - val_loss: 0.3482 - val_accuracy: 0.8676\n",
      "Epoch 12/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.4979 - accuracy: 0.8169 - val_loss: 0.3561 - val_accuracy: 0.8708\n",
      "Epoch 13/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.4949 - accuracy: 0.8201 - val_loss: 0.3500 - val_accuracy: 0.8676\n",
      "Epoch 14/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.4929 - accuracy: 0.8220 - val_loss: 0.3583 - val_accuracy: 0.8648\n",
      "Epoch 15/20\n",
      "782/782 [==============================] - 4s 5ms/step - loss: 0.4893 - accuracy: 0.8213 - val_loss: 0.3549 - val_accuracy: 0.8740\n",
      "Epoch 16/20\n",
      "782/782 [==============================] - 4s 5ms/step - loss: 0.4903 - accuracy: 0.8204 - val_loss: 0.3656 - val_accuracy: 0.8650\n",
      "Epoch 17/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.4948 - accuracy: 0.8198 - val_loss: 0.3692 - val_accuracy: 0.8630\n",
      "Epoch 18/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.4902 - accuracy: 0.8225 - val_loss: 0.3615 - val_accuracy: 0.8712\n",
      "Epoch 19/20\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.4973 - accuracy: 0.8202 - val_loss: 0.3396 - val_accuracy: 0.8684\n",
      "Epoch 20/20\n",
      "782/782 [==============================] - 4s 6ms/step - loss: 0.4803 - accuracy: 0.8264 - val_loss: 0.3385 - val_accuracy: 0.8704\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# training the model\n",
    "history = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n",
    "                    validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Alpha Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.random.set_seed(42)\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 25000 samples, validate on 5000 samples\n",
      "Epoch 1/5\n",
      "25000/25000 [==============================] - 3s 130us/sample - loss: 0.7427 - accuracy: 0.7301 - val_loss: 0.7011 - val_accuracy: 0.8206\n",
      "Epoch 2/5\n",
      "25000/25000 [==============================] - 3s 112us/sample - loss: 0.6102 - accuracy: 0.7749 - val_loss: 0.6291 - val_accuracy: 0.8362\n",
      "Epoch 3/5\n",
      "25000/25000 [==============================] - 3s 112us/sample - loss: 0.5640 - accuracy: 0.7907 - val_loss: 0.6171 - val_accuracy: 0.8366\n",
      "Epoch 4/5\n",
      "25000/25000 [==============================] - 3s 113us/sample - loss: 0.5493 - accuracy: 0.7936 - val_loss: 0.5535 - val_accuracy: 0.8474\n",
      "Epoch 5/5\n",
      "25000/25000 [==============================] - 3s 113us/sample - loss: 0.5246 - accuracy: 0.8036 - val_loss: 0.5671 - val_accuracy: 0.8512\n"
     ]
    }
   ],
   "source": [
    "# using alpha dropout to define a model\n",
    "# it keeps mean and variance of inputs to their original values\n",
    "# in order to ensure the self-normalizing property even after this dropout\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.AlphaDropout(rate=0.2),\n",
    "    keras.layers.Dense(300, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n",
    "    keras.layers.AlphaDropout(rate=0.2),\n",
    "    keras.layers.Dense(100, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n",
    "    keras.layers.AlphaDropout(rate=0.2),\n",
    "    keras.layers.Dense(10, activation=\"softmax\")\n",
    "])\n",
    "optimizer = keras.optimizers.SGD(lr=0.01, momentum=0.9, nesterov=True)\n",
    "\n",
    "# compiling the model\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])\n",
    "n_epochs = 5\n",
    "\n",
    "# training the model\n",
    "history = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n",
    "                    validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5000/5000 [==============================] - 0s 48us/sample - loss: 0.5935 - accuracy: 0.8372\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.59354141061306, 0.8372]"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(X_test_scaled, y_test) # model evaluation on test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25000/25000 [==============================] - 1s 47us/sample - loss: 0.5091 - accuracy: 0.8466\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.509081173028946, 0.84664]"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(X_train_scaled, y_train) # model evaluation on train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 25000 samples\n",
      "25000/25000 [==============================] - 3s 119us/sample - loss: 0.5129 - accuracy: 0.8086\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train_scaled, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MC Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.random.set_seed(42)\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Layer flatten_18 is casting an input tensor from dtype float64 to the layer's dtype of float32, which is new behavior in TensorFlow 2.  The layer has dtype float32 because it's dtype defaults to floatx.\n",
      "\n",
      "If you intended to run this layer in float32, you can safely ignore this warning. If in doubt, this warning is likely only an issue if you are porting a TensorFlow 1.X model to TensorFlow 2.\n",
      "\n",
      "To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# making 100 predictions\n",
    "y_probas = np.stack([model(X_test_scaled, training=True)\n",
    "                     for sample in range(100)])\n",
    "y_proba = y_probas.mean(axis=0)\n",
    "y_std = y_probas.std(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.02, 0.  , 0.97]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.round(model.predict(X_test_scaled[:1]), 2) # prediction for first instance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[0.  , 0.  , 0.  , 0.  , 0.  , 0.15, 0.  , 0.64, 0.  , 0.21]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.34, 0.  , 0.55, 0.  , 0.11]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.69, 0.  , 0.04, 0.  , 0.27]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.19, 0.  , 0.48, 0.01, 0.33]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.12, 0.  , 0.31, 0.  , 0.57]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.35, 0.02, 0.63]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.1 , 0.  , 0.09, 0.  , 0.81]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.29, 0.  , 0.71]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.96, 0.  , 0.01, 0.  , 0.03]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.02, 0.  , 0.96]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.04, 0.  , 0.06, 0.  , 0.9 ]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.91, 0.  , 0.01, 0.  , 0.08]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.64, 0.  , 0.25, 0.  , 0.11]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.17, 0.  , 0.49, 0.  , 0.32]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.19, 0.  , 0.13, 0.  , 0.68]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.3 , 0.  , 0.3 , 0.01, 0.39]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.05, 0.  , 0.  , 0.  , 0.95]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.39, 0.  , 0.23, 0.  , 0.38]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.13, 0.  , 0.86]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.83, 0.  , 0.15, 0.  , 0.02]],\n",
       "\n",
       "       [[0.  , 0.  , 0.01, 0.  , 0.  , 0.51, 0.01, 0.23, 0.  , 0.23]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.22, 0.  , 0.16, 0.  , 0.62]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.03, 0.  , 0.96]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.17, 0.  , 0.82]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.13, 0.  , 0.15, 0.  , 0.72]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.17, 0.  , 0.37, 0.  , 0.47]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.11, 0.  , 0.03, 0.  , 0.86]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.4 , 0.  , 0.24, 0.  , 0.36]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.29, 0.  , 0.16, 0.  , 0.55]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.21, 0.  , 0.68, 0.01, 0.1 ]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.06, 0.  , 0.08, 0.  , 0.86]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.33, 0.  , 0.02, 0.  , 0.65]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.32, 0.  , 0.1 , 0.  , 0.58]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.48, 0.  , 0.47, 0.  , 0.05]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.37, 0.  , 0.23, 0.  , 0.4 ]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.06, 0.  , 0.25, 0.  , 0.69]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.03, 0.  , 0.24, 0.  , 0.73]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.07, 0.  , 0.11, 0.  , 0.82]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.23, 0.  , 0.6 , 0.  , 0.17]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.54, 0.  , 0.37, 0.  , 0.09]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.03, 0.  , 0.21, 0.  , 0.76]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.15, 0.  , 0.02, 0.  , 0.83]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.8 , 0.  , 0.14, 0.  , 0.06]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.03, 0.  , 0.02, 0.  , 0.95]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.26, 0.  , 0.1 , 0.  , 0.64]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.06, 0.  , 0.94]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.04, 0.  , 0.09, 0.  , 0.87]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.28, 0.  , 0.72]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.1 , 0.  , 0.43, 0.  , 0.47]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.17, 0.  , 0.81]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.71, 0.  , 0.05, 0.  , 0.24]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.09, 0.  , 0.77, 0.  , 0.15]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.03, 0.  , 0.96]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.31, 0.  , 0.13, 0.  , 0.55]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.78, 0.  , 0.02, 0.  , 0.2 ]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.05, 0.  , 0.05, 0.  , 0.9 ]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.1 , 0.  , 0.18, 0.  , 0.71]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.83, 0.  , 0.07, 0.  , 0.1 ]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.07, 0.  , 0.74, 0.  , 0.19]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.05, 0.  , 0.23, 0.  , 0.72]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.35, 0.  , 0.25, 0.  , 0.4 ]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.18, 0.  , 0.8 ]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.37, 0.  , 0.14, 0.  , 0.49]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.16, 0.  , 0.82]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.48, 0.  , 0.16, 0.  , 0.36]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.12, 0.  , 0.64, 0.  , 0.23]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.72, 0.  , 0.08, 0.  , 0.2 ]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.04, 0.  , 0.95]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.43, 0.  , 0.09, 0.  , 0.48]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.04, 0.  , 0.24, 0.  , 0.72]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.04, 0.  , 0.96]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.11, 0.  , 0.19, 0.  , 0.7 ]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.22, 0.  , 0.24, 0.  , 0.54]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.09, 0.  , 0.65, 0.  , 0.26]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.41, 0.  , 0.07, 0.  , 0.52]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.21, 0.  , 0.08, 0.  , 0.71]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.04, 0.  , 0.39, 0.  , 0.57]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.03, 0.  , 0.37, 0.  , 0.6 ]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.33, 0.  , 0.33, 0.  , 0.34]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.16, 0.  , 0.82]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.27, 0.  , 0.27, 0.  , 0.46]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.06, 0.  , 0.01, 0.  , 0.93]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.17, 0.  , 0.02, 0.  , 0.81]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.04, 0.  , 0.09, 0.  , 0.87]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.61, 0.  , 0.37]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.44, 0.  , 0.11, 0.  , 0.45]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.63, 0.  , 0.14, 0.  , 0.22]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.7 , 0.  , 0.02, 0.  , 0.27]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.07, 0.  , 0.32, 0.  , 0.61]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.22, 0.  , 0.08, 0.  , 0.7 ]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.16, 0.  , 0.22, 0.  , 0.63]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.05, 0.  , 0.23, 0.02, 0.7 ]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.65, 0.  , 0.16, 0.01, 0.18]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.76, 0.  , 0.07, 0.  , 0.17]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.05, 0.  , 0.1 , 0.  , 0.85]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.04, 0.  , 0.95]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.02, 0.  , 0.32, 0.  , 0.66]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.21, 0.  , 0.08, 0.  , 0.7 ]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.39, 0.  , 0.23, 0.  , 0.38]],\n",
       "\n",
       "       [[0.  , 0.  , 0.  , 0.  , 0.  , 0.63, 0.  , 0.05, 0.  , 0.31]]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.round(y_probas[:, :1], 2) # comparning with predictions made when dropout is activated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.  , 0.  , 0.  , 0.  , 0.  , 0.25, 0.  , 0.21, 0.  , 0.55]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.round(y_proba[:1], 2) # model prediction after we average over the first dimension"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.  , 0.  , 0.  , 0.  , 0.  , 0.25, 0.  , 0.18, 0.  , 0.28]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# standard deviation of the probability estimates\n",
    "y_std = y_probas.std(axis=0)\n",
    "np.round(y_std[:1], 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = np.argmax(y_proba, axis=1) # generating y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.837"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# checking model accuracy\n",
    "accuracy = np.sum(y_pred == y_test) / len(y_test)\n",
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "# replacing the Dropout layers with the following MCDropout class\n",
    "# if your model contains other layers that behave in a special way\n",
    "# during training (such as BatchNormalization layers)\n",
    "class MCDropout(keras.layers.Dropout):\n",
    "    def call(self, inputs):\n",
    "        return super().call(inputs, training=True)\n",
    "\n",
    "# defining MCAlphaDropout by subclassing AlphaDropout\n",
    "class MCAlphaDropout(keras.layers.AlphaDropout):\n",
    "    def call(self, inputs):\n",
    "        return super().call(inputs, training=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.random.set_seed(42)\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "# using the MCAlphaDropout\n",
    "mc_model = keras.models.Sequential([\n",
    "    MCAlphaDropout(layer.rate) if isinstance(layer, keras.layers.AlphaDropout) else layer\n",
    "    for layer in model.layers\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_20\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_18 (Flatten)         (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "mc_alpha_dropout (MCAlphaDro (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_262 (Dense)            (None, 300)               235500    \n",
      "_________________________________________________________________\n",
      "mc_alpha_dropout_1 (MCAlphaD (None, 300)               0         \n",
      "_________________________________________________________________\n",
      "dense_263 (Dense)            (None, 100)               30100     \n",
      "_________________________________________________________________\n",
      "mc_alpha_dropout_2 (MCAlphaD (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "dense_264 (Dense)            (None, 10)                1010      \n",
      "=================================================================\n",
      "Total params: 266,610\n",
      "Trainable params: 266,610\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "mc_model.summary() # model summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compiling the model\n",
    "optimizer = keras.optimizers.SGD(lr=0.01, momentum=0.9, nesterov=True)\n",
    "mc_model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "mc_model.set_weights(model.get_weights()) # setting weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can use the model with MC Dropout:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.  , 0.  , 0.  , 0.  , 0.  , 0.29, 0.  , 0.23, 0.  , 0.48]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.round(np.mean([mc_model.predict(X_test_scaled[:1]) for sample in range(100)], axis=0), 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Max norm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "layer = keras.layers.Dense(100, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                           kernel_constraint=keras.constraints.max_norm(1.))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 25000 samples, validate on 5000 samples\n",
      "Epoch 1/2\n",
      "25000/25000 [==============================] - 4s 159us/sample - loss: 0.5422 - accuracy: 0.8110 - val_loss: 0.5103 - val_accuracy: 0.8274\n",
      "Epoch 2/2\n",
      "25000/25000 [==============================] - 3s 126us/sample - loss: 0.3890 - accuracy: 0.8562 - val_loss: 0.4113 - val_accuracy: 0.8552\n"
     ]
    }
   ],
   "source": [
    "# implementing max-norm regularization by setting max_norm\n",
    "MaxNormDense = partial(keras.layers.Dense,\n",
    "                       activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       kernel_constraint=keras.constraints.max_norm(1.))\n",
    "\n",
    "# defining model\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    MaxNormDense(300),\n",
    "    MaxNormDense(100),\n",
    "    keras.layers.Dense(10, activation=\"softmax\")\n",
    "])\n",
    "\n",
    "# compiling model\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"nadam\", metrics=[\"accuracy\"])\n",
    "n_epochs = 2\n",
    "\n",
    "# training model\n",
    "history = model.fit(X_train_scaled, y_train, epochs=n_epochs,\n",
    "                    validation_data=(X_valid_scaled, y_valid))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tensorflow2",
   "language": "python",
   "name": "tensorflow2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "nav_menu": {
   "height": "360px",
   "width": "416px"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}