CIFAR_BENCHMARK.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import pylab as plt\n",
    "import numpy as np\n",
    "import torch\n",
    "import torchvision\n",
    "# from torch_lr_finder import LRFinder\n",
    "import time as time\n",
    "import sys\n",
    "from ipywidgets import IntProgress\n",
    "from torch import nn\n",
    "import notebook_utils as nbutils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "# SETUP GPU\n",
    "torch.backends.cudnn.benchmark = True\n",
    "device = torch.device(\"cuda:1\")\n",
    "base = \"/home/rahul/lab_work/data/\"\n",
    "download = True\n",
    "train_dataset = torchvision.datasets.CIFAR10(\n",
    "    root=base,\n",
    "    train=True,\n",
    "    download=download\n",
    ")\n",
    "\n",
    "test_dataset = torchvision.datasets.CIFAR10(\n",
    "    root=base,\n",
    "    train=False,\n",
    "    download=download\n",
    ")\n",
    "\n",
    "class reformed_CIFAR10(torchvision.datasets.CIFAR10):\n",
    "    \n",
    "    def __init__(self, root, train, download, transform, start=0, end=-1):\n",
    "        super(reformed_CIFAR10, self).__init__(root=root, train=train, download=download, transform=transform)\n",
    "        if end==-1:\n",
    "            end = len(self.targets)\n",
    "        self.data = self.data[start: end]\n",
    "        self.targets = self.targets[start: end]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CutOut(object):\n",
    "    \"\"\"\n",
    "    CutOut augmentation -- replace a box by its mean (coordinate-wise)\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, side):\n",
    "        self.side = side\n",
    "\n",
    "    def __call__(self, image):\n",
    "        xx = np.random.randint(0, 32-self.side)\n",
    "        yy = np.random.randint(0, 32-self.side)\n",
    "        for c in range(3):\n",
    "            image[c,xx:(xx+self.side), yy:(yy+self.side)]=torch.mean(image[c,xx:(xx+self.side), yy:(yy+self.side)])\n",
    "        return image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PixelJitter(object):\n",
    "    \"\"\"Crop randomly the image in a sample.\n",
    "\n",
    "    Args:\n",
    "        output_size (tuple or int): Desired output size. If int, square crop\n",
    "            is made.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, power_min = 0.4, power_max = 1.4):\n",
    "        assert 0 < power_min\n",
    "        assert power_min < power_max\n",
    "        self.power_min = power_min\n",
    "        self.power_max = power_max\n",
    "\n",
    "    def __call__(self, image):\n",
    "        p = self.power_min + (self.power_max - self.power_min)*np.random.rand()\n",
    "        image = image**p\n",
    "        for c in range(3):\n",
    "            alpha = -0.1 + 0.2*np.random.rand()\n",
    "            strech = 0.8 + 0.4*np.random.rand()\n",
    "            image[c,:,:] = alpha + strech*image[c,:,:]\n",
    "        image = torch.clamp(image, min=0., max=1.)\n",
    "        return image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mixup_data(x, y, alpha=1.0, use_cuda=True):\n",
    "    '''Returns mixed inputs, pairs of targets, and lambda'''\n",
    "    if alpha > 0:\n",
    "        lam = np.random.beta(alpha, alpha)\n",
    "    else:\n",
    "        lam = 1\n",
    "\n",
    "    batch_size = x.size()[0]\n",
    "    if use_cuda:\n",
    "        index = torch.randperm(batch_size).to(device)\n",
    "    else:\n",
    "        index = torch.randperm(batch_size)\n",
    "\n",
    "    mixed_x = lam * x + (1 - lam) * x[index, :]\n",
    "    y_a, y_b = y, y[index]\n",
    "    return mixed_x, y_a, y_b, lam\n",
    "\n",
    "\n",
    "def mixup_criterion(criterion, pred, y_a, y_b, lam):\n",
    "    return lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_augment = torchvision.transforms.Compose([        \n",
    "                                    torchvision.transforms.Pad( 5, padding_mode='edge'),\n",
    "                                    torchvision.transforms.RandomCrop( (32,32), padding_mode='constant'),\n",
    "                                    torchvision.transforms.RandomHorizontalFlip(),\n",
    "                                    #torchvision.transforms.Cutout(8, 8)\n",
    "                                    #torchvision.transforms.Normalize(\n",
    "                                    #    mean=(0.4914, 0.4822, 0.4465),\n",
    "                                    #    std=(0.2471, 0.2435, 0.2616)\n",
    "                                    #),\n",
    "                                    torchvision.transforms.ToTensor(),\n",
    "                                    CutOut(side = 12),\n",
    "                                    PixelJitter(),\n",
    "                                ])\n",
    "\n",
    "\n",
    "test_augment = torchvision.transforms.Compose([        \n",
    "                                    torchvision.transforms.ToTensor(),\n",
    "                                ])\n",
    "\n",
    "\n",
    "#DEFINE DATASET\n",
    "cifar_train_dataset = reformed_CIFAR10(base, train=True, transform=train_augment, download=False, end=5000)\n",
    "\n",
    "cifar_train_dataset_noaugment = reformed_CIFAR10(base, train=True, transform=test_augment, download=False)\n",
    "\n",
    "cifar_test_dataset = reformed_CIFAR10(base, train=False, transform= test_augment, download=False)\n",
    "\n",
    "train_batchsize = 512\n",
    "dataloader_train = torch.utils.data.DataLoader(cifar_train_dataset, batch_size=train_batchsize, \n",
    "                                               shuffle=True, num_workers=5)\n",
    "\n",
    "dataloader_trainnoaugment = torch.utils.data.DataLoader(cifar_train_dataset_noaugment, batch_size=train_batchsize, \n",
    "                                               shuffle=True, num_workers=5)\n",
    "\n",
    "dataloader_test = torch.utils.data.DataLoader(cifar_test_dataset, batch_size=train_batchsize, \n",
    "                                               shuffle=True, num_workers=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Flatten(torch.nn.Module):\n",
    "    def forward(self, x):\n",
    "        return x.view(x.size(0), -1)\n",
    "    \n",
    "def conv_bn(channels_in, channels_out, kernel_size=3, stride=1, padding=1, groups=1, bn=True, activation=True, bias=True):\n",
    "    op = [\n",
    "            torch.nn.Conv2d(channels_in, channels_out,\n",
    "                            kernel_size=kernel_size, stride=stride, padding=padding, groups=groups, bias=bias),\n",
    "    ]\n",
    "    if bn:\n",
    "        op.append(torch.nn.BatchNorm2d(channels_out))\n",
    "    if activation:\n",
    "        op.append(torch.nn.ReLU(inplace=True))\n",
    "    return torch.nn.Sequential(*op)\n",
    "\n",
    "\n",
    "class Residual(torch.nn.Module):\n",
    "    def __init__(self, module):\n",
    "        super(Residual, self).__init__()\n",
    "        self.module = module\n",
    "\n",
    "    def forward(self, x):\n",
    "        return x + self.module(x)\n",
    "\n",
    "\n",
    "class fast_resnet(nn.Module):\n",
    "    \n",
    "    def __init__(self, num_class=10):\n",
    "        super(fast_resnet, self).__init__()\n",
    "        self.encoder = torch.nn.Sequential(\n",
    "            conv_bn(3, 64, kernel_size=3, stride=1, padding=1),\n",
    "            conv_bn(64, 128, kernel_size=5, stride=2, padding=2),\n",
    "            # torch.nn.MaxPool2d(2),\n",
    "\n",
    "            Residual(torch.nn.Sequential(\n",
    "                conv_bn(128, 128),\n",
    "                conv_bn(128, 128),\n",
    "            )),\n",
    "\n",
    "            conv_bn(128, 256, kernel_size=3, stride=1, padding=1),\n",
    "            torch.nn.MaxPool2d(2),\n",
    "\n",
    "            Residual(torch.nn.Sequential(\n",
    "                conv_bn(256, 256),\n",
    "                conv_bn(256, 256),\n",
    "            )),\n",
    "\n",
    "            conv_bn(256, 128, kernel_size=3, stride=1, padding=0),\n",
    "\n",
    "            torch.nn.AdaptiveMaxPool2d((1, 1)),\n",
    "            Flatten())\n",
    "        self.fc = torch.nn.Linear(128, num_class, bias=False)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = self.encoder(x)\n",
    "        x = self.fc(x)\n",
    "        return (x)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, dataloader_train, device, optimizer, scheduler, criterion):\n",
    "    model.train()\n",
    "    train_loss_list = []\n",
    "    start = time.time()\n",
    "    for batch in dataloader_train:\n",
    "        inputs, targets = batch[0].to(device), batch[1].to(device)\n",
    "        \n",
    "        #inputs, targets_a, targets_b, lam = mixup_data(inputs, targets, 0.1, True)\n",
    "        #inputs, targets_a, targets_b = inputs, targets_a, targets_b\n",
    "        \n",
    "        outputs = model(inputs)\n",
    "        #loss = mixup_criterion(criterion, outputs, targets_a, targets_b, lam)\n",
    "        \n",
    "        loss = criterion(outputs, targets)\n",
    "        train_loss_list.append(loss.item())\n",
    "        loss.backward()\n",
    "        \n",
    "        optimizer.step()\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        #update learning rate\n",
    "        scheduler.step()\n",
    "        \n",
    "    compute_time = time.time() - start\n",
    "    avg_loss = np.mean(np.array(train_loss_list))\n",
    "    return {\"avg_loss\": avg_loss,\n",
    "            \"compute_time\": compute_time}\n",
    "\n",
    "\n",
    "def evaluate(model, dataloader, criterion, device):\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    total_loss = 0.\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for batch in dataloader:\n",
    "            images, labels = batch[0].to(device), batch[1].to(device)\n",
    "            pred = model(images)\n",
    "            # LOSS\n",
    "            loss = criterion(pred, labels)\n",
    "            total_loss += loss.item() * labels.size(0)\n",
    "            # ACCURACY\n",
    "            predicted = torch.argmax(pred.data, 1)\n",
    "            correct += (predicted == labels).sum().item()\n",
    "            \n",
    "            total += labels.size(0)\n",
    "    accuracy = correct / float(total)\n",
    "    avg_loss = total_loss / float(total)\n",
    "    return {\"accuracy\": accuracy,\n",
    "            \"avg_loss\": avg_loss}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_lr(optimizer):\n",
    "    for param_group in optimizer.param_groups:\n",
    "        return {\"lr\": param_group['lr'],\n",
    "                \"momentum\": param_group[\"momentum\"]}\n",
    "    \n",
    "def create_optim_schedule(model, n_epoch, max_lr):\n",
    "    optimizer = torch.optim.SGD(model.parameters(),\n",
    "                            lr=0.01,\n",
    "                            momentum=0.9,\n",
    "                            weight_decay=5e-4,\n",
    "                            nesterov=True)\n",
    "\n",
    "    scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, \n",
    "                                                max_lr = max_lr, \n",
    "                                                epochs=n_epoch, \n",
    "                                                steps_per_epoch=len(dataloader_train), \n",
    "                                                pct_start=0.05,  # 0.25 \n",
    "                                                anneal_strategy='linear', \n",
    "                                                cycle_momentum=False, \n",
    "                                                #cycle_momentum=True, base_momentum=0.9, max_momentum=0.9, \n",
    "                                                div_factor=1000.0, # 25\n",
    "                                                final_div_factor=10000.0, \n",
    "                                                last_epoch=-1)\n",
    "    return optimizer, scheduler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CHECK REASONABLE LEARNING RATE\n",
    "model = fast_resnet(num_class=10).to(device)\n",
    "criterion = torch.nn.CrossEntropyLoss(reduction='mean')\n",
    "\n",
    "n_epoch = 200\n",
    "optimizer, scheduler = create_optim_schedule(model, n_epoch, max_lr=0.1)\n",
    "\n",
    "# lr_finder = LRFinder(model, optimizer, criterion, device=device)\n",
    "# lr_finder.range_test(dataloader_train, end_lr=10, num_iter=200)\n",
    "# plt.grid(True)\n",
    "# lr_finder.plot()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'Epoch')"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEWCAYAAACKSkfIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xd8VHX2//HXSQdCLxHpCiyCCgKKUiKsKwQs6Ar2LmJDBNf9quuu67q6+9PdFRCx94oFC0t3XTD03gWkCIgUgSRAEsKknN8fc+OOMSETMpk7mXuej0cezNz53LlnbsI7N5+5c66oKsYYY7whxu0CjDHGhI+FvjHGeIiFvjHGeIiFvjHGeIiFvjHGeIiFvjHGeIiFvol4IjJdRG5yuw43icgcERkWoud6TETeDfVYUz1Y6Jsyich2EfmN23Wo6kBVfcvtOqDy4SsinURklohkikiWiCwXkUGhrNGY47HQN64SkTi3aygWplr+DXwJpABNgJHA4TBs1xjAQt+cIBG5WERWOUerC0TkzIDHHhKRrSJyRES+EZHLAx67WUTmi8gYEckAHnOWzRORfzpHwN+JyMCAdX46ug5ibBsRSXe2/R8RmVDW9ISI9BWRXSLyoIjsBd4QkfoiMkVE9jvPP0VEmjvjnwT6AM+JSLaIPOcs7yAiX4pIhohsEpEry9heI6AN8Iqq+pyv+ao6L2DMYGe/Hnb2YVrAU7Ry9t0R56+FRgHrnet8H7JEZLWI9C2xT7521vsSCFyvr4jsKlFnmX/hHW87pnqw0DcVJiJdgdeBO4CGwEvAZBFJdIZsxR+OdYG/AO+KSNOAp+gBbMN/pPtkwLJN+APpaeA1EZEySjje2PeBJU5djwE3lPNyTgIaAK2A4fj/T7zh3G8JHAWeA1DVR4C5wAhVTVbVESJSC/+R+/vO67kGeF5EOpWyrYPAFmd/XCYiKYEPisg5wNvA74F6QCqwPWDItcAtznYSgAec9ZoBU4EnnNfyADBJRBoH7JPlzv76K3BC748EsR1TDVjomxNxO/CSqi5W1UJnvv0YcC6Aqn6sqrtVtUhVPwQ2A+cErL9bVceraoGqHnWW7VDVV1S1EHgLaIp/CqQ0pY4VkZbA2cCjzlH0PGByOa+lCPizqh5T1aOqelBVJ6lqrqoewf9L6fzjrH8xsF1V33BezwpgEjCk5ED1N7rqhz/I/wXscf4qaecMuQ14XVW/dPbdD6q6MeAp3lDVb5199hHQxVl+PTBNVac5630JLAMGBeyTPzmvMR3/FNOJKHM7J/h8xgUW+uZEtAJ+5/yJnyUiWUAL4GQAEbkxYOonCzidgCkF4PtSnnNv8Q1VzXVuJpex/bLGngxkBCwra1uB9qtqXvEdEakpIi+JyA4ROQykA/VEJLaM9VsBPUrsi+vw/wXxC6q6S1VHqOqpzro5+I/uwb8Ptx6n1r0Bt3P53/5pBQwtUUNv/L8MTwYyVTUnYN0dx9nG8RxvO6aaiJg30Uy18j3wpKo+WfIBEWkFvAJcACxU1UIRWQUETtVUVWvXPUADEakZEPwtylmnZC2/A34F9FDVvSLSBVjJ/+ovOf574GtVvbCixarq9yIyAfgg4LlOrejzOOu9o6q3l3zA+X7UF5FaAcHfkv+9jhygZsD4WKCs6Zoyt2OqDzvSN+WJF5GkgK84/KF+p4j0EL9aInKRiNQGauEPlP0AInIL/iP9KqeqO/BPNzwmIgkich5wSQWfpjb+efwsEWkA/LnE4/uAUwLuTwHai8gNIhLvfJ0tIqeVfGLnTeK/iEhbEYlx3oi9FVjkDHkNuEVELnAebyYiHYKo+V3gEhEZICKxzvepr4g0D9gnf3H2SW9+vk++BZKc71888Ecg8ZebOP52gqjRRAgLfVOeafhDsPjrMVVdhn9e/zkgE/+bkzcDqOo3+OerF+IPyDOA+WGs9zrgPPxvmj4BfIj//YZgjQVqAAfwh/GMEo+PA4aI/8yeZ515//7A1cBu/FMwT1F6cPqA1sB/8J+muc6p7WYAVV2C/43aMcAh4Gv8UyrHparfA4OBP+D/Zfs9/jeDi/9/X4v/ze8M/L/E3g5Y9xBwN/Aq8AP+I/+fnc1Tge2YakDsIiommonIh8BGVS15xG6MJ9lvaBNVnKmVU53pkTT8R6afu12XMZHC3sg10eYk4FP85+nvAu5S1ZXulmRM5LDpHWOM8RCb3jHGGA+JuOmdRo0aaevWrU94/ZycHGrVqhW6gkLE6qoYq6tirK6Kica6li9ffkBVy2+JoaoR9dWtWzetjNmzZ1dq/apidVWM1VUxVlfFRGNdwDINImNtescYYzzEQt8YYzzEQt8YYzzEQt8YYzzEQt8YYzwkqNAXkTTxXwZui4g8VMrjqSKyQkQKRGRIicduEpHNztcJXbHHGGNMaJQb+k5/7QnAQKAjcI2IdCwxbCf+ToHvl1i3uDVtD/xXTvqziNSvfNnGGGNORDBH+ucAW1R1m6r6gIn4m1j9RFW3q+oa/JeeCzQA+FJVM1Q1E/+1RNOIMKrK5yt/YFdmbvmDjTGmGgvmE7nN+Pkl53bhP3IPRmnrNis5SESG478oNSkpKcyZMyfIp/+l7OzsCq+/N6eIh+YeJSkWrjstgd7N4ij7mtzhqyscrK6KsboqxuqqmHDUFUzol5Z+wXZpC2pdVX0ZeBmge/fu2rdv3yCf/pfmzJlDRddfuj0D5i6kYe0avLbuKDsK6/G3355Bk9pJJ1xHKOoKB6urYqyuirG6KiYcdQUzvbOLn19ntDn+KwQFozLrhk1Gjg+AF67vyp8u7sjczQcYMCadaWv3uFyZMcaEVjChvxRoJyJtRCQB/2XhJgf5/DOB/s61Qevjv6zczBMrtepk5fpDv2FyIrf1bsPUkb1p2aAmd7+3gvsmrvzpcWOMqe7KDX1VLQBG4A/rDcBHqrpeRB4XkUvhp6sV7QKGAi+JyHpn3Qzgr/h/cSwFHneWRZSMnHwA6teMB6Btk9pMuqsn91/Ynqlr9jBgbDpzNv3oZonGGBMSQbVWVtVp+C+QHbjs0YDbS/FP3ZS27uvA65Woscpl5vpIjIuhRnzsT8viYmMYeUE7ft2hCfd/tIqb31jKtT1a8sig06iVGHEdqY0xJij2iVwgM8dH/ZoJpZ6xc3qzukwe0Zs7Uk/hgyU7SRuXzpLvIu6PFWOMCYqFPv4j/fq1Esp8PCk+locHncZHd5yHIFz18kKenPoNefmFYazSGGMqz0IfyMzN/2k+/3jObt2A6ff14boeLXll7ndcMn4ea3cdCkOFxhgTGhb6ONM7xznSD1QrMY4nLjuDt249hyN5BVz+/HzG/udb8gtLfhjZGGMij4U+zvROEEf6gc5v35iZo1K5pPPJjP3PZn77/AI27ztSRRUaY0xoeD70C4uUrKP5NKgZ3JF+oLo14xlzVRdevL4rP2Qd5aLx83h17jaKioL9wLIxxoSX50P/8NF8VKHeCYR+sbTTmzJzVCrnt2/ME1M3cPUri9h50Jq3GWMij+dDP8P5tG2DIOf0y9K4diIv39CNfw7tzIbdh0kbl877i3fiv0i9McZEBs+HfnGLhXoVnNMvjYgwpFtzZoxO5ayW9fjDZ2u55c2l7DucV+nnNsaYUPB86Be3YKjskX6gZvVq8M6tPfjLpZ1YtO0g/ceks2hPQcie3xhjTpTnQz/TOdKvX4k5/dLExAg39WzNtJF9OKVxLV5cfYx73l/xU0dPY4xxg4W+E8LBnqdfUac0TubjO85jSLt4Zq3fS/8x6Xy1YV+VbMsYY8rj+dDPyPWREBtDrYTY8gefoLjYGC4+NYHJI3rTKDmB295axv99spojeflVtk1jjCmN50M/KyefejXjQ355xNKc1rQOk0f05p5+p/LJ8l2kjZ3Lgq0Hqny7xhhTzPOhn5HrC+mbuOVJiIvh9wM68MldPUmIi+HaVxbzl3+vt+Ztxpiw8HzoZ+X6QnK6ZkV1bVmfaSP7cHPP1rwxfzuDnp3Lqu+zwl6HMcZbPB/6GTnhPdIPVCMhlscu7cR7w3qQ5yvkihcW8K9Zm/AVWPM2Y0zV8HzoZ+XmV6oFQyj0atuIGaNTufysZoz/7xYumzCfjXsPu1qTMSY6eTr0i4qUzFzfCTVbC7U6SfH8c2hnXrmxOz8eyePS8fN5Yc5WCq15mzEmhDwd+kfyCijS0LRgCJULO6Ywc1QqF5zWhKdmbOTKlxay/UCO22UZY6KEp0M/VM3WQq1hciLPX9eVsVd1YfO+IwwcN5d3Fm635m3GmErzdOhXVQuGUBARLjurGbNGn8/ZbRrwpy/Wc+PrS9hz6KjbpRljqjFvh34Vt2AIhZPqJvHWLWfz5OWns3xHJv3HpPPpil121G+MOSHeDv1cp8NmBB7pBxIRruvRiun39aHDSbW5/6PV3Pnucg5kH3O7NGNMNePt0HeO9OvVipw3co+nVcNaTBx+Hn8Y1IHZG/czYEw6M9btdbssY0w14unQz8j1ERcj1E6Mc7uUoMXGCMNTT2XKyN40rZfEne8u5/6PVnHoqDVvM8aUz9Oh72/BkBCWZmuh1j6lNp/d3YuRF7Tji1W7SRubztzN+90uyxgT4Twd+v4WDNVjaqc08bEx3H9hez69qyc1E2K54bUl/OnzdeT67CpdxpjSeTr0MyOgBUModG5Rj6kj+3Bb7za8u3gHg8bNZfmODLfLMsZEIG+Hfk5ktGAIhaT4WP50cUc+uP1cCoqUoS8u5P9N38ixAmvZbIz5H2+Hfm4+9avx9E5pzj2lITNGpXJl9xa8+PVWBj83n/W7D7ldljEmQgQV+iKSJiKbRGSLiDxUyuOJIvKh8/hiEWntLI8XkbdEZK2IbBCRh0Nb/olTVbJyfRH5adzKSk6M4/9dcSav39ydgzk+Lpswn+f+u5mCQmvZbIzXlRv6IhILTAAGAh2Ba0SkY4lhtwGZqtoWGAM85SwfCiSq6hlAN+CO4l8IbjtyrICCIo3K0C/26w4pzBqVStrpTfnnrG+54sWFbN2f7XZZxhgXBXOkfw6wRVW3qaoPmAgMLjFmMPCWc/sT4ALxnwepQC0RiQNqAD4gIhrFV4cWDKFQv1YC4685i+euPYsdB3MYNG4ub8z/jiJr2WyMJ0l5PVxEZAiQpqrDnPs3AD1UdUTAmHXOmF3O/a1AD+AQ8A5wAVATGK2qL5eyjeHAcICUlJRuEydOPOEXlJ2dTXJycrnjtmUV8viiPEZ1TaRLk6r/cFawdVWlrLwi3ljvY/X+Qk5rEMNtZySSVJjrel2liYT9VRqrq2KsroqpTF39+vVbrqrdyx2oqsf9wj9F82rA/RuA8SXGrAeaB9zfCjQEegHvAfFAE2ATcMrxttetWzetjNmzZwc17r8b9mmrB6fo8h0ZldpesIKtq6oVFRXpxCU7tOOfpmunR2foX96ZpUVFRW6X9QuRsr9KsroqxuqqmMrUBSzTcvJcVYOa3tkFtAi43xzYXdYYZyqnLpABXAvMUNV8Vf0RmA+U/5soDIrbKkfLKZvBEhGuOrslM0al0unkOry+zsewt5bx45E8t0szxoRBMKG/FGgnIm1EJAG4GphcYsxk4Cbn9hDgv85vnp3Ar8WvFnAusDE0pVdORk7k9tIPhxYNavLB7edyTYcE5m05wIAx6Uxds8ftsowxVazc0FfVAmAEMBPYAHykqutF5HERudQZ9hrQUES2APcDxad1TgCSgXX4f3m8oaprQvwaTkhmro/YGKF2UvVpthZqMTHCgNbxTB3Zh5YNanLP+ysY+cFKspy/gowx0SeoxFPVacC0EsseDbidh3/uv+R62aUtjwSZufnUqxFPTEz1a7YWam2bJDPprp68MGcr477azKJtB3lqyJn0+1UTt0szxoSYZz+Rm5nji/rTNSsiLjaGey9ox+f39KJezXhueWMpD3+6luxj1rzNmGji3dDP9VG/ZnS1YAiF05vV5d/39uaO809h4tKdDByXzpLvrHmbMdHCu6Gfk+/ZN3HLkxgXy8MDT+PjO84jRoSrXl7Ik1O/IS/fmrcZU915N/SjtO9OKHVv3YBpI/twXY+WvDL3Oy4eP481u7LcLssYUwmeDH1V9Ye+zemXq1ZiHE9cdgZv33oO2XkFXP78AsZ8+S351rzNmGrJk6Gf4yskv1BtTr8CUts3ZuaoVC7tfDLjvtrMb59fwOZ9R9wuyxhTQZ4Mfa80Wwu1ujXjGXNVF168vis/ZB3lovHzeCV9G4XWvM2YasOboe/RFgyhknZ6U2aOSuX89o15ctoGrnl5ETsP5rpdljEmCJ4M/Z9aMETZVbPCqXHtRF6+oRv/HNqZDXsOkzYunfcX7yxuuGeMiVCeDP2s3HzAu313QkVEGNKtOTNGp3JWy3r84bO13PLmUvYdtuZtxkQqT4a+15uthVqzejV459YePD64E4u2HaT/mHS+WPWDHfUbE4E8GfqZuT5iBOrUsOmdUImJEW48rzXTRvbhlMa1uG/iKka8v/KnX7DGmMjg2dCvWyOeWGu2FnKnNE7mkzt78n9pv2LWN3vpPyad/3yzz+2yjDEOb4Z+Tr6drlmFYmOEu/u2ZfKI3jRKTmDY28v4/cerOZKX73ZpxnieN0PfWjCExWlN6zB5RG/u6Xcqk1bsIm3sXBZsPeB2WcZ4midDPyPHQj9cEuJi+P2ADnxyV08S42K49pXFPDZ5PUd91rzNGDd4MvSzcvOtBUOYdW1Zn6kj+3Bzz9a8uWA7Fz07l5U7M90uyxjP8VzoqyoZuT4a2Jx+2NVIiOWxSzvx3rAe5OUXcsULC/jnzE34Cqx5mzHh4rnQP5pfiK+giHo2veOaXm0bMWN0Kld0bc5zs7dw2YT5bNx72O2yjPEEz4V+8XnjDawFg6vqJMXzj6GdeeXG7vx4JI9Lxs/jhTlbrXmbMVXMc6FvLRgiy4UdU5g1+nwu7JjCUzM2cuVLC9l+IMftsoyJWp4L/QxrqxxxGtRKYMK1XRl3dRc27zvCwHFz+c+OfIrsqN+YkPNc6Be3VbYj/cgiIgzu0oxZo8/n7DYNeHeDjxtfX8LurKNul2ZMVPFe6P/UbM3m9CPRSXWTeOuWs7mpYwIrdmYyYGw6k5bvsuZtxoSI50I/IzcfEahrzdYilojQr2U80+/rQ4eTavO7j1dzxzvLOZB9zO3SjKn2PBf6Wbk+6iTFExfruZde7bRqWIuJw8/jD4M6MGfTfgaMSWfGur1ul2VMtea55MvIsQ9mVSexMcLw1FOZMrI3Teslcee7y7n/w1UcOmrN24w5EZ4L/azcfOrZfH610z6lNp/d3YuRF7Tji9W7SRubztzN+90uy5hqx3Ohn5HjswuiV1PxsTHcf2F7Pr2rJzUTYrnhtSX86fN15PoK3C7NmGrDc6GfleuzFgzVXOcW9Zg6sg/Derfh3cU7GDRuLst3ZLhdljHVQlChLyJpIrJJRLaIyEOlPJ4oIh86jy8WkdYBj50pIgtFZL2IrBWRpNCVX3H+Zms2vVPdJcXH8seLO/LB7edSUKQMfXEhf5++gWMF1rLZmOMpN/RFJBaYAAwEOgLXiEjHEsNuAzJVtS0wBnjKWTcOeBe4U1U7AX0B196BO+orJC/fmq1Fk3NPaciMUalcdXYLXvp6G5eOn8/63YfcLsuYiBXMkf45wBZV3aaqPmAiMLjEmMHAW87tT4ALRESA/sAaVV0NoKoHVdW1Q7HiT+Pa2TvRJTkxjr//9kzeuPlsMnN9DH5uPuO/2kxBobVsNqYkKe+TjiIyBEhT1WHO/RuAHqo6ImDMOmfMLuf+VqAHcD3QDWgCNAYmqurTpWxjODAcICUlpdvEiRNP+AVlZ2eTnJxc6mM7Dhfy5wV53HtWIt1S4k54G6Guy03RVle2T3nnm2Ms3lvIKXVjuP2MRJomh+6tq2jbX1XN6qqYytTVr1+/5aravdyBqnrcL2Ao8GrA/RuA8SXGrAeaB9zfCjQEHgC+AxoBNYGFwAXH2163bt20MmbPnl3mY3O/3a+tHpyii7YeqNQ2TsTx6nJTtNb179U/aOe/zNT2j0zT1+Zu08LCooioq6pYXRUTjXUBy7ScPFfVoKZ3dgEtAu43B3aXNcaZx68LZDjLv1bVA6qaC0wDugaxzSph0zvecfGZJzNrVCq92jbi8SnfcN2ri9mVmet2Wca4LpjQXwq0E5E2IpIAXA1MLjFmMnCTc3sI8F/nN89M4EwRqen8Mjgf+CY0pVdccejbG7ne0KROEq/d1J2nrziTNbuySBs7l4+Wfm/N24ynlRv6qloAjMAf4BuAj1R1vYg8LiKXOsNeAxqKyBbgfuAhZ91M4Bn8vzhWAStUdWroX0Zwinvp2ydyvUNEuPLsFswYlcrpzerwf5PWMOytZfx4JM/t0oxxRVDvZqrqNPxTM4HLHg24nYd/7r+0dd/Ff9qm67Jy86mdFEe8NVvznBYNavL+sHN5c8F2npqxkf5j0nnistO5+MyT3S7NmLDyVPpZszVvi4kRbu3dhqkj+9CqQU1GvL+Sez9YSZYz7WeMF3gq9DOtBYMB2jZJZtJdPfndhe2ZvnYP/cekM3vTj26XZUxYeC70G9h8vgHiYmO494J2fH5PL+rXTOCWN5by8KdryD5mzdtMdPNW6Ofk27Vxzc+c3qwuk+/txR3nn8LEpd8zcFw6i7cddLssY6qMt0I/10d9m9M3JSTGxfLwwNP4+I7ziBHh6lcW8cSUb8jLt+ZtJvp4JvTz8gvJ9RXaBdFNmbq3bsC0kX24rkdLXp33HRePn8eaXVlul2VMSHkm9LNy/c097UjfHE+txDieuOwM3r71HLLzCrj8+QWM+fJb8q15m4kSngn9n1ow2Jy+CUJq+8bMHJXK4M4nM+6rzfz2+QVs3nfE7bKMqTTvhH6OtWAwFVO3ZjzPXNWFF6/vyg9ZR7lo/Dymf5dPYZG1cTDVl3dC35nesQ9nmYpKO70pM0elcn77xny4ycc1Ly9i50Fr3maqJ8+EfoYzvWNv5JoT0bh2Ii/f0I3bz0hgw57DpI1L573FO6x5m6l2PBP6Nr1jKktE6NUsnpmjU+nasj6PfLaOm99Yyt5D1rzNVB/eCf1cH8mJcSTEeeYlmypycr0avH3rOTw+uBOLvztI/zFf88WqH+yo31QLnknAzBwf9WvZ1I4JjZgY4cbzWjP9vlTaNknmvomruOf9FT+17zYmUnkn9HOtBYMJvTaNavHxnT35v7Rf8eU3++g/Jp3/fLPP7bKMKZOHQt9noW+qRGyMcHfftkwe0ZvGtRMZ9vYyfv/xao7k5btdmjG/4LHQt+kdU3VOa1qHL+7pxYh+bZm0YhdpY+eyYMsBt8sy5me8E/o5+daCwVS5hLgYHhjwKybd1ZPEuBiufXUxj01ez1GfNW8zkcEToe8rKCL7WIG1YDBhc1bL+kwd2Yebe7bmzQXbuejZuazcmel2WcZ4I/SLL4dXz470TRjVSIjlsUs78f6wHhwrKOKKFxbwj5kb8RVY8zbjHk+E/k8tGOxI37igZ9tGTB/Vhyu6NmfC7K0MnjCfDXsOu12W8ShPhH7xudP2Rq5xS52keP4xtDOv3Nid/UeOcelz83h+zhZr3mbCzhOhXzy9Y2/kGrdd2DGFWaNTubBjCk/P2MTQFxfw3YEct8syHuKJ0P9fszULfeO+BrUSmHBtV8Zd3YUtP2YzaNxc3l64nSI76jdh4InQ/1+zNZveMZFBRBjcpRmzRp/POW0a8OgX67nx9SXszjrqdmkmynkj9HPzqZkQS1J8rNulGPMzJ9VN4s1bzubJy09nxc5MBoxNZ9LyXda8zVQZb4R+jrVgMJFLRLiuRyum39eHDifV5ncfr+aOd5ZzIPuY26WZKOSN0M+1Dpsm8rVqWIuJw8/jkUGnMefb/QwYk86MdXvdLstEGU+EfoZ12DTVRGyMcHvqKUy5tzdN6yVx57vLGf3hKg4dteZtJjQ8EfpZ1mHTVDPtU2rz2d29uO+CdkxevZsBY9JJ/3a/22WZKOCJ0M/I8dkF0U21Ex8bw+gL2/PZ3T1JTorjxteX8MfP15LrK3C7NFONBRX6IpImIptEZIuIPFTK44ki8qHz+GIRaV3i8ZYiki0iD4Sm7ODlFxZxJK/AjvRNtXVm83pMubc3w3q34b3FOxk4bi7Ltme4XZappsoNfRGJBSYAA4GOwDUi0rHEsNuATFVtC4wBnirx+BhgeuXLrbgsp++OvZFrqrOk+Fj+eHFHPrj9XAqLlKEvLeTv0zdwrMBaNpuKCeZI/xxgi6puU1UfMBEYXGLMYOAt5/YnwAUiIgAichmwDVgfmpIrJss+jWuiyLmnNGTGqFSuPrslL329jUvHz2fdD4fcLstUI1Leh0BEZAiQpqrDnPs3AD1UdUTAmHXOmF3O/a1AD+Ao8B/gQuABIFtV/1nKNoYDwwFSUlK6TZw48YRfUHZ2NsnJyT/d35RRyN+X5PH77kl0auTeh7NK1hUprK6KiaS6Vu8v4I11Po74lIEtlMs71CI2Rtwu62ciaX8Fisa6+vXrt1xVu5c3Li6I5yrtp6jkb4qyxvwFGKOq2c6Bf6lU9WXgZYDu3btr3759gyirdHPmzCFw/bx1e2HJcvr27E6nk+ue8PNWVsm6IoXVVTGRVFdf4KaLfDz6xXomr97N90Xx/OvKLrRtEjlhFkn7K5CX6wpmemcX0CLgfnNgd1ljRCQOqAtk4D/af1pEtgOjgD+IyAjCKNOmd0wUq1czgWevOYu7uySyMyOXi56dy+vzvrPmbaZMwYT+UqCdiLQRkQTgamByiTGTgZuc20OA/6pfH1VtraqtgbHA31T1uRDVHpT/9dK30DfR65yT4pg5OpXebRvx+JRvuPbVRXyfket2WSYClRv6qloAjABmAhuAj1R1vYg8LiKXOsNeAxqKyBbgfuAXp3W6JSvXR1J8DDUSrNmaiW5Naifx6k3defqKM1n3w2EGjpvLh0t3WvM28zPBzOmjqtOAaSWWPRpwOw8YWs5zPHYC9VVaRk6+XSbReIaIcOXZLTjv1Ib8/pPVPDhpLbPW7+Pvvz2DJnWS3C7PRICo/0RuVq6Pehb6xmNaNKjJ+8PO5dGLOzJvywH6j01nypqSb8UO3cm6AAAUjklEQVQZL4r60M/ItRYMxptiYoRbe7dh6sg+tGpYixHvr+TeD1b+9NkV401RH/pZufl2xSzjaW2bJDPpzvN4oH97pq/dQ/8x6cze+KPbZRmXRH3oW7M1YyAuNoYRv27H5/f0on7NBG55cykPf7qG7GPWvM1rojr0CwqLOJxnvfSNKXZ6s7pMvrcXd55/Kh8u/Z60seks2nbQ7bJMGEV16B86mo8q1LfpHWN+khgXy0MDO/DRHecRGyNc88oi/jrlG/LyrXmbF0R16Gf+1GHTjvSNKal76wZMG9mH63u04rV533Hx+Hms2ZXldlmmikV56NuncY05nlqJcfz1stN5+9ZzyM4r4PLnF/DMl9+SX1jkdmmmikR36DstGOyNXGOOL7V9Y2aOTmVw55N59qvNXP78fL7dd8TtskwViO7Qd4707ZRNY8pXt0Y8z1zVhRev78qerDwuHj+Pl9O3UmjN26JKVId+Ro5/Tt+O9I0JXtrpTZk5OpW+7Rvzt2kbufrlhew8aM3bokVUh35Wro+EuBhqxFuzNWMqolFyIi/d0I1/De3Mxj1HSBuXznuLd1jztigQ1aGfkeOjQc0EjncBF2NM6USEK7o1Z+boVLq2rM8jn63jpjeWsvdQntulmUqI6tDPtBYMxlTayfVq8Pat5/DXwZ1Y+l0G/cd8zecrf7Cj/moqykPfWjAYEwoxMcIN57Vm2n19aNskmVEfruLu91ZwMPuY26WZCor60Ldz9I0JnTaNavHxnT15MK0DX234kQFj0/nym31ul2UqILpDP8dH/Vo2vWNMKMXGCHf1PZXJ9/aice0kbn97GQ98vJrDeflul2aCELWhX1ikHDpqV80ypqp0OKkOX9zTixH92vLpil0MHDuXBVsOuF2WKUfUhv7ho/kUKXbVLGOqUEJcDA8M+BWT7upJYlwM1766mMcmr+eoz5q3RaqoDf3iT+PaG7nGVL2zWtZn6sg+3NyzNW8u2M5Fz85lxc5Mt8sypYj60LdTNo0JjxoJsTx2aSfeH9aDYwVFDHlhAZ9868NXYM3bIkn0hr61YDDGFT3bNmL6qD5c0bU5U7blM3jCfDbsOex2WcYRtaGfYW2VjXFNnaR4/jG0M/d1TWT/kWNc+tw8np+zhQJr2ey6qA394rbKdgEVY9xzVpM4Zo1O5cKOKTw9YxNDX1rIdwdy3C7L06I39HPziY8VaiVYszVj3NSgVgITru3KuKu7sG1/DgPHpfPWgu0UWctmV0Rv6Of4P41rzdaMcZ+IMLhLM2aNTqVHm4b8efJ6bnx9CbuzjrpdmudEb+hbCwZjIk5KnSTevOVs/nb5GazYmcmAMel8snyXNW8Lo+gOfWvBYEzEERGu7dGSGfelclrTOjzw8WqGv7OcA9a8LSyiOPTz7XRNYyJYy4Y1+WD4uTwy6DS+/nY//cekM2PdHrfLinrRG/o5PmvBYEyEi40Rbk89hSn39ubkeknc+e4KRn+4ikO51rytqgQV+iKSJiKbRGSLiDxUyuOJIvKh8/hiEWntLL9QRJaLyFrn31+HtvzSFRUpWdZszZhqo31KbT67uxf3XdCOyat3M2BsOunf7ne7rKhUbuiLSCwwARgIdASuEZGOJYbdBmSqaltgDPCUs/wAcImqngHcBLwTqsKP50heAYVFai0YjKlG4mNjGH1hez67uyfJSXHc+PoS/vj5WnKOFbhdWlQJ5kj/HGCLqm5TVR8wERhcYsxg4C3n9ifABSIiqrpSVXc7y9cDSSKSGIrCj8earRlTfZ3ZvB5T7u3NsN5teG/xTgY9O5el2zPcLitqSHmnSonIECBNVYc5928AeqjqiIAx65wxu5z7W50xB0o8z52q+ptStjEcGA6QkpLSbeLEiSf8grKzs9lbUIMnFuUxulsinRvHnfBzhVJ2djbJyclul/ELVlfFWF0VU9m6NmUU8uraYxw4qqS1iefytvEkxFb+szfRuL/69eu3XFW7lztQVY/7BQwFXg24fwMwvsSY9UDzgPtbgYYB9zs5y04tb3vdunXTypg9e7Z+tWGvtnpwiq7cmVmp5wql2bNnu11CqayuirG6KiYUdR3Jy9eHJq3RVg9O0QufmaNrd2VFRF1VoTJ1Acu0nHxV1aCmd3YBLQLuNwd2lzVGROKAukCGc7858Blwo6puDWJ7lZbhdNisb3P6xlR7yYlx/P23Z/DGLWeTlZvPZRPm8+xXm6152wkKJvSXAu1EpI2IJABXA5NLjJmM/41agCHAf1VVRaQeMBV4WFXnh6ro8lizNWOiT79fNWHW6FQGndGUZ778liteWMCWH7PdLqvaKTf0VbUAGAHMBDYAH6nqehF5XEQudYa9BjQUkS3A/UDxaZ0jgLbAn0RklfPVJOSvooTMXB9xMULtxMiYzzfGhEa9mgk8e81ZTLi2Kzszcrno2bm8Nu87a95WAUGloqpOA6aVWPZowO08/HP/Jdd7AniikjVWWGau/4NZ1mzNmOh00ZlNObtNfR6etJa/TvmGWev38s+hnWnRoKbbpUW8qPxEbmZOvs3nGxPlmtRO4tWbuvP0FWeyfvdh0sam8+HSnda8rRxRGfoZuT6bzzfGA0SEK89uwYxRfTijeV0enLSW295axo+H89wuLWJFZehn5fqsBYMxHtK8fk3eH3Yuf76kI/O3HKD/2HT+vbrkSYYGojT0M3Lyra2yMR4TEyPc0qsNU0f2oVXDWtz7wUpGvL/ip7P5jF/Uhb6qkmUXUDHGs9o2SWbSnefxQP/2zFi3l/5j05m98Ue3y4oYURf6RwugoEgt9I3xsLjYGEb8uh1fjOhFg5oJ3PLmUh6atIZsa94WfaGfne9/597eyDXGdDq5LpPv7cWd55/KR8u+J21sOou2HXS7LFdFX+j7nNC3UzaNMUBiXCwPDezAx3eeR2yMcM0ri/hgwzHy8gvdLs0V0Rf6dqRvjClFt1YNmH5fH67v0YqZOwq46Nm5rNmV5XZZYRd1oX/kpyN9C31jzM/VTIjjr5edzgPdk8g5Vsjlzy/gmS+/Jd9DzduiLvSznUtr2nn6xpiynN4olpmjUxnc+WSe/Wozl02Yz6a9R9wuKyyiL/R9SoxA7SRrtmaMKVvdGvE8c1UXXry+G3sP5XHJ+Hm89PVWCqO8eVvUhf6RfP/pmjEx1mzNGFO+tNNPYuboVPp1aMzfp2/k6pcXsuNgjttlVZmoC/1sn10Q3RhTMY2SE3nx+m48c2VnNu49wsBxc3l30Y6obN4WfaGfr3ZBdGNMhYkIv+3anJmjUunWqj5//HwdN72xlL2Hoqt5W/SFvs8+jWuMOXEn16vB27eew18Hd2Lpdxn0H/M1n6/8IWqO+qMv9PPtdE1jTOWICDec15pp9/WhXUptRn24irvfW8HB7GNul1ZpURX6quo/0rfpHWNMCLRpVIuP7jiPB9M68NWGHxkwNp0vv9nndlmVElWhn+MrpECtBYMxJnRiY4S7+p7K5Ht70bh2Ere/vYwHPl7N4bx8t0s7IVEV+sV9s+1I3xgTah1OqsMX9/RiRL+2fLpiF2lj0pm/5YDbZVVYdIV+rhP6NqdvjKkCCXExPDDgV0y6qydJCbFc9+pi/vzFOo76qk/ztigLff+fWw3sqlnGmCp0Vsv6TL23D7f0as1bC3cw6Nm5rNiZ6XZZQYmu0Hemd+rZkb4xporVSIjlz5d04v3be+ArKGLICwt4esZGjhVE9lF/VIV+hhP61mzNGBMuPU9txIxRfRjSrTnPz9nK4Ofms2HPYbfLKlNUhX5Wrg8B6tSw6R1jTPjUTorn6SGdefXG7hzI9nHpc/OYMHsLBRHYsjmqQj8j10eteP8pVsYYE26/6ZjCrNGp9O94Ev+YuYmhLy1k2/5st8v6magK/czcfJLjLfCNMe5pUCuB5649i3FXd2Hb/hwGPTuXtxZspyhCWjZHV+jn+EhOsNA3xrhLRBjcpRmzRqfSo01D/jx5PTe8vpgfso66XVqUhX5uPrUt9I0xESKlThJv3nI2f7v8DFbuzCJtTDofL/ve1eZt0RX6OT6b3jHGRBQR4doeLZlxXyqnNa3D7z9Zw+1vL2f/EXeat0VN6Ksqmbk2vWOMiUwtG9bkg+Hn8sig00jfvJ8BY9OZvnZP2OsIKvRFJE1ENonIFhF5qJTHE0XkQ+fxxSLSOuCxh53lm0RkQOhK/7mj+YUcKygi2c7WNMZEqNgY4fbUU5h6b2+a1avBXe+tYNTElRzKDV/ztnJDX0RigQnAQKAjcI2IdCwx7DYgU1XbAmOAp5x1OwJXA52ANOB55/lCrrgFgx3pG2MiXbuU2nx6d09G/aYd/16zhwFj0/n62/1h2XYwR/rnAFtUdZuq+oCJwOASYwYDbzm3PwEuEBFxlk9U1WOq+h2wxXm+kCtuwWBz+saY6iA+NoZRv2nP53f3onZSHDe9voQPNlb9PL+U9y6yiAwB0lR1mHP/BqCHqo4IGLPOGbPLub8V6AE8BixS1Xed5a8B01X1kxLbGA4MB0hJSek2ceLECr+QzLwi5u8u4PQ6Plo3Sq7w+lUtOzub5GSrK1hWV8VYXRUTaXX5CpVPN/uoG5vPwHYnVle/fv2Wq2r38sbFBfFcpR06l/xNUdaYYNZFVV8GXgbo3r279u3bN4iyfulyYM6cOZzo+lXJ6qoYq6tirK6KicS6+l8QnrqCmd7ZBbQIuN8c2F3WGBGJA+oCGUGua4wxJkyCCf2lQDsRaSMiCfjfmJ1cYsxk4Cbn9hDgv+qfN5oMXO2c3dMGaAcsCU3pxhhjKqrc6R1VLRCREcBMIBZ4XVXXi8jjwDJVnQy8BrwjIlvwH+Ff7ay7XkQ+Ar4BCoB7VDWym00bY0wUC2ZOH1WdBkwrsezRgNt5wNAy1n0SeLISNRpjjAmRqPlErjHGmPJZ6BtjjIdY6BtjjIdY6BtjjIeU+4nccBOR/cCOSjxFI+BAiMoJJaurYqyuirG6KiYa62qlqo3LGxRxoV9ZIrIsmI8ih5vVVTFWV8VYXRXj5bpsescYYzzEQt8YYzwkGkP/ZbcLKIPVVTFWV8VYXRXj2bqibk7fGGNM2aLxSN8YY0wZLPSNMcZDoib0y7t4exjraCEis0Vkg4isF5H7nOWPicgPIrLK+RrkQm3bRWSts/1lzrIGIvKliGx2/q0f5pp+FbBPVonIYREZ5cb+EpHXReRH50pwxctK3T/i96zz87ZGRLqGua5/iMhGZ9ufiUg9Z3lrETkasN9erKq6jlNbmd87EXnY2WebRGRAmOv6MKCm7SKyylkeln12nGwI78+Yqlb7L/wtn7cCpwAJwGqgo0u1NAW6OrdrA9/iv6D8Y8ADLu+n7UCjEsueBh5ybj8EPOXy93Ev0MqN/QWkAl2BdeXtH2AQMB3/1eHOBRaHua7+QJxz+6mAuloHjnNpn5X6vXP+H6wGEoE2zv/Z2HDVVeLxfwGPhnOfHScbwvozFi1H+sFcvD0sVHWPqq5wbh8BNgDN3KglSIEXtX8LuMzFWi4AtqpqZT6RfcJUNR3/9SAClbV/BgNvq98ioJ6INA1XXao6S1ULnLuL8F+VLuzK2GdlGQxMVNVjqvodsAX//92w1iUiAlwJfFAV2z5OTWVlQ1h/xqIl9JsB3wfc30UEBK2ItAbOAhY7i0Y4f6a9Hu5pFIcCs0RkufgvRg+Qoqp7wP9DCTRxoa5iV/Pz/4hu7y8oe/9E0s/crfiPCIu1EZGVIvK1iPRxqabSvneRss/6APtUdXPAsrDusxLZENafsWgJ/aAuwB5OIpIMTAJGqeph4AXgVKALsAf/n5fh1ktVuwIDgXtEJNWFGkol/ktxXgp87CyKhP11PBHxMycij+C/Kt17zqI9QEtVPQu4H3hfROqEuayyvncRsc+Aa/j5wUVY91kp2VDm0FKWVXp/RUvoR9QF2EUkHv839T1V/RRAVfepaqGqFgGvUEV/1h6Pqu52/v0R+MypYV/xn4zOvz+Guy7HQGCFqu5zanR9fznK2j+u/8yJyE3AxcB16kwCO1MnB53by/HPm7cPZ13H+d5Fwj6LA34LfFi8LJz7rLRsIMw/Y9ES+sFcvD0snPnC14ANqvpMwPLAubjLgXUl163iumqJSO3i2/jfCFzHzy9qfxPwRTjrCvCzoy+391eAsvbPZOBG5wyLc4FDxX+ih4OIpAEPApeqam7A8sYiEuvcPgVoB2wLV13Odsv63k0GrhaRRBFp49S2JJy1Ab8BNqrqruIF4dpnZWUD4f4Zq+p3rMP1hf+d7m/x/5Z+xMU6euP/E2wNsMr5GgS8A6x1lk8Gmoa5rlPwnzmxGlhfvI+AhsBXwGbn3wYu7LOawEGgbsCysO8v/L909gD5+I+ybitr/+D/03uC8/O2Fuge5rq24J/vLf4Ze9EZe4Xz/V0NrAAucWGflfm9Ax5x9tkmYGA463KWvwncWWJsWPbZcbIhrD9j1obBGGM8JFqmd4wxxgTBQt8YYzzEQt8YYzzEQt8YYzzEQt8YYzzEQt94kogUys+7e4asM6vTtdGtzxUYc1xxbhdgjEuOqmoXt4swJtzsSN+YAE6f9adEZInz1dZZ3kpEvnKaiH0lIi2d5Sni72e/2vnq6TxVrIi84vRNnyUiNVx7UcYEsNA3XlWjxPTOVQGPHVbVc4DngLHOsufwt7k9E39zs2ed5c8CX6tqZ/z929c7y9sBE1S1E5CF/1OfxrjOPpFrPElEslU1uZTl24Ffq+o2pznWXlVtKCIH8LcTyHeW71HVRiKyH2iuqscCnqM18KWqtnPuPwjEq+oTVf/KjDk+O9I35pe0jNtljSnNsYDbhdj7ZyZCWOgb80tXBfy70Lm9AH/3VoDrgHnO7a+AuwBEJNaF3vXGVIgdfRivqiHOhbEdM1S1+LTNRBFZjP+g6Bpn2UjgdRH5PbAfuMVZfh/wsojchv+I/i783R2NiUg2p29MAGdOv7uqHnC7FmOqgk3vGGOMh9iRvjHGeIgd6RtjjIdY6BtjjIdY6BtjjIdY6BtjjIdY6BtjjIf8f1zlRvW3afBzAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "lr_history = []\n",
    "for e in range(n_epoch):\n",
    "    for _ in range(len(dataloader_train)):\n",
    "        lr_history.append(scheduler.get_lr())\n",
    "        scheduler.step()\n",
    "        \n",
    "plt.plot(np.arange(n_epoch*len(dataloader_train)) / len(dataloader_train), lr_history)\n",
    "plt.grid(True)\n",
    "plt.title(\"Learning rate Schedule\")\n",
    "plt.xlabel(\"Epoch\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# RE-INITIALIZE WEIGHTS (which have been destroyed during the learning-rate finder step)\n",
    "model = fast_resnet(num_class=10).to(device)  \n",
    "optimizer, scheduler = create_optim_schedule(model, n_epoch, max_lr=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [1/200]  Loss: 2.31(2.31)  Acc: 10.0%(10.0%)  Training: 1.6sec (1.6sec)  lr/momentum: 0.0102/0.90\n",
      "Epoch: [2/200]  Loss: 2.34(2.34)  Acc: 10.6%(10.7%)  Training: 3.1sec (1.5sec)  lr/momentum: 0.0203/0.90\n",
      "Epoch: [3/200]  Loss: 2.15(2.15)  Acc: 22.8%(22.8%)  Training: 4.6sec (1.5sec)  lr/momentum: 0.0304/0.90\n",
      "Epoch: [4/200]  Loss: 1.72(1.71)  Acc: 37.6%(37.7%)  Training: 6.1sec (1.5sec)  lr/momentum: 0.0405/0.90\n",
      "Epoch: [5/200]  Loss: 1.78(1.78)  Acc: 36.2%(36.2%)  Training: 7.7sec (1.6sec)  lr/momentum: 0.0506/0.90\n",
      "Epoch: [6/200]  Loss: 1.99(1.99)  Acc: 32.9%(32.5%)  Training: 9.2sec (1.5sec)  lr/momentum: 0.0606/0.90\n",
      "Epoch: [7/200]  Loss: 2.94(2.93)  Acc: 27.9%(28.3%)  Training: 10.7sec (1.5sec)  lr/momentum: 0.0707/0.90\n",
      "Epoch: [8/200]  Loss: 2.51(2.54)  Acc: 27.1%(26.8%)  Training: 12.3sec (1.5sec)  lr/momentum: 0.0808/0.90\n",
      "Epoch: [9/200]  Loss: 4.25(4.28)  Acc: 17.9%(17.6%)  Training: 13.9sec (1.7sec)  lr/momentum: 0.0909/0.90\n",
      "Epoch: [10/200]  Loss: 2.13(2.13)  Acc: 33.9%(34.2%)  Training: 15.5sec (1.5sec)  lr/momentum: 0.0999/0.90\n",
      "Epoch: [11/200]  Loss: 3.26(3.28)  Acc: 25.2%(24.7%)  Training: 17.0sec (1.5sec)  lr/momentum: 0.0994/0.90\n",
      "Epoch: [12/200]  Loss: 1.36(1.38)  Acc: 51.4%(51.1%)  Training: 18.5sec (1.5sec)  lr/momentum: 0.0989/0.90\n",
      "Epoch: [13/200]  Loss: 1.32(1.32)  Acc: 52.6%(52.1%)  Training: 20.1sec (1.5sec)  lr/momentum: 0.0984/0.90\n",
      "Epoch: [14/200]  Loss: 1.30(1.31)  Acc: 52.7%(52.4%)  Training: 21.6sec (1.5sec)  lr/momentum: 0.0978/0.90\n",
      "Epoch: [15/200]  Loss: 1.82(1.84)  Acc: 40.8%(40.1%)  Training: 23.1sec (1.5sec)  lr/momentum: 0.0973/0.90\n",
      "Epoch: [16/200]  Loss: 1.09(1.10)  Acc: 61.3%(60.4%)  Training: 24.7sec (1.5sec)  lr/momentum: 0.0968/0.90\n",
      "Epoch: [17/200]  Loss: 1.55(1.56)  Acc: 51.5%(50.8%)  Training: 26.2sec (1.5sec)  lr/momentum: 0.0963/0.90\n",
      "Epoch: [18/200]  Loss: 1.46(1.46)  Acc: 52.1%(51.6%)  Training: 27.8sec (1.6sec)  lr/momentum: 0.0957/0.90\n",
      "Epoch: [19/200]  Loss: 1.35(1.37)  Acc: 54.3%(54.0%)  Training: 29.4sec (1.6sec)  lr/momentum: 0.0952/0.90\n",
      "Epoch: [20/200]  Loss: 1.10(1.12)  Acc: 61.6%(60.2%)  Training: 31.0sec (1.6sec)  lr/momentum: 0.0947/0.90\n",
      "Epoch: [21/200]  Loss: 1.08(1.10)  Acc: 62.2%(61.4%)  Training: 32.5sec (1.5sec)  lr/momentum: 0.0942/0.90\n",
      "Epoch: [22/200]  Loss: 1.84(1.86)  Acc: 45.7%(44.7%)  Training: 34.0sec (1.6sec)  lr/momentum: 0.0936/0.90\n",
      "Epoch: [23/200]  Loss: 1.37(1.42)  Acc: 57.5%(56.6%)  Training: 35.6sec (1.5sec)  lr/momentum: 0.0931/0.90\n",
      "Epoch: [24/200]  Loss: 2.12(2.18)  Acc: 42.8%(42.1%)  Training: 37.1sec (1.6sec)  lr/momentum: 0.0926/0.90\n",
      "Epoch: [25/200]  Loss: 1.11(1.14)  Acc: 62.5%(61.4%)  Training: 38.7sec (1.5sec)  lr/momentum: 0.0921/0.90\n",
      "Epoch: [26/200]  Loss: 1.43(1.47)  Acc: 55.3%(54.6%)  Training: 40.2sec (1.6sec)  lr/momentum: 0.0915/0.90\n",
      "Epoch: [27/200]  Loss: 1.46(1.48)  Acc: 56.0%(55.6%)  Training: 41.7sec (1.5sec)  lr/momentum: 0.0910/0.90\n",
      "Epoch: [28/200]  Loss: 1.20(1.25)  Acc: 62.2%(61.2%)  Training: 43.4sec (1.6sec)  lr/momentum: 0.0905/0.90\n",
      "Epoch: [29/200]  Loss: 2.81(2.89)  Acc: 35.8%(34.7%)  Training: 44.9sec (1.5sec)  lr/momentum: 0.0899/0.90\n",
      "Epoch: [30/200]  Loss: 0.97(1.01)  Acc: 67.8%(66.6%)  Training: 46.4sec (1.5sec)  lr/momentum: 0.0894/0.90\n",
      "Epoch: [31/200]  Loss: 0.91(0.95)  Acc: 69.7%(68.5%)  Training: 47.9sec (1.5sec)  lr/momentum: 0.0889/0.90\n",
      "Epoch: [32/200]  Loss: 0.88(0.92)  Acc: 70.5%(69.0%)  Training: 49.5sec (1.5sec)  lr/momentum: 0.0884/0.90\n",
      "Epoch: [33/200]  Loss: 1.12(1.17)  Acc: 64.0%(62.6%)  Training: 51.0sec (1.6sec)  lr/momentum: 0.0878/0.90\n",
      "Epoch: [34/200]  Loss: 1.03(1.09)  Acc: 66.3%(65.1%)  Training: 52.6sec (1.5sec)  lr/momentum: 0.0873/0.90\n",
      "Epoch: [35/200]  Loss: 0.95(1.00)  Acc: 69.6%(67.6%)  Training: 54.1sec (1.5sec)  lr/momentum: 0.0868/0.90\n",
      "Epoch: [36/200]  Loss: 1.12(1.18)  Acc: 64.7%(63.3%)  Training: 55.7sec (1.6sec)  lr/momentum: 0.0863/0.90\n",
      "Epoch: [37/200]  Loss: 0.85(0.91)  Acc: 72.7%(71.3%)  Training: 57.2sec (1.6sec)  lr/momentum: 0.0857/0.90\n",
      "Epoch: [38/200]  Loss: 0.89(0.94)  Acc: 70.6%(68.7%)  Training: 58.8sec (1.5sec)  lr/momentum: 0.0852/0.90\n",
      "Epoch: [39/200]  Loss: 0.83(0.88)  Acc: 72.8%(71.3%)  Training: 60.3sec (1.6sec)  lr/momentum: 0.0847/0.90\n",
      "Epoch: [40/200]  Loss: 0.88(0.95)  Acc: 72.3%(71.0%)  Training: 61.9sec (1.6sec)  lr/momentum: 0.0842/0.90\n",
      "Epoch: [41/200]  Loss: 0.97(1.05)  Acc: 70.9%(69.0%)  Training: 63.4sec (1.5sec)  lr/momentum: 0.0836/0.90\n",
      "Epoch: [42/200]  Loss: 0.91(0.97)  Acc: 71.1%(69.0%)  Training: 64.9sec (1.6sec)  lr/momentum: 0.0831/0.90\n",
      "Epoch: [43/200]  Loss: 1.12(1.19)  Acc: 66.7%(64.5%)  Training: 66.5sec (1.5sec)  lr/momentum: 0.0826/0.90\n",
      "Epoch: [44/200]  Loss: 0.94(0.99)  Acc: 70.6%(69.3%)  Training: 68.0sec (1.5sec)  lr/momentum: 0.0821/0.90\n",
      "Epoch: [45/200]  Loss: 0.82(0.88)  Acc: 73.4%(70.9%)  Training: 69.6sec (1.5sec)  lr/momentum: 0.0815/0.90\n",
      "Epoch: [46/200]  Loss: 1.01(1.10)  Acc: 70.2%(67.9%)  Training: 71.2sec (1.6sec)  lr/momentum: 0.0810/0.90\n",
      "Epoch: [47/200]  Loss: 0.98(1.06)  Acc: 70.8%(68.9%)  Training: 72.8sec (1.6sec)  lr/momentum: 0.0805/0.90\n",
      "Epoch: [48/200]  Loss: 1.08(1.19)  Acc: 68.6%(65.8%)  Training: 74.3sec (1.6sec)  lr/momentum: 0.0799/0.90\n",
      "Epoch: [49/200]  Loss: 0.98(1.05)  Acc: 71.6%(69.6%)  Training: 75.8sec (1.5sec)  lr/momentum: 0.0794/0.90\n",
      "Epoch: [50/200]  Loss: 0.76(0.82)  Acc: 76.0%(74.5%)  Training: 77.4sec (1.5sec)  lr/momentum: 0.0789/0.90\n",
      "Epoch: [51/200]  Loss: 0.91(1.00)  Acc: 73.1%(70.7%)  Training: 78.9sec (1.5sec)  lr/momentum: 0.0784/0.90\n",
      "Epoch: [52/200]  Loss: 0.89(0.98)  Acc: 73.1%(70.9%)  Training: 80.4sec (1.6sec)  lr/momentum: 0.0778/0.90\n",
      "Epoch: [53/200]  Loss: 0.82(0.92)  Acc: 75.4%(72.6%)  Training: 82.0sec (1.5sec)  lr/momentum: 0.0773/0.90\n",
      "Epoch: [54/200]  Loss: 0.96(1.03)  Acc: 72.2%(70.3%)  Training: 83.6sec (1.6sec)  lr/momentum: 0.0768/0.90\n",
      "Epoch: [55/200]  Loss: 0.91(1.02)  Acc: 73.2%(71.3%)  Training: 85.1sec (1.6sec)  lr/momentum: 0.0763/0.90\n",
      "Epoch: [56/200]  Loss: 0.82(0.91)  Acc: 75.4%(73.0%)  Training: 86.7sec (1.6sec)  lr/momentum: 0.0757/0.90\n",
      "Epoch: [57/200]  Loss: 1.25(1.37)  Acc: 66.0%(63.5%)  Training: 88.3sec (1.6sec)  lr/momentum: 0.0752/0.90\n",
      "Epoch: [58/200]  Loss: 0.82(0.91)  Acc: 75.8%(73.3%)  Training: 89.8sec (1.6sec)  lr/momentum: 0.0747/0.90\n",
      "Epoch: [59/200]  Loss: 0.90(0.99)  Acc: 74.8%(72.8%)  Training: 91.4sec (1.6sec)  lr/momentum: 0.0742/0.90\n",
      "Epoch: [60/200]  Loss: 0.72(0.80)  Acc: 77.6%(75.3%)  Training: 93.0sec (1.5sec)  lr/momentum: 0.0736/0.90\n",
      "Epoch: [61/200]  Loss: 0.80(0.91)  Acc: 76.3%(73.4%)  Training: 94.6sec (1.6sec)  lr/momentum: 0.0731/0.90\n",
      "Epoch: [62/200]  Loss: 0.79(0.89)  Acc: 76.1%(73.0%)  Training: 96.1sec (1.5sec)  lr/momentum: 0.0726/0.90\n",
      "Epoch: [63/200]  Loss: 0.91(1.00)  Acc: 73.6%(71.4%)  Training: 97.7sec (1.6sec)  lr/momentum: 0.0721/0.90\n",
      "Epoch: [64/200]  Loss: 0.88(0.98)  Acc: 74.3%(72.0%)  Training: 99.3sec (1.6sec)  lr/momentum: 0.0715/0.90\n",
      "Epoch: [65/200]  Loss: 0.73(0.81)  Acc: 78.1%(75.9%)  Training: 100.8sec (1.5sec)  lr/momentum: 0.0710/0.90\n",
      "Epoch: [66/200]  Loss: 1.25(1.37)  Acc: 68.6%(66.6%)  Training: 102.4sec (1.5sec)  lr/momentum: 0.0705/0.90\n",
      "Epoch: [67/200]  Loss: 0.75(0.81)  Acc: 77.5%(75.9%)  Training: 103.9sec (1.5sec)  lr/momentum: 0.0699/0.90\n",
      "Epoch: [68/200]  Loss: 0.86(0.96)  Acc: 75.3%(72.6%)  Training: 105.4sec (1.5sec)  lr/momentum: 0.0694/0.90\n",
      "Epoch: [69/200]  Loss: 0.74(0.83)  Acc: 77.9%(75.5%)  Training: 107.0sec (1.5sec)  lr/momentum: 0.0689/0.90\n",
      "Epoch: [70/200]  Loss: 0.99(1.09)  Acc: 73.6%(71.0%)  Training: 108.5sec (1.5sec)  lr/momentum: 0.0684/0.90\n",
      "Epoch: [71/200]  Loss: 0.99(1.12)  Acc: 74.2%(71.4%)  Training: 110.0sec (1.5sec)  lr/momentum: 0.0678/0.90\n",
      "Epoch: [72/200]  Loss: 0.96(1.06)  Acc: 74.3%(72.1%)  Training: 111.6sec (1.5sec)  lr/momentum: 0.0673/0.90\n",
      "Epoch: [73/200]  Loss: 0.80(0.88)  Acc: 77.6%(75.7%)  Training: 113.2sec (1.6sec)  lr/momentum: 0.0668/0.90\n",
      "Epoch: [74/200]  Loss: 0.81(0.91)  Acc: 77.2%(75.1%)  Training: 114.7sec (1.5sec)  lr/momentum: 0.0663/0.90\n",
      "Epoch: [75/200]  Loss: 0.83(0.93)  Acc: 77.0%(74.3%)  Training: 116.3sec (1.6sec)  lr/momentum: 0.0657/0.90\n",
      "Epoch: [76/200]  Loss: 0.97(1.08)  Acc: 74.6%(72.5%)  Training: 117.8sec (1.6sec)  lr/momentum: 0.0652/0.90\n",
      "Epoch: [77/200]  Loss: 0.97(1.07)  Acc: 73.7%(70.9%)  Training: 119.3sec (1.5sec)  lr/momentum: 0.0647/0.90\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [78/200]  Loss: 0.88(0.97)  Acc: 75.3%(72.8%)  Training: 120.9sec (1.6sec)  lr/momentum: 0.0642/0.90\n",
      "Epoch: [79/200]  Loss: 0.85(0.95)  Acc: 76.9%(74.1%)  Training: 122.5sec (1.6sec)  lr/momentum: 0.0636/0.90\n",
      "Epoch: [80/200]  Loss: 0.69(0.77)  Acc: 80.1%(77.6%)  Training: 124.1sec (1.6sec)  lr/momentum: 0.0631/0.90\n",
      "Epoch: [81/200]  Loss: 0.75(0.83)  Acc: 79.0%(76.6%)  Training: 125.6sec (1.6sec)  lr/momentum: 0.0626/0.90\n",
      "Epoch: [82/200]  Loss: 0.85(0.94)  Acc: 76.5%(74.2%)  Training: 127.2sec (1.6sec)  lr/momentum: 0.0621/0.90\n",
      "Epoch: [83/200]  Loss: 0.81(0.89)  Acc: 77.8%(74.9%)  Training: 128.8sec (1.6sec)  lr/momentum: 0.0615/0.90\n",
      "Epoch: [84/200]  Loss: 0.85(0.94)  Acc: 77.0%(74.3%)  Training: 130.3sec (1.5sec)  lr/momentum: 0.0610/0.90\n",
      "Epoch: [85/200]  Loss: 1.06(1.18)  Acc: 73.3%(70.7%)  Training: 131.9sec (1.5sec)  lr/momentum: 0.0605/0.90\n",
      "Epoch: [86/200]  Loss: 0.81(0.90)  Acc: 78.5%(76.1%)  Training: 133.4sec (1.5sec)  lr/momentum: 0.0599/0.90\n",
      "Epoch: [87/200]  Loss: 0.71(0.82)  Acc: 79.8%(77.0%)  Training: 134.9sec (1.5sec)  lr/momentum: 0.0594/0.90\n",
      "Epoch: [88/200]  Loss: 0.84(0.93)  Acc: 77.6%(75.2%)  Training: 136.5sec (1.6sec)  lr/momentum: 0.0589/0.90\n",
      "Epoch: [89/200]  Loss: 0.77(0.89)  Acc: 78.8%(75.3%)  Training: 138.0sec (1.5sec)  lr/momentum: 0.0584/0.90\n",
      "Epoch: [90/200]  Loss: 0.78(0.87)  Acc: 78.5%(75.6%)  Training: 139.6sec (1.5sec)  lr/momentum: 0.0578/0.90\n",
      "Epoch: [91/200]  Loss: 0.70(0.79)  Acc: 80.4%(78.0%)  Training: 141.1sec (1.5sec)  lr/momentum: 0.0573/0.90\n",
      "Epoch: [92/200]  Loss: 1.08(1.21)  Acc: 75.1%(72.3%)  Training: 142.6sec (1.5sec)  lr/momentum: 0.0568/0.90\n",
      "Epoch: [93/200]  Loss: 0.78(0.89)  Acc: 79.2%(76.3%)  Training: 144.2sec (1.5sec)  lr/momentum: 0.0563/0.90\n",
      "Epoch: [94/200]  Loss: 0.73(0.81)  Acc: 80.2%(77.8%)  Training: 145.7sec (1.5sec)  lr/momentum: 0.0557/0.90\n",
      "Epoch: [95/200]  Loss: 0.85(0.96)  Acc: 78.3%(75.5%)  Training: 147.3sec (1.6sec)  lr/momentum: 0.0552/0.90\n",
      "Epoch: [96/200]  Loss: 0.74(0.83)  Acc: 80.2%(77.6%)  Training: 148.8sec (1.6sec)  lr/momentum: 0.0547/0.90\n",
      "Epoch: [97/200]  Loss: 0.80(0.91)  Acc: 78.5%(75.8%)  Training: 150.4sec (1.6sec)  lr/momentum: 0.0542/0.90\n",
      "Epoch: [98/200]  Loss: 0.71(0.80)  Acc: 80.5%(77.9%)  Training: 151.9sec (1.5sec)  lr/momentum: 0.0536/0.90\n",
      "Epoch: [99/200]  Loss: 0.96(1.08)  Acc: 76.6%(73.7%)  Training: 153.5sec (1.6sec)  lr/momentum: 0.0531/0.90\n",
      "Epoch: [100/200]  Loss: 0.78(0.86)  Acc: 79.3%(76.6%)  Training: 155.1sec (1.6sec)  lr/momentum: 0.0526/0.90\n",
      "Epoch: [101/200]  Loss: 0.80(0.90)  Acc: 78.6%(75.7%)  Training: 156.7sec (1.6sec)  lr/momentum: 0.0521/0.90\n",
      "Epoch: [102/200]  Loss: 0.69(0.78)  Acc: 81.6%(79.3%)  Training: 158.3sec (1.6sec)  lr/momentum: 0.0515/0.90\n",
      "Epoch: [103/200]  Loss: 0.73(0.82)  Acc: 80.8%(78.1%)  Training: 159.9sec (1.5sec)  lr/momentum: 0.0510/0.90\n",
      "Epoch: [104/200]  Loss: 0.71(0.80)  Acc: 80.9%(78.5%)  Training: 161.5sec (1.6sec)  lr/momentum: 0.0505/0.90\n",
      "Epoch: [105/200]  Loss: 0.79(0.89)  Acc: 79.3%(76.8%)  Training: 163.0sec (1.6sec)  lr/momentum: 0.0499/0.90\n",
      "Epoch: [106/200]  Loss: 0.86(0.97)  Acc: 78.1%(75.2%)  Training: 164.6sec (1.6sec)  lr/momentum: 0.0494/0.90\n",
      "Epoch: [107/200]  Loss: 0.72(0.83)  Acc: 81.3%(78.7%)  Training: 166.2sec (1.6sec)  lr/momentum: 0.0489/0.90\n",
      "Epoch: [108/200]  Loss: 0.80(0.91)  Acc: 78.9%(76.2%)  Training: 167.8sec (1.6sec)  lr/momentum: 0.0484/0.90\n",
      "Epoch: [109/200]  Loss: 0.72(0.81)  Acc: 81.0%(79.0%)  Training: 169.4sec (1.6sec)  lr/momentum: 0.0478/0.90\n",
      "Epoch: [110/200]  Loss: 0.76(0.85)  Acc: 80.7%(77.8%)  Training: 170.9sec (1.5sec)  lr/momentum: 0.0473/0.90\n",
      "Epoch: [111/200]  Loss: 0.74(0.83)  Acc: 80.8%(78.3%)  Training: 172.5sec (1.5sec)  lr/momentum: 0.0468/0.90\n",
      "Epoch: [112/200]  Loss: 0.77(0.86)  Acc: 80.6%(77.8%)  Training: 174.0sec (1.5sec)  lr/momentum: 0.0463/0.90\n",
      "Epoch: [113/200]  Loss: 0.76(0.86)  Acc: 80.2%(77.5%)  Training: 175.5sec (1.5sec)  lr/momentum: 0.0457/0.90\n",
      "Epoch: [114/200]  Loss: 0.77(0.87)  Acc: 80.3%(77.4%)  Training: 177.1sec (1.6sec)  lr/momentum: 0.0452/0.90\n",
      "Epoch: [115/200]  Loss: 0.78(0.87)  Acc: 80.4%(78.3%)  Training: 178.7sec (1.6sec)  lr/momentum: 0.0447/0.90\n",
      "Epoch: [116/200]  Loss: 0.75(0.86)  Acc: 80.7%(77.9%)  Training: 180.3sec (1.6sec)  lr/momentum: 0.0442/0.90\n",
      "Epoch: [117/200]  Loss: 0.78(0.89)  Acc: 80.2%(77.5%)  Training: 181.9sec (1.6sec)  lr/momentum: 0.0436/0.90\n",
      "Epoch: [118/200]  Loss: 0.77(0.86)  Acc: 80.7%(78.0%)  Training: 183.4sec (1.5sec)  lr/momentum: 0.0431/0.90\n",
      "Epoch: [119/200]  Loss: 0.75(0.86)  Acc: 80.5%(77.5%)  Training: 185.0sec (1.6sec)  lr/momentum: 0.0426/0.90\n",
      "Epoch: [120/200]  Loss: 0.76(0.88)  Acc: 80.4%(77.3%)  Training: 186.5sec (1.6sec)  lr/momentum: 0.0421/0.90\n",
      "Epoch: [121/200]  Loss: 0.73(0.84)  Acc: 81.0%(78.1%)  Training: 188.1sec (1.6sec)  lr/momentum: 0.0415/0.90\n",
      "Epoch: [122/200]  Loss: 0.71(0.83)  Acc: 81.7%(78.8%)  Training: 189.6sec (1.5sec)  lr/momentum: 0.0410/0.90\n",
      "Epoch: [123/200]  Loss: 0.70(0.81)  Acc: 81.9%(79.3%)  Training: 191.2sec (1.5sec)  lr/momentum: 0.0405/0.90\n",
      "Epoch: [124/200]  Loss: 0.74(0.87)  Acc: 81.5%(78.4%)  Training: 192.7sec (1.5sec)  lr/momentum: 0.0399/0.90\n",
      "Epoch: [125/200]  Loss: 0.77(0.89)  Acc: 80.3%(77.7%)  Training: 194.3sec (1.6sec)  lr/momentum: 0.0394/0.90\n",
      "Epoch: [126/200]  Loss: 0.70(0.80)  Acc: 81.5%(79.2%)  Training: 195.9sec (1.6sec)  lr/momentum: 0.0389/0.90\n",
      "Epoch: [127/200]  Loss: 0.76(0.86)  Acc: 81.0%(78.0%)  Training: 197.5sec (1.6sec)  lr/momentum: 0.0384/0.90\n",
      "Epoch: [128/200]  Loss: 0.83(0.98)  Acc: 79.4%(75.9%)  Training: 199.1sec (1.6sec)  lr/momentum: 0.0378/0.90\n",
      "Epoch: [129/200]  Loss: 0.82(0.92)  Acc: 79.6%(76.5%)  Training: 200.6sec (1.6sec)  lr/momentum: 0.0373/0.90\n",
      "Epoch: [130/200]  Loss: 0.78(0.88)  Acc: 80.6%(78.2%)  Training: 202.2sec (1.5sec)  lr/momentum: 0.0368/0.90\n",
      "Epoch: [131/200]  Loss: 0.74(0.84)  Acc: 81.4%(78.9%)  Training: 203.8sec (1.6sec)  lr/momentum: 0.0363/0.90\n",
      "Epoch: [132/200]  Loss: 0.81(0.93)  Acc: 79.7%(76.6%)  Training: 205.3sec (1.6sec)  lr/momentum: 0.0357/0.90\n",
      "Epoch: [133/200]  Loss: 0.72(0.83)  Acc: 81.2%(78.3%)  Training: 206.9sec (1.6sec)  lr/momentum: 0.0352/0.90\n",
      "Epoch: [134/200]  Loss: 0.85(0.98)  Acc: 80.1%(77.3%)  Training: 208.5sec (1.6sec)  lr/momentum: 0.0347/0.90\n",
      "Epoch: [135/200]  Loss: 0.72(0.83)  Acc: 82.2%(79.7%)  Training: 210.0sec (1.6sec)  lr/momentum: 0.0342/0.90\n",
      "Epoch: [136/200]  Loss: 0.75(0.87)  Acc: 81.3%(78.3%)  Training: 211.6sec (1.6sec)  lr/momentum: 0.0336/0.90\n",
      "Epoch: [137/200]  Loss: 0.73(0.86)  Acc: 81.8%(78.6%)  Training: 213.2sec (1.5sec)  lr/momentum: 0.0331/0.90\n",
      "Epoch: [138/200]  Loss: 0.71(0.82)  Acc: 82.1%(80.1%)  Training: 214.8sec (1.7sec)  lr/momentum: 0.0326/0.90\n",
      "Epoch: [139/200]  Loss: 0.73(0.84)  Acc: 81.4%(78.7%)  Training: 216.4sec (1.5sec)  lr/momentum: 0.0321/0.90\n",
      "Epoch: [140/200]  Loss: 0.70(0.81)  Acc: 82.2%(79.4%)  Training: 217.9sec (1.5sec)  lr/momentum: 0.0315/0.90\n",
      "Epoch: [141/200]  Loss: 0.70(0.82)  Acc: 82.4%(79.5%)  Training: 219.4sec (1.5sec)  lr/momentum: 0.0310/0.90\n",
      "Epoch: [142/200]  Loss: 0.73(0.84)  Acc: 82.1%(79.2%)  Training: 221.0sec (1.6sec)  lr/momentum: 0.0305/0.90\n",
      "Epoch: [143/200]  Loss: 0.72(0.84)  Acc: 81.9%(79.2%)  Training: 222.5sec (1.5sec)  lr/momentum: 0.0299/0.90\n",
      "Epoch: [144/200]  Loss: 0.71(0.82)  Acc: 82.2%(79.7%)  Training: 224.1sec (1.6sec)  lr/momentum: 0.0294/0.90\n",
      "Epoch: [145/200]  Loss: 0.74(0.84)  Acc: 82.0%(79.1%)  Training: 225.7sec (1.5sec)  lr/momentum: 0.0289/0.90\n",
      "Epoch: [146/200]  Loss: 0.72(0.82)  Acc: 82.3%(79.9%)  Training: 227.3sec (1.6sec)  lr/momentum: 0.0284/0.90\n",
      "Epoch: [147/200]  Loss: 0.74(0.85)  Acc: 82.3%(79.7%)  Training: 228.8sec (1.6sec)  lr/momentum: 0.0278/0.90\n",
      "Epoch: [148/200]  Loss: 0.70(0.81)  Acc: 82.9%(80.4%)  Training: 230.4sec (1.5sec)  lr/momentum: 0.0273/0.90\n",
      "Epoch: [149/200]  Loss: 0.68(0.78)  Acc: 83.2%(80.8%)  Training: 231.9sec (1.5sec)  lr/momentum: 0.0268/0.90\n",
      "Epoch: [150/200]  Loss: 0.67(0.77)  Acc: 83.3%(80.7%)  Training: 233.5sec (1.6sec)  lr/momentum: 0.0263/0.90\n",
      "Epoch: [151/200]  Loss: 0.74(0.86)  Acc: 82.2%(79.1%)  Training: 235.1sec (1.6sec)  lr/momentum: 0.0257/0.90\n",
      "Epoch: [152/200]  Loss: 0.71(0.82)  Acc: 82.4%(79.7%)  Training: 236.7sec (1.5sec)  lr/momentum: 0.0252/0.90\n",
      "Epoch: [153/200]  Loss: 0.67(0.77)  Acc: 83.0%(80.3%)  Training: 238.3sec (1.6sec)  lr/momentum: 0.0247/0.90\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [154/200]  Loss: 0.69(0.79)  Acc: 82.7%(80.3%)  Training: 239.8sec (1.6sec)  lr/momentum: 0.0242/0.90\n",
      "Epoch: [155/200]  Loss: 0.71(0.80)  Acc: 82.6%(80.3%)  Training: 241.4sec (1.6sec)  lr/momentum: 0.0236/0.90\n",
      "Epoch: [156/200]  Loss: 0.78(0.90)  Acc: 81.7%(79.0%)  Training: 243.0sec (1.6sec)  lr/momentum: 0.0231/0.90\n",
      "Epoch: [157/200]  Loss: 0.71(0.82)  Acc: 82.6%(80.2%)  Training: 244.5sec (1.6sec)  lr/momentum: 0.0226/0.90\n",
      "Epoch: [158/200]  Loss: 0.68(0.78)  Acc: 83.3%(80.9%)  Training: 246.1sec (1.5sec)  lr/momentum: 0.0221/0.90\n",
      "Epoch: [159/200]  Loss: 0.66(0.76)  Acc: 83.6%(81.4%)  Training: 247.7sec (1.6sec)  lr/momentum: 0.0215/0.90\n",
      "Epoch: [160/200]  Loss: 0.66(0.77)  Acc: 83.3%(80.6%)  Training: 249.2sec (1.5sec)  lr/momentum: 0.0210/0.90\n",
      "Epoch: [161/200]  Loss: 0.69(0.79)  Acc: 82.7%(80.2%)  Training: 250.8sec (1.6sec)  lr/momentum: 0.0205/0.90\n",
      "Epoch: [162/200]  Loss: 0.69(0.80)  Acc: 82.9%(80.5%)  Training: 252.4sec (1.5sec)  lr/momentum: 0.0199/0.90\n",
      "Epoch: [163/200]  Loss: 0.69(0.78)  Acc: 83.1%(80.6%)  Training: 253.9sec (1.6sec)  lr/momentum: 0.0194/0.90\n",
      "Epoch: [164/200]  Loss: 0.68(0.79)  Acc: 83.1%(80.3%)  Training: 255.5sec (1.5sec)  lr/momentum: 0.0189/0.90\n",
      "Epoch: [165/200]  Loss: 0.68(0.78)  Acc: 83.0%(80.3%)  Training: 257.0sec (1.6sec)  lr/momentum: 0.0184/0.90\n",
      "Epoch: [166/200]  Loss: 0.67(0.78)  Acc: 83.5%(81.0%)  Training: 258.5sec (1.5sec)  lr/momentum: 0.0178/0.90\n",
      "Epoch: [167/200]  Loss: 0.67(0.77)  Acc: 83.6%(81.3%)  Training: 260.1sec (1.6sec)  lr/momentum: 0.0173/0.90\n",
      "Epoch: [168/200]  Loss: 0.68(0.79)  Acc: 83.6%(81.1%)  Training: 261.7sec (1.6sec)  lr/momentum: 0.0168/0.90\n",
      "Epoch: [169/200]  Loss: 0.67(0.79)  Acc: 83.6%(80.9%)  Training: 263.3sec (1.6sec)  lr/momentum: 0.0163/0.90\n",
      "Epoch: [170/200]  Loss: 0.67(0.79)  Acc: 83.3%(80.8%)  Training: 264.8sec (1.5sec)  lr/momentum: 0.0157/0.90\n",
      "Epoch: [171/200]  Loss: 0.67(0.77)  Acc: 83.4%(80.8%)  Training: 266.4sec (1.6sec)  lr/momentum: 0.0152/0.90\n",
      "Epoch: [172/200]  Loss: 0.70(0.80)  Acc: 83.0%(80.5%)  Training: 268.0sec (1.6sec)  lr/momentum: 0.0147/0.90\n",
      "Epoch: [173/200]  Loss: 0.66(0.76)  Acc: 83.6%(80.9%)  Training: 269.6sec (1.7sec)  lr/momentum: 0.0142/0.90\n",
      "Epoch: [174/200]  Loss: 0.66(0.78)  Acc: 83.4%(80.5%)  Training: 271.2sec (1.6sec)  lr/momentum: 0.0136/0.90\n",
      "Epoch: [175/200]  Loss: 0.65(0.75)  Acc: 83.8%(81.4%)  Training: 272.7sec (1.5sec)  lr/momentum: 0.0131/0.90\n",
      "Epoch: [176/200]  Loss: 0.66(0.77)  Acc: 83.5%(81.1%)  Training: 274.3sec (1.6sec)  lr/momentum: 0.0126/0.90\n",
      "Epoch: [177/200]  Loss: 0.65(0.75)  Acc: 83.8%(81.1%)  Training: 275.9sec (1.6sec)  lr/momentum: 0.0121/0.90\n",
      "Epoch: [178/200]  Loss: 0.68(0.80)  Acc: 83.0%(80.2%)  Training: 277.4sec (1.6sec)  lr/momentum: 0.0115/0.90\n",
      "Epoch: [179/200]  Loss: 0.66(0.76)  Acc: 83.5%(80.8%)  Training: 279.0sec (1.6sec)  lr/momentum: 0.0110/0.90\n",
      "Epoch: [180/200]  Loss: 0.66(0.77)  Acc: 83.6%(80.9%)  Training: 280.6sec (1.5sec)  lr/momentum: 0.0105/0.90\n",
      "Epoch: [181/200]  Loss: 0.66(0.77)  Acc: 83.9%(81.2%)  Training: 282.1sec (1.5sec)  lr/momentum: 0.0099/0.90\n",
      "Epoch: [182/200]  Loss: 0.65(0.75)  Acc: 84.0%(81.1%)  Training: 283.6sec (1.5sec)  lr/momentum: 0.0094/0.90\n",
      "Epoch: [183/200]  Loss: 0.66(0.76)  Acc: 83.7%(80.8%)  Training: 285.2sec (1.6sec)  lr/momentum: 0.0089/0.90\n",
      "Epoch: [184/200]  Loss: 0.65(0.74)  Acc: 83.8%(81.3%)  Training: 286.7sec (1.5sec)  lr/momentum: 0.0084/0.90\n",
      "Epoch: [185/200]  Loss: 0.64(0.74)  Acc: 84.1%(81.5%)  Training: 288.3sec (1.5sec)  lr/momentum: 0.0078/0.90\n",
      "Epoch: [186/200]  Loss: 0.65(0.75)  Acc: 83.9%(81.5%)  Training: 289.8sec (1.6sec)  lr/momentum: 0.0073/0.90\n",
      "Epoch: [187/200]  Loss: 0.64(0.74)  Acc: 84.1%(81.5%)  Training: 291.3sec (1.5sec)  lr/momentum: 0.0068/0.90\n",
      "Epoch: [188/200]  Loss: 0.64(0.74)  Acc: 84.1%(81.6%)  Training: 293.0sec (1.6sec)  lr/momentum: 0.0063/0.90\n",
      "Epoch: [189/200]  Loss: 0.64(0.75)  Acc: 84.2%(81.6%)  Training: 294.5sec (1.6sec)  lr/momentum: 0.0057/0.90\n",
      "Epoch: [190/200]  Loss: 0.64(0.75)  Acc: 84.2%(81.5%)  Training: 296.1sec (1.5sec)  lr/momentum: 0.0052/0.90\n",
      "Epoch: [191/200]  Loss: 0.65(0.75)  Acc: 84.0%(81.3%)  Training: 297.6sec (1.5sec)  lr/momentum: 0.0047/0.90\n",
      "Epoch: [192/200]  Loss: 0.65(0.75)  Acc: 84.0%(81.5%)  Training: 299.2sec (1.6sec)  lr/momentum: 0.0042/0.90\n",
      "Epoch: [193/200]  Loss: 0.64(0.74)  Acc: 84.1%(81.7%)  Training: 300.8sec (1.6sec)  lr/momentum: 0.0036/0.90\n",
      "Epoch: [194/200]  Loss: 0.65(0.75)  Acc: 84.0%(81.6%)  Training: 302.4sec (1.6sec)  lr/momentum: 0.0031/0.90\n",
      "Epoch: [195/200]  Loss: 0.65(0.75)  Acc: 84.0%(81.5%)  Training: 304.0sec (1.6sec)  lr/momentum: 0.0026/0.90\n",
      "Epoch: [196/200]  Loss: 0.65(0.75)  Acc: 84.0%(81.4%)  Training: 305.6sec (1.6sec)  lr/momentum: 0.0021/0.90\n",
      "Epoch: [197/200]  Loss: 0.64(0.74)  Acc: 84.0%(81.6%)  Training: 307.2sec (1.6sec)  lr/momentum: 0.0015/0.90\n",
      "Epoch: [198/200]  Loss: 0.64(0.74)  Acc: 84.1%(81.7%)  Training: 308.8sec (1.5sec)  lr/momentum: 0.0010/0.90\n",
      "Epoch: [199/200]  Loss: 0.64(0.74)  Acc: 84.1%(81.7%)  Training: 310.3sec (1.6sec)  lr/momentum: 0.0005/0.90\n",
      "Epoch: [200/200]  Loss: 0.64(0.74)  Acc: 84.1%(81.8%)  Training: 311.9sec (1.5sec)  lr/momentum: -0.0001/0.90\n"
     ]
    }
   ],
   "source": [
    "train_loss_list = []\n",
    "test_loss_list = []\n",
    "train_acc_list = []\n",
    "test_acc_list = []\n",
    "\n",
    "total_train_time = 0\n",
    "\n",
    "for epoch in range(n_epoch):\n",
    "    train_results = train(model, dataloader_train, device, optimizer, scheduler, criterion)\n",
    "    total_train_time += train_results[\"compute_time\"]\n",
    "    evaluate_test = evaluate(model, dataloader_test, criterion, device)\n",
    "    evaluate_train = evaluate(model, dataloader_trainnoaugment, criterion, device)\n",
    "    \n",
    "    \n",
    "    optim_param = get_lr(optimizer)\n",
    "    \n",
    "    print(\"Epoch: [{epoch}/{n_epoch}]  \"\n",
    "          \"Loss: {train_loss:.2f}({test_loss:.2f})  \"\n",
    "          \"Acc: {train_acc:.1f}%({test_acc:.1f}%)  \"\n",
    "          \"Training: {total_train_time:.1f}sec ({compute:.1f}sec)  \"\n",
    "          \"lr/momentum: {lr:.4f}/{momentum:.2f}\".format(\n",
    "        epoch=epoch+1, n_epoch=n_epoch,\n",
    "        train_loss = evaluate_train[\"avg_loss\"],\n",
    "        test_loss = evaluate_test[\"avg_loss\"],\n",
    "        train_acc = 100*evaluate_train[\"accuracy\"],\n",
    "        test_acc = 100*evaluate_test[\"accuracy\"],\n",
    "        total_train_time = total_train_time,\n",
    "        compute = train_results[\"compute_time\"],\n",
    "        lr = optim_param[\"lr\"],\n",
    "        momentum = optim_param[\"momentum\"],\n",
    "          ))\n",
    "    \n",
    "    #save statistics for later plotting\n",
    "    train_loss_list.append(evaluate_train[\"avg_loss\"])\n",
    "    test_loss_list.append(evaluate_test[\"avg_loss\"])\n",
    "    train_acc_list.append(evaluate_train[\"accuracy\"])\n",
    "    test_acc_list.append(evaluate_test[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "savefile = 'MixUp30000AugmentedFastResNet'\n",
    "savedir = 'saved_models/'\n",
    "checkpoint = {'model_state': model.state_dict(),\n",
    "              'optim_state': optimizer.state_dict(),\n",
    "              'acc': evaluate_test['accuracy']}\n",
    "torch.save(checkpoint, savedir + savefile + '.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dumping 10000 size data at Fast_resnet5000MixupCutout.pt1\n",
      "dumping 10000 size data at Fast_resnet5000MixupCutout.pt2\n",
      "dumping 10000 size data at Fast_resnet5000MixupCutout.pt3\n",
      "dumping 10000 size data at Fast_resnet5000MixupCutout.pt4\n",
      "dumping 10000 size data at Fast_resnet5000MixupCutout.pt5\n",
      "dumping 10000 size data at Fast_resnet5000MixupCutout_test1\n"
     ]
    }
   ],
   "source": [
    "trainset = reformed_CIFAR10(base, train=True, transform=train_augment, download=False)\n",
    "trainloader = torch.utils.data.DataLoader(trainset, batch_size=500,\n",
    "                                               shuffle=True, num_workers=5)\n",
    "testset = reformed_CIFAR10(base, train=False, transform=test_augment, download=False)\n",
    "testloader = torch.utils.data.DataLoader(testset, batch_size=500,\n",
    "                                               shuffle=True, num_workers=5)\n",
    "\n",
    "nbutils.encode_dump(model.encoder, 'Fast_resnet5000MixupCutout.pt', trainloader, device, False)\n",
    "nbutils.encode_dump(model.encoder, 'Fast_resnet5000MixupCutout_test', testloader, device, True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}