FuseNet_Full.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "FuseNet_Full_Adam.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-tgqAhYHvNg2"
      },
      "source": [
        "# Check GPU"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PA580dbFbyEn",
        "outputId": "a6eca76b-c7cc-40eb-b3c0-5c733c8ef5a0"
      },
      "source": [
        "gpu_info = !nvidia-smi\n",
        "gpu_info = '\\n'.join(gpu_info)\n",
        "if gpu_info.find('failed') >= 0:\n",
        "  print('Select the Runtime > \"Change runtime type\" menu to enable a GPU accelerator, ')\n",
        "  print('and then re-execute this cell.')\n",
        "else:\n",
        "  print(gpu_info)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Fri Dec 18 06:39:22 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 455.45.01    Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla V100-SXM2...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   39C    P0    25W / 300W |      0MiB / 16130MiB |      0%      Default |\n",
            "|                               |                      |                 ERR! |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JOC44nKYDOn8"
      },
      "source": [
        "# Run this command once and restart the runtime"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MhZffaB6C7Z7",
        "outputId": "7b4c61e3-d2bb-4b19-d174-5440449971a5"
      },
      "source": [
        "!pip install av"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting av\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9e/62/9a992be76f8e13ce0e3a24a838191b546805545116f9fc869bd11bd21b5f/av-8.0.2-cp36-cp36m-manylinux2010_x86_64.whl (36.9MB)\n",
            "\u001b[K     |████████████████████████████████| 36.9MB 151kB/s \n",
            "\u001b[?25hInstalling collected packages: av\n",
            "Successfully installed av-8.0.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ati1HrCZNHE_"
      },
      "source": [
        "# HMDB51 + DataLoader, Spatial"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rQwd2BDyLFxT"
      },
      "source": [
        "import glob\n",
        "import os\n",
        "import numpy as np\n",
        "\n",
        "from torchvision.datasets.utils import list_dir\n",
        "from torchvision.datasets.folder import make_dataset\n",
        "from torchvision.datasets.video_utils import VideoClips\n",
        "from torchvision.datasets.vision import VisionDataset\n",
        "\n",
        "class HMDB51(VisionDataset):\n",
        "    \"\"\"\n",
        "    Internally, it uses a VideoClips object to handle clip creation.\n",
        "    Args:\n",
        "        root (string): Root directory of the HMDB51 Dataset.\n",
        "        frames_per_clip (int): Number of frames in a clip.\n",
        "        step_between_clips (int): Number of frames between each clip.\n",
        "        train (bool, optional): If ``True``, creates a dataset from the train split,\n",
        "            otherwise from the ``test`` split.\n",
        "        transform (callable, optional): A function/transform that takes in a TxHxWxC video\n",
        "            and returns a transformed version.\n",
        "    Returns:\n",
        "        video (Tensor[T, H, W, C]): the `T` video frames\n",
        "        audio(Tensor[K, L]): the audio frames, where `K` is the number of channels\n",
        "            and `L` is the number of points\n",
        "        label (int): class of the video clip\n",
        "    \"\"\"\n",
        "    def __init__(self, root=\"\"):\n",
        "        super(HMDB51, self).__init__(root)\n",
        "\n",
        "    def init_data(self, root, frames_per_clip, step_between_clips=1,\n",
        "                 frame_rate=None, train=True, transform=None,\n",
        "                 _precomputed_metadata=None, num_workers=1, _video_width=0,\n",
        "                 _video_height=0, _video_min_dimension=0, _audio_samples=0):\n",
        "        super(HMDB51, self).__init__(root)\n",
        "        extensions = ('avi',)\n",
        "        if train:\n",
        "            root = root + \"/train\"\n",
        "        else:\n",
        "            root = root + \"/test\"\n",
        "        classes = sorted(list_dir(root))\n",
        "        class_to_idx = {class_: i for (i, class_) in enumerate(classes)}\n",
        "        print(class_to_idx)\n",
        "        self.samples = []\n",
        "        for target_class in sorted(class_to_idx.keys()):\n",
        "            class_index = class_to_idx[target_class]\n",
        "            target_dir = os.path.join(root, target_class)\n",
        "            for root_curr, _, fnames in sorted(os.walk(target_dir, followlinks=True)):\n",
        "                for fname in sorted(fnames):\n",
        "                    path = os.path.join(root_curr, fname)\n",
        "                    if os.path.isfile(path):\n",
        "                        item = path, class_index\n",
        "                        self.samples.append(item)\n",
        "\n",
        "        video_paths = [path for (path, _) in self.samples]\n",
        "        video_clips = VideoClips(\n",
        "            video_paths,\n",
        "            frames_per_clip,\n",
        "            step_between_clips,\n",
        "            frame_rate,\n",
        "            _precomputed_metadata,\n",
        "            num_workers=num_workers,\n",
        "            _video_width=_video_width,\n",
        "            _video_height=_video_height,\n",
        "            _video_min_dimension=_video_min_dimension,\n",
        "            _audio_samples=_audio_samples,\n",
        "        )\n",
        "        self.train = train\n",
        "        self.classes = classes\n",
        "        self.video_clips_metadata = video_clips.metadata\n",
        "        self.indices = self.get_indices(video_paths)\n",
        "        self.video_clips = video_clips.subset(self.indices)\n",
        "        self.transform = transform\n",
        "\n",
        "    @property\n",
        "    def metadata(self):\n",
        "        return self.video_clips_metadata\n",
        "\n",
        "    def get_indices(self, video_list):\n",
        "        indices = []\n",
        "        for video_index, video_path in enumerate(video_list):\n",
        "            indices.append(video_index)\n",
        "        return indices\n",
        "\n",
        "    def __len__(self):\n",
        "        return self.video_clips.num_clips()\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        video, _, _, video_idx = self.video_clips.get_clip(idx)\n",
        "        sample_index = self.indices[video_idx]\n",
        "        _, class_index = self.samples[sample_index]\n",
        "\n",
        "        if self.transform is not None:\n",
        "            video = video.permute(0, 3, 1, 2)\n",
        "            video = self.transform(video)\n",
        "            video = video.permute(0, 2, 3, 1)\n",
        "\n",
        "        return video, class_index, sample_index\n",
        "\n",
        "    def state_dict(self):\n",
        "        state = {\"video_clips\": self.video_clips,\n",
        "                 \"indices\": self.indices,\n",
        "                 \"samples\": self.samples,\n",
        "                 \"transform\": self.transform,\n",
        "                 \"metadata\": self.video_clips_metadata}\n",
        "\n",
        "        return state\n",
        "\n",
        "    def load_state_dict(self, state):\n",
        "        self.video_clips = state[\"video_clips\"]\n",
        "        self.indices = state[\"indices\"]\n",
        "        self.samples = state[\"samples\"]\n",
        "        self.transform = state[\"transform\"]\n",
        "        self.video_clips_metadata = state[\"metadata\"]\n",
        "\n",
        "import torch\n",
        "from torch.utils.data import DataLoader\n",
        "\n",
        "class BuildDataLoader(torch.utils.data.DataLoader):\n",
        "    def __init__(self, dataset, batch_size, shuffle, num_workers):\n",
        "        self.dataset = dataset\n",
        "        self.batch_size = batch_size\n",
        "        self.shuffle = shuffle\n",
        "        self.num_workers = num_workers\n",
        "        self.w_frame = 224\n",
        "        self.h_frame = 224\n",
        "\n",
        "    def collect_fn(self, batch):\n",
        "        video_list = []\n",
        "        label_list = []\n",
        "        index_list = []\n",
        "\n",
        "        for video, cl_index, s_index in batch:\n",
        "            video_list.append(video)\n",
        "            label_list.append(cl_index)\n",
        "            index_list.append(s_index)\n",
        "\n",
        "        data = {\"videos\": torch.stack(video_list),\n",
        "                \"labels\": label_list,\n",
        "                \"indexes\": index_list\n",
        "                }\n",
        "\n",
        "        return data\n",
        "\n",
        "    def loader(self):\n",
        "        return DataLoader(self.dataset,\n",
        "                          batch_size=self.batch_size,\n",
        "                          shuffle=self.shuffle,\n",
        "                          num_workers=self.num_workers,\n",
        "                          collate_fn=self.collect_fn)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nwpGXLdtuq8i"
      },
      "source": [
        "# Mount drive\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F6nsoL9Kuqil",
        "outputId": "b2eaa8e0-8539-4d10-bde2-d9547504916c"
      },
      "source": [
        "import os\n",
        "import io\n",
        "from google.colab import drive\n",
        "\n",
        "# Mount google drive\n",
        "DRIVE_MOUNT='/content/drive'\n",
        "\n",
        "drive.mount(DRIVE_MOUNT)\n",
        "\n",
        "\n",
        "# create folder to write data to\n",
        "DATA_FOLDER = os.path.join(DRIVE_MOUNT, 'Shared drives', 'CIS680 Final Project', 'data')"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n37IkHXBt2sn"
      },
      "source": [
        "# Load Dataset and Dataloaders"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Tpb1CP4et6AW",
        "outputId": "c905adb0-5240-4a41-fe35-45be542af7ca"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "\n",
        "\n",
        "train_dataset = HMDB51(DATA_FOLDER)\n",
        "test_dataset = HMDB51(DATA_FOLDER)\n",
        "\n",
        "train_dataset.load_state_dict(torch.load(DATA_FOLDER + \"/train_dataset_3.pt\"))\n",
        "test_dataset.load_state_dict(torch.load(DATA_FOLDER + \"/test_dataset_3.pt\"))\n",
        "\n",
        "batch_size = 64 #256\n",
        "train_build_loader = BuildDataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=0)\n",
        "train_loader = train_build_loader.loader()\n",
        "test_build_loader = BuildDataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0)\n",
        "test_loader = test_build_loader.loader()\n",
        "\n",
        "print(\"Length of train_loader: {}\".format(len(train_loader)))\n",
        "print(\"Length of test_loader: {}\".format(len(test_loader)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Length of train_loader: 7536\n",
            "Length of test_loader: 2208\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yo6FpL-ddpSI"
      },
      "source": [
        "# Spatial Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g8J0AEPhdqg8"
      },
      "source": [
        "import torch\n",
        "from torch.nn import functional as F\n",
        "from torchvision import transforms\n",
        "from torch import nn, Tensor\n",
        "import random\n",
        "import torchvision\n",
        "import torchvision.models as models\n",
        "import torch.optim as optim\n",
        "\n",
        "\n",
        "class SpatialStream(torch.nn.Module):\n",
        "\n",
        "    def __init__(self, \n",
        "                 device='cuda',\n",
        "                 num_classes=51,\n",
        "                 dropout_probability=0.5,\n",
        "                 train_resnet=True):\n",
        "\n",
        "        # Initialize the stream layers\n",
        "        super(SpatialStream, self).__init__()\n",
        "        self.device = device\n",
        "        self.num_classes = num_classes\n",
        "\n",
        "        # Spatial Backbone\n",
        "        self.spatial = models.resnet50(pretrained=True)\n",
        "        for param in self.spatial.parameters():\n",
        "            param.requires_grad = train_resnet  # False: Freezes the weights of the pre-trained model\n",
        "\n",
        "        # Add FC layers to Spatial Backbone\n",
        "        self.spatial.fc = nn.Sequential(nn.Linear(2048, 1024),\n",
        "                                nn.ReLU(),\n",
        "                                nn.Dropout(p=dropout_probability),\n",
        "                                nn.Linear(1024, self.num_classes),\n",
        "                                nn.ReLU(),\n",
        "                                nn.Dropout(p=dropout_probability),\n",
        "                                nn.Softmax())\n",
        "\n",
        "    def forward(self, X):\n",
        "        return self.spatial(X)\n",
        "\n",
        "    def compute_loss(self, output, labels):\n",
        "        criterion = nn.CrossEntropyLoss()\n",
        "        loss = criterion(output, labels)\n",
        "        return loss"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MR8qQX0YT83S"
      },
      "source": [
        "# Temporal Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wlaLhE8VT8U0"
      },
      "source": [
        "class TemporalStream(torch.nn.Module):\n",
        "\n",
        "    def __init__(self,\n",
        "                 device='cuda',\n",
        "                 num_classes=51,\n",
        "                 dropout_probability=0.5):\n",
        "\n",
        "        # Initialize the stream layers\n",
        "        super(TemporalStream, self).__init__()\n",
        "        self.device = device\n",
        "        self.num_classes = num_classes\n",
        "\n",
        "        # Temporal Backbone\n",
        "        self.temporal = models.resnet50(pretrained=True)\n",
        "        for param in self.temporal.parameters():\n",
        "            param.requires_grad = True\n",
        "\n",
        "        # Add FC layers to Spatial Backbone\n",
        "        self.temporal.fc = nn.Sequential(nn.Linear(2048, 1024),\n",
        "                                nn.ReLU(),\n",
        "                                nn.Dropout(p=dropout_probability),\n",
        "                                nn.Linear(1024, self.num_classes),\n",
        "                                nn.ReLU(),\n",
        "                                nn.Dropout(p=dropout_probability),\n",
        "                                nn.Softmax())\n",
        "        \n",
        "\n",
        "    def forward(self, X):\n",
        "        X = self.temporal(X)\n",
        "        return X\n",
        "\n",
        "    def compute_loss(self, output, target):\n",
        "        criterion = nn.CrossEntropyLoss()\n",
        "        loss = criterion(output, target)\n",
        "        return loss"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_BkCNZhhT_70"
      },
      "source": [
        "# Fuse Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xeUFQULLT_Kl"
      },
      "source": [
        "class FuseNET(torch.nn.Module):\n",
        "\n",
        "    def __init__(self, \n",
        "                 device='cuda',\n",
        "                 num_classes=51):\n",
        "\n",
        "        # Initialize the fuseNet FC layers\n",
        "        super(FuseNET, self).__init__()\n",
        "        self.device = device\n",
        "        self.num_classes = num_classes\n",
        "\n",
        "        self.fc1 = nn.Linear(102, 204)\n",
        "        self.output = nn.Linear(204, self.num_classes)\n",
        "\n",
        "    def forward(self, X):\n",
        "        X = F.relu(self.fc1(X))\n",
        "        X = (self.output(X))\n",
        "        return X\n",
        "\n",
        "    def compute_loss(self, output, labels):\n",
        "        criterion = nn.CrossEntropyLoss()\n",
        "        loss = criterion(output, labels)\n",
        "        return loss"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8pEvnrSVeftj"
      },
      "source": [
        "# Video Frame Stacking (SG3I)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1LIHVXrkzK1N"
      },
      "source": [
        "from google.colab.patches import cv2_imshow\n",
        "import cv2 as cv\n",
        "\n",
        "def getSG3I(videos):\n",
        "    bz = videos.size(0)\n",
        "\n",
        "    frame_list_batch = []\n",
        "\n",
        "    for b in range(bz):\n",
        "        images = videos[b]\n",
        "\n",
        "        w = images[0].size(0)\n",
        "        h = images[0].size(1)\n",
        "        num_frames = images.size(0)\n",
        "        frame_list = []\n",
        "\n",
        "        for i in range(0, num_frames):\n",
        "            frame_list.append(cv.cvtColor(images[i].numpy(), cv.COLOR_BGR2GRAY))\n",
        "\n",
        "        frame_list_batch.append(np.stack(frame_list, axis=-1))\n",
        "\n",
        "    SG3I = np.stack(frame_list_batch, axis=0)\n",
        "\n",
        "    return torch.Tensor(SG3I)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jlz-bn5oUJxJ"
      },
      "source": [
        "# Train and Test Functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ggkzqmwzefQq"
      },
      "source": [
        "  def train(epoch):\n",
        "    spatial.eval()\n",
        "    temporal.eval()\n",
        "    fused.train()\n",
        "\n",
        "    counter = 0\n",
        "    train_loss = 0\n",
        "    log_interval = 100\n",
        "    save_interval = 250\n",
        "\n",
        "    epoch_loss = []\n",
        "    log_int_loss = 0\n",
        "    for iter, data in enumerate(train_loader, 0):\n",
        "\n",
        "        videos = data[\"videos\"]\n",
        "        labels = torch.tensor(data[\"labels\"])\n",
        "        indexes = data[\"indexes\"]\n",
        "        \n",
        "        SG3I = getSG3I(videos)\n",
        "        SG3I = SG3I.permute(0,3,1,2)\n",
        "        SG3I = SG3I.to(device)\n",
        "\n",
        "        videos = videos.type(torch.FloatTensor)\n",
        "        videos = videos.to(device)\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        # spatial\n",
        "        spatial_input = videos[:,0,:,:].permute(0,3,2,1)\n",
        "        spatial_output = spatial(spatial_input)\n",
        "\n",
        "        # temporal        \n",
        "        temporal_output = temporal(SG3I)\n",
        "        \n",
        "        # fused\n",
        "        fused_input = torch.hstack((spatial_output, temporal_output))\n",
        "        fused_output = fused(fused_input)\n",
        "\n",
        "        fused_output = fused_output.to(device)\n",
        "        labels = labels.to(device)\n",
        "\n",
        "        # calculate losses\n",
        "        loss = fused.compute_loss(fused_output, labels)\n",
        "\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        train_loss += loss.item()\n",
        "\n",
        "        # Logging\n",
        "        log_int_loss += loss.item()\n",
        "        epoch_loss.append(loss.item())\n",
        "\n",
        "        if counter == 0:\n",
        "            print('Epoch: ', epoch, ', Batch: ', iter, ', loss avg over log interval: ', log_int_loss)\n",
        "            train_loss_list.append(train_loss / (iter + 1) * batch_size)\n",
        "            train_counter.append((iter + 1) * batch_size + epoch * len(train_loader.dataset))\n",
        "            log_int_loss = 0\n",
        "        elif counter % log_interval == log_interval - 1:\n",
        "            print('Epoch: ', epoch, ', Batch: ', iter, ', loss avg over log interval: ', log_int_loss / log_interval)\n",
        "            train_loss_list.append(train_loss / (iter + 1) * batch_size)\n",
        "            train_counter.append((iter + 1) * batch_size + epoch * len(train_loader.dataset))\n",
        "            log_int_loss = 0\n",
        "\n",
        "        # Save model every save_interval\n",
        "        if counter % save_interval == save_interval - 1:\n",
        "            print('saving model')\n",
        "            save_path = os.path.join(EPOCH_SAVE_PREFIX, 'fused_epoch' + str(epoch) + '_iter_' + str(counter))\n",
        "            torch.save({\n",
        "                'epoch': epoch,\n",
        "                'train_total_loss_list': train_loss_list,\n",
        "                'epoch_total_loss_list': epoch_loss_list,\n",
        "                'test_loss_list': test_loss_list,\n",
        "                'train_counter': train_counter,\n",
        "                'accuracy_list': accuracy_list,\n",
        "                'model_state_dict': fused.state_dict(),\n",
        "                'optimizer_state_dict': optimizer.state_dict()\n",
        "            }, save_path)\n",
        "\n",
        "        counter += 1\n",
        "\n",
        "\n",
        "    avg_loss = sum(epoch_loss) / len(epoch_loss)\n",
        "    epoch_loss_list.append(avg_loss)\n",
        "    print('Epoch: ', epoch, ', avg total loss: ', avg_loss)\n",
        "\n",
        "def test():\n",
        "    fused.eval()\n",
        "    test_loss = 0\n",
        "    correct = 0\n",
        "\n",
        "    with torch.no_grad():\n",
        "        for iter, data in enumerate(test_loader, 0):\n",
        "            videos = data[\"videos\"]\n",
        "            labels = torch.tensor(data[\"labels\"])\n",
        "            indexes = data[\"indexes\"]\n",
        "            \n",
        "            SG3I = getSG3I(videos)\n",
        "            SG3I = SG3I.permute(0,3,1,2)\n",
        "            SG3I = SG3I.to(device)\n",
        "\n",
        "            videos = videos.type(torch.FloatTensor)\n",
        "            videos = videos.to(device)\n",
        "\n",
        "            # spatial\n",
        "            spatial_input = videos[:,0,:,:].permute(0,3,1,2)\n",
        "            spatial_output = spatial(spatial_input)\n",
        "\n",
        "            # temporal        \n",
        "            temporal_output = temporal(SG3I)\n",
        "            \n",
        "            # fused\n",
        "            fused_input = torch.hstack((spatial_output, temporal_output))\n",
        "            fused_output = fused(fused_input)\n",
        "\n",
        "            fused_output = fused_output.to(device)\n",
        "            labels = labels.to(device)\n",
        "\n",
        "            # calculate losses\n",
        "            loss = fused.compute_loss(fused_output, labels)\n",
        "\n",
        "            test_loss += loss.item()\n",
        "\n",
        "            # calculate number of correct predictions in batch for accuracy\n",
        "            correct += sum(torch.argmax(fused_output,1) == labels).item()\n",
        "            if iter % 100 == 0:\n",
        "                print (\"iter  \", iter)\n",
        "                print(\"accuracy so far = \", correct / ((iter + 1) * len(labels)))\n",
        "\n",
        "    # Log\n",
        "    test_loss_list.append(test_loss / len(test_loader.dataset))\n",
        "    accuracy = correct / len(test_loader.dataset)\n",
        "    accuracy_list.append(accuracy)\n",
        "    print('Avg Validation Loss: ', test_loss / len(test_loader.dataset))\n",
        "    print('Accuracy: ', accuracy)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sxNfXCLLYcbW"
      },
      "source": [
        "# Main Training Loop"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "SiW7i49DYbwc",
        "outputId": "7afd97dc-ec4c-4074-bbe7-7751b735f669"
      },
      "source": [
        "EPOCH_SAVE_PREFIX = '/content/drive/Shared drives/CIS680 Final Project/models/fused_full_rev1/'\n",
        "\n",
        "# Device\n",
        "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
        "torch.random.manual_seed(1)\n",
        "\n",
        "# load trained spatial\n",
        "spatial = SpatialStream()\n",
        "spatial.to(device)\n",
        "spatial_network_path = '/content/drive/Shared drives/CIS680 Final Project/models/spatial/spatial_epoch10'\n",
        "checkpoint_spatial = torch.load(spatial_network_path)\n",
        "spatial.load_state_dict(checkpoint_spatial['model_state_dict'])\n",
        "\n",
        "# load trained temporal\n",
        "temporal = TemporalStream()\n",
        "temporal.to(device)\n",
        "temporal_network_path = '/content/drive/Shared drives/CIS680 Final Project/models/temporal_SG3I/temporal_epoch13'\n",
        "checkpoint_temporal = torch.load(temporal_network_path)\n",
        "temporal.load_state_dict(checkpoint_temporal['model_state_dict'])\n",
        "\n",
        "\n",
        "# fused model\n",
        "learning_rate = 0.001\n",
        "fused = FuseNET()\n",
        "fused.to(device)\n",
        "optimizer = optim.Adam(fused.parameters(), lr=learning_rate)\n",
        "\n",
        "# Epochs\n",
        "num_epochs = 50\n",
        "\n",
        "# Logging setup: train\n",
        "train_loss_list = []\n",
        "epoch_loss_list = []\n",
        "train_counter = []\n",
        "\n",
        "# Logging setup: test\n",
        "test_loss_list = []\n",
        "accuracy_list = []\n",
        "epoch_list = np.arange(num_epochs)\n",
        "\n",
        "# epoch loop\n",
        "for epoch in range(num_epochs):\n",
        "\n",
        "    # Train & Validate\n",
        "    train(epoch)\n",
        "    test()\n",
        "\n",
        "    # Save Model Version\n",
        "    save_path = os.path.join(EPOCH_SAVE_PREFIX, 'fused_epoch' + str(epoch))\n",
        "    torch.save({\n",
        "        'epoch': epoch,\n",
        "        'train_total_loss_list': train_loss_list,\n",
        "        'epoch_total_loss_list': epoch_loss_list,\n",
        "        'test_loss_list': test_loss_list,\n",
        "        'train_counter': train_counter,\n",
        "        'accuracy_list': accuracy_list,\n",
        "        'model_state_dict': fused.state_dict(),\n",
        "        'optimizer_state_dict': optimizer.state_dict()\n",
        "    }, save_path)\n",
        "\n",
        "    print(\"Epoch %d/%d Completed\" % (epoch, num_epochs - 1))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torchvision/io/video.py:117: UserWarning: The pts_unit 'pts' gives wrong results and will be removed in a follow-up version. Please use pts_unit 'sec'.\n",
            "  + \"follow-up version. Please use pts_unit 'sec'.\"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/container.py:117: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
            "  input = module(input)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch:  0 , Batch:  0 , loss avg over log interval:  3.9202094078063965\n",
            "Epoch:  0 , Batch:  99 , loss avg over log interval:  3.8763588643074036\n",
            "Epoch:  0 , Batch:  199 , loss avg over log interval:  3.8927001333236695\n",
            "saving model\n",
            "Epoch:  0 , Batch:  299 , loss avg over log interval:  3.871374843120575\n",
            "Epoch:  0 , Batch:  399 , loss avg over log interval:  3.8504884529113768\n",
            "Epoch:  0 , Batch:  499 , loss avg over log interval:  3.8290352511405943\n",
            "saving model\n",
            "Epoch:  0 , Batch:  599 , loss avg over log interval:  3.794836151599884\n",
            "Epoch:  0 , Batch:  699 , loss avg over log interval:  3.775216383934021\n",
            "saving model\n",
            "Epoch:  0 , Batch:  799 , loss avg over log interval:  3.749879169464111\n",
            "Epoch:  0 , Batch:  899 , loss avg over log interval:  3.7256114721298217\n",
            "Epoch:  0 , Batch:  999 , loss avg over log interval:  3.696100504398346\n",
            "saving model\n",
            "Epoch:  0 , Batch:  1099 , loss avg over log interval:  3.669394779205322\n",
            "Epoch:  0 , Batch:  1199 , loss avg over log interval:  3.647586965560913\n",
            "saving model\n",
            "Epoch:  0 , Batch:  1299 , loss avg over log interval:  3.6281874918937684\n",
            "Epoch:  0 , Batch:  1399 , loss avg over log interval:  3.583846368789673\n",
            "Epoch:  0 , Batch:  1499 , loss avg over log interval:  3.5654109430313112\n",
            "saving model\n",
            "Epoch:  0 , Batch:  1599 , loss avg over log interval:  3.5526434326171876\n",
            "Epoch:  0 , Batch:  1699 , loss avg over log interval:  3.5322236108779905\n",
            "saving model\n",
            "Epoch:  0 , Batch:  1799 , loss avg over log interval:  3.4988862466812134\n",
            "Epoch:  0 , Batch:  1899 , loss avg over log interval:  3.4564456868171693\n",
            "Epoch:  0 , Batch:  1999 , loss avg over log interval:  3.448100368976593\n",
            "saving model\n",
            "Epoch:  0 , Batch:  2099 , loss avg over log interval:  3.423282060623169\n",
            "Epoch:  0 , Batch:  2199 , loss avg over log interval:  3.4265311980247497\n",
            "saving model\n",
            "Epoch:  0 , Batch:  2299 , loss avg over log interval:  3.3668394017219545\n",
            "Epoch:  0 , Batch:  2399 , loss avg over log interval:  3.3714454221725463\n",
            "Epoch:  0 , Batch:  2499 , loss avg over log interval:  3.3604759311676027\n",
            "saving model\n",
            "Epoch:  0 , Batch:  2599 , loss avg over log interval:  3.321577837467194\n",
            "Epoch:  0 , Batch:  2699 , loss avg over log interval:  3.321051936149597\n",
            "saving model\n",
            "Epoch:  0 , Batch:  2799 , loss avg over log interval:  3.2849931931495666\n",
            "Epoch:  0 , Batch:  2899 , loss avg over log interval:  3.248868727684021\n",
            "Epoch:  0 , Batch:  2999 , loss avg over log interval:  3.2532954478263854\n",
            "saving model\n",
            "Epoch:  0 , Batch:  3099 , loss avg over log interval:  3.2205949664115905\n",
            "Epoch:  0 , Batch:  3199 , loss avg over log interval:  3.2033049178123476\n",
            "saving model\n",
            "Epoch:  0 , Batch:  3299 , loss avg over log interval:  3.192997612953186\n",
            "Epoch:  0 , Batch:  3399 , loss avg over log interval:  3.144367926120758\n",
            "Epoch:  0 , Batch:  3499 , loss avg over log interval:  3.1325935387611388\n",
            "saving model\n",
            "Epoch:  0 , Batch:  3599 , loss avg over log interval:  3.132238013744354\n",
            "Epoch:  0 , Batch:  3699 , loss avg over log interval:  3.09243791103363\n",
            "saving model\n",
            "Epoch:  0 , Batch:  3799 , loss avg over log interval:  3.066187219619751\n",
            "Epoch:  0 , Batch:  3899 , loss avg over log interval:  3.07783940076828\n",
            "Epoch:  0 , Batch:  3999 , loss avg over log interval:  3.042679002285004\n",
            "saving model\n",
            "Epoch:  0 , Batch:  4099 , loss avg over log interval:  3.0216386127471924\n",
            "Epoch:  0 , Batch:  4199 , loss avg over log interval:  2.984067163467407\n",
            "saving model\n",
            "Epoch:  0 , Batch:  4299 , loss avg over log interval:  2.9719097948074342\n",
            "Epoch:  0 , Batch:  4399 , loss avg over log interval:  2.9137402772903442\n",
            "Epoch:  0 , Batch:  4499 , loss avg over log interval:  2.8924204969406127\n",
            "saving model\n",
            "Epoch:  0 , Batch:  4599 , loss avg over log interval:  2.8729081749916077\n",
            "Epoch:  0 , Batch:  4699 , loss avg over log interval:  2.8552292442321776\n",
            "saving model\n",
            "Epoch:  0 , Batch:  4799 , loss avg over log interval:  2.871864531040192\n",
            "Epoch:  0 , Batch:  4899 , loss avg over log interval:  2.8231671118736266\n",
            "Epoch:  0 , Batch:  4999 , loss avg over log interval:  2.7894518041610716\n",
            "saving model\n",
            "Epoch:  0 , Batch:  5099 , loss avg over log interval:  2.757572386264801\n",
            "Epoch:  0 , Batch:  5199 , loss avg over log interval:  2.726556701660156\n",
            "saving model\n",
            "Epoch:  0 , Batch:  5299 , loss avg over log interval:  2.7270304584503173\n",
            "Epoch:  0 , Batch:  5399 , loss avg over log interval:  2.741217212677002\n",
            "Epoch:  0 , Batch:  5499 , loss avg over log interval:  2.6759472942352294\n",
            "saving model\n",
            "Epoch:  0 , Batch:  5599 , loss avg over log interval:  2.627638909816742\n",
            "Epoch:  0 , Batch:  5699 , loss avg over log interval:  2.6530385565757753\n",
            "saving model\n",
            "Epoch:  0 , Batch:  5799 , loss avg over log interval:  2.580761227607727\n",
            "Epoch:  0 , Batch:  5899 , loss avg over log interval:  2.5383498859405518\n",
            "Epoch:  0 , Batch:  5999 , loss avg over log interval:  2.5009910118579866\n",
            "saving model\n",
            "Epoch:  0 , Batch:  6099 , loss avg over log interval:  2.5070508527755737\n",
            "Epoch:  0 , Batch:  6199 , loss avg over log interval:  2.467837426662445\n",
            "saving model\n",
            "Epoch:  0 , Batch:  6299 , loss avg over log interval:  2.45408566236496\n",
            "Epoch:  0 , Batch:  6399 , loss avg over log interval:  2.4518697690963744\n",
            "Epoch:  0 , Batch:  6499 , loss avg over log interval:  2.3730763137340545\n",
            "saving model\n",
            "Epoch:  0 , Batch:  6599 , loss avg over log interval:  2.3760646069049836\n",
            "Epoch:  0 , Batch:  6699 , loss avg over log interval:  2.3307242572307585\n",
            "saving model\n",
            "Epoch:  0 , Batch:  6799 , loss avg over log interval:  2.3539966130256653\n",
            "Epoch:  0 , Batch:  6899 , loss avg over log interval:  2.278090064525604\n",
            "Epoch:  0 , Batch:  6999 , loss avg over log interval:  2.2663308584690096\n",
            "saving model\n",
            "Epoch:  0 , Batch:  7099 , loss avg over log interval:  2.2415694332122804\n",
            "Epoch:  0 , Batch:  7199 , loss avg over log interval:  2.244133937358856\n",
            "saving model\n",
            "Epoch:  0 , Batch:  7299 , loss avg over log interval:  2.1837424623966215\n",
            "Epoch:  0 , Batch:  7399 , loss avg over log interval:  2.197947541475296\n",
            "Epoch:  0 , Batch:  7499 , loss avg over log interval:  2.1732301926612854\n",
            "saving model\n",
            "Epoch:  0 , avg total loss:  3.046092553250096\n",
            "iter   0\n",
            "accuracy so far =  0.921875\n",
            "iter   100\n",
            "accuracy so far =  0.5456373762376238\n",
            "iter   200\n",
            "accuracy so far =  0.40469527363184077\n",
            "iter   300\n",
            "accuracy so far =  0.4619497508305648\n",
            "iter   400\n",
            "accuracy so far =  0.43788965087281795\n",
            "iter   500\n",
            "accuracy so far =  0.4465444111776447\n",
            "iter   600\n",
            "accuracy so far =  0.41846921797004993\n",
            "iter   700\n",
            "accuracy so far =  0.3592412624821683\n",
            "iter   800\n",
            "accuracy so far =  0.31439216604244696\n",
            "iter   900\n",
            "accuracy so far =  0.31621115427303\n",
            "iter   1000\n",
            "accuracy so far =  0.36909965034965037\n",
            "iter   1100\n",
            "accuracy so far =  0.3856579246139873\n",
            "iter   1200\n",
            "accuracy so far =  0.3864878226477935\n",
            "iter   1300\n",
            "accuracy so far =  0.38419965411222134\n",
            "iter   1400\n",
            "accuracy so far =  0.40192273376159887\n",
            "iter   1500\n",
            "accuracy so far =  0.4104555296469021\n",
            "iter   1600\n",
            "accuracy so far =  0.4369632261086821\n",
            "iter   1700\n",
            "accuracy so far =  0.4201664462081129\n",
            "iter   1800\n",
            "accuracy so far =  0.41756315935591337\n",
            "iter   1900\n",
            "accuracy so far =  0.4124720541820095\n",
            "iter   2000\n",
            "accuracy so far =  0.42182033983008493\n",
            "iter   2100\n",
            "accuracy so far =  0.42085613993336507\n",
            "iter   2200\n",
            "accuracy so far =  0.4208953316674239\n",
            "Avg Validation Loss:  0.04731358077505281\n",
            "Accuracy:  0.41973621799183025\n",
            "Epoch 0/49 Completed\n",
            "Epoch:  1 , Batch:  0 , loss avg over log interval:  1.883988380432129\n",
            "Epoch:  1 , Batch:  99 , loss avg over log interval:  2.1311711120605468\n",
            "Epoch:  1 , Batch:  199 , loss avg over log interval:  2.1206587064266205\n",
            "saving model\n",
            "Epoch:  1 , Batch:  299 , loss avg over log interval:  2.1113971674442293\n",
            "Epoch:  1 , Batch:  399 , loss avg over log interval:  2.077808645963669\n",
            "Epoch:  1 , Batch:  499 , loss avg over log interval:  2.052529709339142\n",
            "saving model\n",
            "Epoch:  1 , Batch:  599 , loss avg over log interval:  2.031461454629898\n",
            "Epoch:  1 , Batch:  699 , loss avg over log interval:  1.9983661842346192\n",
            "saving model\n",
            "Epoch:  1 , Batch:  799 , loss avg over log interval:  2.005646152496338\n",
            "Epoch:  1 , Batch:  899 , loss avg over log interval:  1.9140135991573333\n",
            "Epoch:  1 , Batch:  999 , loss avg over log interval:  1.9480234313011169\n",
            "saving model\n",
            "Epoch:  1 , Batch:  1099 , loss avg over log interval:  1.9239746606349946\n",
            "Epoch:  1 , Batch:  1199 , loss avg over log interval:  1.907155717611313\n",
            "saving model\n",
            "Epoch:  1 , Batch:  1299 , loss avg over log interval:  1.8803278970718384\n",
            "Epoch:  1 , Batch:  1399 , loss avg over log interval:  1.898829060792923\n",
            "Epoch:  1 , Batch:  1499 , loss avg over log interval:  1.874874449968338\n",
            "saving model\n",
            "Epoch:  1 , Batch:  1599 , loss avg over log interval:  1.8306509625911713\n",
            "Epoch:  1 , Batch:  1699 , loss avg over log interval:  1.8104448568820954\n",
            "saving model\n",
            "Epoch:  1 , Batch:  1799 , loss avg over log interval:  1.7840197551250458\n",
            "Epoch:  1 , Batch:  1899 , loss avg over log interval:  1.8061914837360382\n",
            "Epoch:  1 , Batch:  1999 , loss avg over log interval:  1.8135310995578766\n",
            "saving model\n",
            "Epoch:  1 , Batch:  2099 , loss avg over log interval:  1.7568202316761017\n",
            "Epoch:  1 , Batch:  2199 , loss avg over log interval:  1.7577429234981536\n",
            "saving model\n",
            "Epoch:  1 , Batch:  2299 , loss avg over log interval:  1.7643522536754608\n",
            "Epoch:  1 , Batch:  2399 , loss avg over log interval:  1.690257203578949\n",
            "Epoch:  1 , Batch:  2499 , loss avg over log interval:  1.7040135991573333\n",
            "saving model\n",
            "Epoch:  1 , Batch:  2599 , loss avg over log interval:  1.6796086716651917\n",
            "Epoch:  1 , Batch:  2699 , loss avg over log interval:  1.6660977745056151\n",
            "saving model\n",
            "Epoch:  1 , Batch:  2799 , loss avg over log interval:  1.6598419451713562\n",
            "Epoch:  1 , Batch:  2899 , loss avg over log interval:  1.6501330482959746\n",
            "Epoch:  1 , Batch:  2999 , loss avg over log interval:  1.6527220702171326\n",
            "saving model\n",
            "Epoch:  1 , Batch:  3099 , loss avg over log interval:  1.6195733428001404\n",
            "Epoch:  1 , Batch:  3199 , loss avg over log interval:  1.6233492231369018\n",
            "saving model\n",
            "Epoch:  1 , Batch:  3299 , loss avg over log interval:  1.6072201311588288\n",
            "Epoch:  1 , Batch:  3399 , loss avg over log interval:  1.590078376531601\n",
            "Epoch:  1 , Batch:  3499 , loss avg over log interval:  1.594063390493393\n",
            "saving model\n",
            "Epoch:  1 , Batch:  3599 , loss avg over log interval:  1.531196460723877\n",
            "Epoch:  1 , Batch:  3699 , loss avg over log interval:  1.5371031379699707\n",
            "saving model\n",
            "Epoch:  1 , Batch:  3799 , loss avg over log interval:  1.537160283923149\n",
            "Epoch:  1 , Batch:  3899 , loss avg over log interval:  1.5060790920257567\n",
            "Epoch:  1 , Batch:  3999 , loss avg over log interval:  1.4906975847482682\n",
            "saving model\n",
            "Epoch:  1 , Batch:  4099 , loss avg over log interval:  1.5086534750461578\n",
            "Epoch:  1 , Batch:  4199 , loss avg over log interval:  1.4685200917720795\n",
            "saving model\n",
            "Epoch:  1 , Batch:  4299 , loss avg over log interval:  1.4685293662548065\n",
            "Epoch:  1 , Batch:  4399 , loss avg over log interval:  1.448313807249069\n",
            "Epoch:  1 , Batch:  4499 , loss avg over log interval:  1.4504746961593629\n",
            "saving model\n",
            "Epoch:  1 , Batch:  4599 , loss avg over log interval:  1.4534693098068237\n",
            "Epoch:  1 , Batch:  4699 , loss avg over log interval:  1.3734715276956557\n",
            "saving model\n",
            "Epoch:  1 , Batch:  4799 , loss avg over log interval:  1.415719166994095\n",
            "Epoch:  1 , Batch:  4899 , loss avg over log interval:  1.3935149919986725\n",
            "Epoch:  1 , Batch:  4999 , loss avg over log interval:  1.3790942412614822\n",
            "saving model\n",
            "Epoch:  1 , Batch:  5099 , loss avg over log interval:  1.360223849415779\n",
            "Epoch:  1 , Batch:  5199 , loss avg over log interval:  1.3850828361511232\n",
            "saving model\n",
            "Epoch:  1 , Batch:  5299 , loss avg over log interval:  1.3322588843107224\n",
            "Epoch:  1 , Batch:  5399 , loss avg over log interval:  1.3163396614789962\n",
            "Epoch:  1 , Batch:  5499 , loss avg over log interval:  1.3775915729999542\n",
            "saving model\n",
            "Epoch:  1 , Batch:  5599 , loss avg over log interval:  1.314101337790489\n",
            "Epoch:  1 , Batch:  5699 , loss avg over log interval:  1.3339652341604233\n",
            "saving model\n",
            "Epoch:  1 , Batch:  5799 , loss avg over log interval:  1.2959849548339843\n",
            "Epoch:  1 , Batch:  5899 , loss avg over log interval:  1.270459218621254\n",
            "Epoch:  1 , Batch:  5999 , loss avg over log interval:  1.2727720326185226\n",
            "saving model\n",
            "Epoch:  1 , Batch:  6099 , loss avg over log interval:  1.284832255244255\n",
            "Epoch:  1 , Batch:  6199 , loss avg over log interval:  1.2498415845632553\n",
            "saving model\n",
            "Epoch:  1 , Batch:  6299 , loss avg over log interval:  1.252184765934944\n",
            "Epoch:  1 , Batch:  6399 , loss avg over log interval:  1.2376639491319656\n",
            "Epoch:  1 , Batch:  6499 , loss avg over log interval:  1.2306624537706374\n",
            "saving model\n",
            "Epoch:  1 , Batch:  6599 , loss avg over log interval:  1.2310466516017913\n",
            "Epoch:  1 , Batch:  6699 , loss avg over log interval:  1.1918037605285645\n",
            "saving model\n",
            "Epoch:  1 , Batch:  6799 , loss avg over log interval:  1.1831345641613007\n",
            "Epoch:  1 , Batch:  6899 , loss avg over log interval:  1.2363199692964555\n",
            "Epoch:  1 , Batch:  6999 , loss avg over log interval:  1.1754719758033751\n",
            "saving model\n",
            "Epoch:  1 , Batch:  7099 , loss avg over log interval:  1.2102083820104599\n",
            "Epoch:  1 , Batch:  7199 , loss avg over log interval:  1.143309515118599\n",
            "saving model\n",
            "Epoch:  1 , Batch:  7299 , loss avg over log interval:  1.172715979218483\n",
            "Epoch:  1 , Batch:  7399 , loss avg over log interval:  1.180872465968132\n",
            "Epoch:  1 , Batch:  7499 , loss avg over log interval:  1.1931259953975677\n",
            "saving model\n",
            "Epoch:  1 , avg total loss:  1.5626647468189652\n",
            "iter   0\n",
            "accuracy so far =  0.875\n",
            "iter   100\n",
            "accuracy so far =  0.4973700495049505\n",
            "iter   200\n",
            "accuracy so far =  0.4087375621890547\n",
            "iter   300\n",
            "accuracy so far =  0.4853612956810631\n",
            "iter   400\n",
            "accuracy so far =  0.49018079800498754\n",
            "iter   500\n",
            "accuracy so far =  0.48393837325349304\n",
            "iter   600\n",
            "accuracy so far =  0.463914309484193\n",
            "iter   700\n",
            "accuracy so far =  0.4351150142653352\n",
            "iter   800\n",
            "accuracy so far =  0.42780508739076156\n",
            "iter   900\n",
            "accuracy so far =  0.41363762486126526\n",
            "iter   1000\n",
            "accuracy so far =  0.4497065434565435\n",
            "iter   1100\n",
            "accuracy so far =  0.4557930290644868\n",
            "iter   1200\n",
            "accuracy so far =  0.47784398417985013\n",
            "iter   1300\n",
            "accuracy so far =  0.49429525365103766\n",
            "iter   1400\n",
            "accuracy so far =  0.5112419700214133\n",
            "iter   1500\n",
            "accuracy so far =  0.5148234510326449\n",
            "iter   1600\n",
            "accuracy so far =  0.5371447532792005\n",
            "iter   1700\n",
            "accuracy so far =  0.5285218253968254\n",
            "iter   1800\n",
            "accuracy so far =  0.515468836757357\n",
            "iter   1900\n",
            "accuracy so far =  0.5089015649658075\n",
            "iter   2000\n",
            "accuracy so far =  0.5134073588205897\n",
            "iter   2100\n",
            "accuracy so far =  0.5094672179914327\n",
            "iter   2200\n",
            "accuracy so far =  0.5033933439345752\n",
            "Avg Validation Loss:  0.038769804439267724\n",
            "Accuracy:  0.5020070370186828\n",
            "Epoch 1/49 Completed\n",
            "Epoch:  2 , Batch:  0 , loss avg over log interval:  1.2011412382125854\n",
            "Epoch:  2 , Batch:  99 , loss avg over log interval:  1.1045860624313355\n",
            "Epoch:  2 , Batch:  199 , loss avg over log interval:  1.1885276067256927\n",
            "saving model\n",
            "Epoch:  2 , Batch:  299 , loss avg over log interval:  1.1217454063892365\n",
            "Epoch:  2 , Batch:  399 , loss avg over log interval:  1.1518084287643433\n",
            "Epoch:  2 , Batch:  499 , loss avg over log interval:  1.114681327342987\n",
            "saving model\n",
            "Epoch:  2 , Batch:  599 , loss avg over log interval:  1.0778560608625412\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-20-90fff396af27>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     43\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m     \u001b[0;31m# Train & Validate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m     \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     46\u001b[0m     \u001b[0mtest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-11-8bfda0f4acf3>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(epoch)\u001b[0m\n\u001b[1;32m     11\u001b[0m   \u001b[0mepoch_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m   \u001b[0mlog_int_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m   \u001b[0;32mfor\u001b[0m \u001b[0miter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m       \u001b[0mvideos\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"videos\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    433\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sampler_iter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    434\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 435\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    436\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    437\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    473\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    474\u001b[0m         \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 475\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_fetcher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    476\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    477\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36mfetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     42\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_collation\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     42\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_collation\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-4-088534f00ba3>\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, idx)\u001b[0m\n\u001b[1;32m     86\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     87\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__getitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 88\u001b[0;31m         \u001b[0mvideo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvideo_idx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvideo_clips\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_clip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     89\u001b[0m         \u001b[0msample_index\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindices\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mvideo_idx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m         \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclass_index\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msamples\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0msample_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torchvision/datasets/video_utils.py\u001b[0m in \u001b[0;36mget_clip\u001b[0;34m(self, idx)\u001b[0m\n\u001b[1;32m    317\u001b[0m             \u001b[0mstart_pts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclip_pts\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    318\u001b[0m             \u001b[0mend_pts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclip_pts\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 319\u001b[0;31m             \u001b[0mvideo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maudio\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minfo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mread_video\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvideo_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstart_pts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend_pts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    320\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    321\u001b[0m             \u001b[0minfo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_probe_video_from_file\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvideo_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torchvision/io/video.py\u001b[0m in \u001b[0;36mread_video\u001b[0;34m(filename, start_pts, end_pts, pts_unit)\u001b[0m\n\u001b[1;32m    254\u001b[0m                     \u001b[0mpts_unit\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    255\u001b[0m                     \u001b[0mcontainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstreams\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvideo\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 256\u001b[0;31m                     \u001b[0;34m{\u001b[0m\u001b[0;34m\"video\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    257\u001b[0m                 )\n\u001b[1;32m    258\u001b[0m                 \u001b[0mvideo_fps\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcontainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstreams\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvideo\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maverage_rate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torchvision/io/video.py\u001b[0m in \u001b[0;36m_read_from_stream\u001b[0;34m(container, start_offset, end_offset, pts_unit, stream, stream_name)\u001b[0m\n\u001b[1;32m    106\u001b[0m     \u001b[0m_CALLED_TIMES\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    107\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0m_CALLED_TIMES\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0m_GC_COLLECTION_INTERVAL\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_GC_COLLECTION_INTERVAL\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 108\u001b[0;31m         \u001b[0mgc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    109\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    110\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mpts_unit\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"sec\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PDmKw9pMH3NJ"
      },
      "source": [
        "# Resume Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0UlgNih0H2DD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f9dc6a72-c31f-447a-9a2a-380fea55d744"
      },
      "source": [
        "EPOCH_SAVE_PREFIX = '/content/drive/Shared drives/CIS680 Final Project/models/fused_full_adam/'\n",
        "\n",
        "# Device\n",
        "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
        "torch.random.manual_seed(1)\n",
        "\n",
        "# load trained spatial\n",
        "spatial = SpatialStream()\n",
        "spatial.to(device)\n",
        "spatial_network_path = '/content/drive/Shared drives/CIS680 Final Project/models/spatial/spatial_epoch10'\n",
        "checkpoint_spatial = torch.load(spatial_network_path)\n",
        "spatial.load_state_dict(checkpoint_spatial['model_state_dict'])\n",
        "\n",
        "# load trained temporal\n",
        "temporal = TemporalStream()\n",
        "temporal.to(device)\n",
        "temporal_network_path = '/content/drive/Shared drives/CIS680 Final Project/models/temporal_SG3I/temporal_epoch13'\n",
        "checkpoint_temporal = torch.load(temporal_network_path)\n",
        "temporal.load_state_dict(checkpoint_temporal['model_state_dict'])\n",
        "\n",
        "\n",
        "# fused model\n",
        "learning_rate = 0.001\n",
        "fused = FuseNET()\n",
        "fused.to(device)\n",
        "optimizer = optim.Adam(fused.parameters(), lr=learning_rate)\n",
        "\n",
        "# Epochs\n",
        "num_epochs = 50\n",
        "epoch_list = np.arange(num_epochs)\n",
        "\n",
        "\n",
        "# LOAD NETWORK\n",
        "fused_network_path = '/content/drive/Shared drives/CIS680 Final Project/models/fused_full_adam/fused_epoch4'\n",
        "checkpoint_fused = torch.load(fused_network_path)\n",
        "fused.load_state_dict(checkpoint_fused['model_state_dict'])\n",
        "optimizer.load_state_dict(checkpoint_fused['optimizer_state_dict'])\n",
        "last_epoch = checkpoint_fused['epoch']\n",
        "\n",
        "# Logging setup: train\n",
        "train_loss_list = checkpoint_fused['train_total_loss_list']\n",
        "epoch_loss_list = checkpoint_fused['epoch_total_loss_list']\n",
        "\n",
        "test_loss_list = checkpoint_fused['test_loss_list']\n",
        "accuracy_list = checkpoint_fused['accuracy_list']\n",
        "train_counter = checkpoint_fused['train_counter']\n",
        "\n",
        "\n",
        "# epoch loop\n",
        "for epoch in range(last_epoch + 1, num_epochs):\n",
        "\n",
        "    # Train & Validate\n",
        "    train(epoch)\n",
        "    test()\n",
        "\n",
        "    # Save Model Version\n",
        "    save_path = os.path.join(EPOCH_SAVE_PREFIX, 'fused_epoch' + str(epoch))\n",
        "    torch.save({\n",
        "        'epoch': epoch,\n",
        "        'train_total_loss_list': train_loss_list,\n",
        "        'epoch_total_loss_list': epoch_loss_list,\n",
        "        'test_loss_list': test_loss_list,\n",
        "        'train_counter': train_counter,\n",
        "        'accuracy_list': accuracy_list,\n",
        "        'model_state_dict': fused.state_dict(),\n",
        "        'optimizer_state_dict': optimizer.state_dict()\n",
        "    }, save_path)\n",
        "\n",
        "    print(\"Epoch %d/%d Completed\" % (epoch, num_epochs - 1))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torchvision/io/video.py:117: UserWarning: The pts_unit 'pts' gives wrong results and will be removed in a follow-up version. Please use pts_unit 'sec'.\n",
            "  + \"follow-up version. Please use pts_unit 'sec'.\"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/container.py:117: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
            "  input = module(input)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch:  5 , Batch:  0 , loss avg over log interval:  0.47582653164863586\n",
            "Epoch:  5 , Batch:  99 , loss avg over log interval:  0.5713754890859127\n",
            "Epoch:  5 , Batch:  199 , loss avg over log interval:  0.5684914070367814\n",
            "saving model\n",
            "Epoch:  5 , Batch:  299 , loss avg over log interval:  0.580257708132267\n",
            "Epoch:  5 , Batch:  399 , loss avg over log interval:  0.5783504892885685\n",
            "Epoch:  5 , Batch:  499 , loss avg over log interval:  0.5721642392873764\n",
            "saving model\n",
            "Epoch:  5 , Batch:  599 , loss avg over log interval:  0.5439244997501373\n",
            "Epoch:  5 , Batch:  699 , loss avg over log interval:  0.5592129236459732\n",
            "saving model\n",
            "Epoch:  5 , Batch:  799 , loss avg over log interval:  0.5775558397173881\n",
            "Epoch:  5 , Batch:  899 , loss avg over log interval:  0.5647488498687744\n",
            "Epoch:  5 , Batch:  999 , loss avg over log interval:  0.523560116738081\n",
            "saving model\n",
            "Epoch:  5 , Batch:  1099 , loss avg over log interval:  0.5799291323125363\n",
            "Epoch:  5 , Batch:  1199 , loss avg over log interval:  0.5577403554320335\n",
            "saving model\n",
            "Epoch:  5 , Batch:  1299 , loss avg over log interval:  0.5465998784452677\n",
            "Epoch:  5 , Batch:  1399 , loss avg over log interval:  0.5585545578598976\n",
            "Epoch:  5 , Batch:  1499 , loss avg over log interval:  0.5507049643993378\n",
            "saving model\n",
            "Epoch:  5 , Batch:  1599 , loss avg over log interval:  0.5525855672359467\n",
            "Epoch:  5 , Batch:  1699 , loss avg over log interval:  0.5518142992258072\n",
            "saving model\n",
            "Epoch:  5 , Batch:  1799 , loss avg over log interval:  0.5862903836369514\n",
            "Epoch:  5 , Batch:  1899 , loss avg over log interval:  0.5380107866227627\n",
            "Epoch:  5 , Batch:  1999 , loss avg over log interval:  0.5425318197906017\n",
            "saving model\n",
            "Epoch:  5 , Batch:  2099 , loss avg over log interval:  0.5654461963474751\n",
            "Epoch:  5 , Batch:  2199 , loss avg over log interval:  0.5498278564214707\n",
            "saving model\n",
            "Epoch:  5 , Batch:  2299 , loss avg over log interval:  0.5511406365036965\n",
            "Epoch:  5 , Batch:  2399 , loss avg over log interval:  0.5369597201049328\n",
            "Epoch:  5 , Batch:  2499 , loss avg over log interval:  0.5341305078566074\n",
            "saving model\n",
            "Epoch:  5 , Batch:  2599 , loss avg over log interval:  0.5463207763433456\n",
            "Epoch:  5 , Batch:  2699 , loss avg over log interval:  0.5659303878247738\n",
            "saving model\n",
            "Epoch:  5 , Batch:  2799 , loss avg over log interval:  0.5522648440301419\n",
            "Epoch:  5 , Batch:  2899 , loss avg over log interval:  0.571158631592989\n",
            "Epoch:  5 , Batch:  2999 , loss avg over log interval:  0.5593078328669071\n",
            "saving model\n",
            "Epoch:  5 , Batch:  3099 , loss avg over log interval:  0.5525775867700576\n",
            "Epoch:  5 , Batch:  3199 , loss avg over log interval:  0.548035455942154\n",
            "saving model\n",
            "Epoch:  5 , Batch:  3299 , loss avg over log interval:  0.5487459234893322\n",
            "Epoch:  5 , Batch:  3399 , loss avg over log interval:  0.551231538951397\n",
            "Epoch:  5 , Batch:  3499 , loss avg over log interval:  0.5302421547472477\n",
            "saving model\n",
            "Epoch:  5 , Batch:  3599 , loss avg over log interval:  0.5522617404162884\n",
            "Epoch:  5 , Batch:  3699 , loss avg over log interval:  0.5510217815637588\n",
            "saving model\n",
            "Epoch:  5 , Batch:  3799 , loss avg over log interval:  0.536702455431223\n",
            "Epoch:  5 , Batch:  3899 , loss avg over log interval:  0.5430012926459312\n",
            "Epoch:  5 , Batch:  3999 , loss avg over log interval:  0.5247799451649189\n",
            "saving model\n",
            "Epoch:  5 , Batch:  4099 , loss avg over log interval:  0.5832037898898125\n",
            "Epoch:  5 , Batch:  4199 , loss avg over log interval:  0.5392197202146053\n",
            "saving model\n",
            "Epoch:  5 , Batch:  4299 , loss avg over log interval:  0.5617181798815727\n",
            "Epoch:  5 , Batch:  4399 , loss avg over log interval:  0.5342579951882362\n",
            "Epoch:  5 , Batch:  4499 , loss avg over log interval:  0.538149647116661\n",
            "saving model\n",
            "Epoch:  5 , Batch:  4599 , loss avg over log interval:  0.5432435743510723\n",
            "Epoch:  5 , Batch:  4699 , loss avg over log interval:  0.5226857224106789\n",
            "saving model\n",
            "Epoch:  5 , Batch:  4799 , loss avg over log interval:  0.560646350234747\n",
            "Epoch:  5 , Batch:  4899 , loss avg over log interval:  0.5559079098701477\n",
            "Epoch:  5 , Batch:  4999 , loss avg over log interval:  0.5209858965873718\n",
            "saving model\n",
            "Epoch:  5 , Batch:  5099 , loss avg over log interval:  0.5436450019478798\n",
            "Epoch:  5 , Batch:  5199 , loss avg over log interval:  0.5520159412920475\n",
            "saving model\n",
            "Epoch:  5 , Batch:  5299 , loss avg over log interval:  0.5295445582270623\n",
            "Epoch:  5 , Batch:  5399 , loss avg over log interval:  0.5613389986753464\n",
            "Epoch:  5 , Batch:  5499 , loss avg over log interval:  0.5467452079057693\n",
            "saving model\n",
            "Epoch:  5 , Batch:  5599 , loss avg over log interval:  0.5838939954340457\n",
            "Epoch:  5 , Batch:  5699 , loss avg over log interval:  0.5622398778796196\n",
            "saving model\n",
            "Epoch:  5 , Batch:  5799 , loss avg over log interval:  0.5433294600248337\n",
            "Epoch:  5 , Batch:  5899 , loss avg over log interval:  0.5302880470454693\n",
            "Epoch:  5 , Batch:  5999 , loss avg over log interval:  0.5361930753290653\n",
            "saving model\n",
            "Epoch:  5 , Batch:  6099 , loss avg over log interval:  0.5307937327027321\n",
            "Epoch:  5 , Batch:  6199 , loss avg over log interval:  0.5124640142917634\n",
            "saving model\n",
            "Epoch:  5 , Batch:  6299 , loss avg over log interval:  0.5192778480052948\n",
            "Epoch:  5 , Batch:  6399 , loss avg over log interval:  0.5603229823708534\n",
            "Epoch:  5 , Batch:  6499 , loss avg over log interval:  0.5332589846849441\n",
            "saving model\n",
            "Epoch:  5 , Batch:  6599 , loss avg over log interval:  0.5599873846769333\n",
            "Epoch:  5 , Batch:  6699 , loss avg over log interval:  0.5184911526739597\n",
            "saving model\n",
            "Epoch:  5 , Batch:  6799 , loss avg over log interval:  0.5539360155165195\n",
            "Epoch:  5 , Batch:  6899 , loss avg over log interval:  0.5101351787149906\n",
            "Epoch:  5 , Batch:  6999 , loss avg over log interval:  0.5475755725800991\n",
            "saving model\n",
            "Epoch:  5 , Batch:  7099 , loss avg over log interval:  0.5431745488941669\n",
            "Epoch:  5 , Batch:  7199 , loss avg over log interval:  0.516155794262886\n",
            "saving model\n",
            "Epoch:  5 , Batch:  7299 , loss avg over log interval:  0.5442997168004513\n",
            "Epoch:  5 , Batch:  7399 , loss avg over log interval:  0.5445071525871754\n",
            "Epoch:  5 , Batch:  7499 , loss avg over log interval:  0.5453938776999712\n",
            "saving model\n",
            "Epoch:  5 , avg total loss:  0.5490683081150529\n",
            "iter   0\n",
            "accuracy so far =  0.859375\n",
            "iter   100\n",
            "accuracy so far =  0.4786509900990099\n",
            "iter   200\n",
            "accuracy so far =  0.47201492537313433\n",
            "iter   300\n",
            "accuracy so far =  0.522217607973422\n",
            "iter   400\n",
            "accuracy so far =  0.49957138403990026\n",
            "iter   500\n",
            "accuracy so far =  0.5225486526946108\n",
            "iter   600\n",
            "accuracy so far =  0.49773814475873546\n",
            "iter   700\n",
            "accuracy so far =  0.4863364835948645\n",
            "iter   800\n",
            "accuracy so far =  0.4745240324594257\n",
            "iter   900\n",
            "accuracy so far =  0.4777330743618202\n",
            "iter   1000\n",
            "accuracy so far =  0.5014516733266733\n",
            "iter   1100\n",
            "accuracy so far =  0.49697717983651224\n",
            "iter   1200\n",
            "accuracy so far =  0.5164055995004163\n",
            "iter   1300\n",
            "accuracy so far =  0.530793620292083\n",
            "iter   1400\n",
            "accuracy so far =  0.5476668451106352\n",
            "iter   1500\n",
            "accuracy so far =  0.5448346935376416\n",
            "iter   1600\n",
            "accuracy so far =  0.5673895221736415\n",
            "iter   1700\n",
            "accuracy so far =  0.5612323633156967\n",
            "iter   1800\n",
            "accuracy so far =  0.5553598695169351\n",
            "iter   1900\n",
            "accuracy so far =  0.5424365465544451\n",
            "iter   2000\n",
            "accuracy so far =  0.5449697026486756\n",
            "iter   2100\n",
            "accuracy so far =  0.5376978224654926\n",
            "iter   2200\n",
            "accuracy so far =  0.5311151181281236\n",
            "Avg Validation Loss:  0.038107118473223255\n",
            "Accuracy:  0.5297161830191217\n",
            "Epoch 5/49 Completed\n",
            "Epoch:  6 , Batch:  0 , loss avg over log interval:  0.303065687417984\n",
            "Epoch:  6 , Batch:  99 , loss avg over log interval:  0.5175274220108986\n",
            "Epoch:  6 , Batch:  199 , loss avg over log interval:  0.5484818375110626\n",
            "saving model\n",
            "Epoch:  6 , Batch:  299 , loss avg over log interval:  0.5447055985033512\n",
            "Epoch:  6 , Batch:  399 , loss avg over log interval:  0.5466179019212722\n",
            "Epoch:  6 , Batch:  499 , loss avg over log interval:  0.5467489606142044\n",
            "saving model\n",
            "Epoch:  6 , Batch:  599 , loss avg over log interval:  0.5413720607757568\n",
            "Epoch:  6 , Batch:  699 , loss avg over log interval:  0.5240764695405961\n",
            "saving model\n",
            "Epoch:  6 , Batch:  799 , loss avg over log interval:  0.573781404197216\n",
            "Epoch:  6 , Batch:  899 , loss avg over log interval:  0.5167210192978382\n",
            "Epoch:  6 , Batch:  999 , loss avg over log interval:  0.5206863284111023\n",
            "saving model\n",
            "Epoch:  6 , Batch:  1099 , loss avg over log interval:  0.5235072512924671\n",
            "Epoch:  6 , Batch:  1199 , loss avg over log interval:  0.5268566879630089\n",
            "saving model\n",
            "Epoch:  6 , Batch:  1299 , loss avg over log interval:  0.5314368908107281\n",
            "Epoch:  6 , Batch:  1399 , loss avg over log interval:  0.5466885101795197\n",
            "Epoch:  6 , Batch:  1499 , loss avg over log interval:  0.5198717401921749\n",
            "saving model\n",
            "Epoch:  6 , Batch:  1599 , loss avg over log interval:  0.5284764567017555\n",
            "Epoch:  6 , Batch:  1699 , loss avg over log interval:  0.5199301056563854\n",
            "saving model\n",
            "Epoch:  6 , Batch:  1799 , loss avg over log interval:  0.5110266840457917\n",
            "Epoch:  6 , Batch:  1899 , loss avg over log interval:  0.5277077727019787\n",
            "Epoch:  6 , Batch:  1999 , loss avg over log interval:  0.5268743200600148\n",
            "saving model\n",
            "Epoch:  6 , Batch:  2099 , loss avg over log interval:  0.5226550011336804\n",
            "Epoch:  6 , Batch:  2199 , loss avg over log interval:  0.5343832278251648\n",
            "saving model\n",
            "Epoch:  6 , Batch:  2299 , loss avg over log interval:  0.5341726940870285\n",
            "Epoch:  6 , Batch:  2399 , loss avg over log interval:  0.5152749133110046\n",
            "Epoch:  6 , Batch:  2499 , loss avg over log interval:  0.5256990545988083\n",
            "saving model\n",
            "Epoch:  6 , Batch:  2599 , loss avg over log interval:  0.5384708765149117\n",
            "Epoch:  6 , Batch:  2699 , loss avg over log interval:  0.5369035258889199\n",
            "saving model\n",
            "Epoch:  6 , Batch:  2799 , loss avg over log interval:  0.5376747907698154\n",
            "Epoch:  6 , Batch:  2899 , loss avg over log interval:  0.5241218434274196\n",
            "Epoch:  6 , Batch:  2999 , loss avg over log interval:  0.5448334673047066\n",
            "saving model\n",
            "Epoch:  6 , Batch:  3099 , loss avg over log interval:  0.5144991961121559\n",
            "Epoch:  6 , Batch:  3199 , loss avg over log interval:  0.5253493277728558\n",
            "saving model\n",
            "Epoch:  6 , Batch:  3299 , loss avg over log interval:  0.5308088162541389\n",
            "Epoch:  6 , Batch:  3399 , loss avg over log interval:  0.55633222065866\n",
            "Epoch:  6 , Batch:  3499 , loss avg over log interval:  0.5443721942603588\n",
            "saving model\n",
            "Epoch:  6 , Batch:  3599 , loss avg over log interval:  0.5161040854454041\n",
            "Epoch:  6 , Batch:  3699 , loss avg over log interval:  0.5322381862998009\n",
            "saving model\n",
            "Epoch:  6 , Batch:  3799 , loss avg over log interval:  0.5264838525652885\n",
            "Epoch:  6 , Batch:  3899 , loss avg over log interval:  0.5130518786609173\n",
            "Epoch:  6 , Batch:  3999 , loss avg over log interval:  0.5285137134790421\n",
            "saving model\n",
            "Epoch:  6 , Batch:  4099 , loss avg over log interval:  0.541938344091177\n",
            "Epoch:  6 , Batch:  4199 , loss avg over log interval:  0.5108763480186462\n",
            "saving model\n",
            "Epoch:  6 , Batch:  4299 , loss avg over log interval:  0.5295326654613018\n",
            "Epoch:  6 , Batch:  4399 , loss avg over log interval:  0.5089798155426979\n",
            "Epoch:  6 , Batch:  4499 , loss avg over log interval:  0.5151952318847179\n",
            "saving model\n",
            "Epoch:  6 , Batch:  4599 , loss avg over log interval:  0.5378127844631672\n",
            "Epoch:  6 , Batch:  4699 , loss avg over log interval:  0.515813267827034\n",
            "saving model\n",
            "Epoch:  6 , Batch:  4799 , loss avg over log interval:  0.5271941438317299\n",
            "Epoch:  6 , Batch:  4899 , loss avg over log interval:  0.5041253270208835\n",
            "Epoch:  6 , Batch:  4999 , loss avg over log interval:  0.5211884860694408\n",
            "saving model\n",
            "Epoch:  6 , Batch:  5099 , loss avg over log interval:  0.5128688445687294\n",
            "Epoch:  6 , Batch:  5199 , loss avg over log interval:  0.5327454505860806\n",
            "saving model\n",
            "Epoch:  6 , Batch:  5299 , loss avg over log interval:  0.5341070199012756\n",
            "Epoch:  6 , Batch:  5399 , loss avg over log interval:  0.48798280492424967\n",
            "Epoch:  6 , Batch:  5499 , loss avg over log interval:  0.5604672032594681\n",
            "saving model\n",
            "Epoch:  6 , Batch:  5599 , loss avg over log interval:  0.5384932520985604\n",
            "Epoch:  6 , Batch:  5699 , loss avg over log interval:  0.5385087260603905\n",
            "saving model\n",
            "Epoch:  6 , Batch:  5799 , loss avg over log interval:  0.5037811557948589\n",
            "Epoch:  6 , Batch:  5899 , loss avg over log interval:  0.5273500311374665\n",
            "Epoch:  6 , Batch:  5999 , loss avg over log interval:  0.5300418403744698\n",
            "saving model\n",
            "Epoch:  6 , Batch:  6099 , loss avg over log interval:  0.5222949405014515\n",
            "Epoch:  6 , Batch:  6199 , loss avg over log interval:  0.517743611484766\n",
            "saving model\n",
            "Epoch:  6 , Batch:  6299 , loss avg over log interval:  0.5087929567694665\n",
            "Epoch:  6 , Batch:  6399 , loss avg over log interval:  0.5186145533621311\n",
            "Epoch:  6 , Batch:  6499 , loss avg over log interval:  0.5094023522734642\n",
            "saving model\n",
            "Epoch:  6 , Batch:  6599 , loss avg over log interval:  0.5112880331277847\n",
            "Epoch:  6 , Batch:  6699 , loss avg over log interval:  0.49858511105179787\n",
            "saving model\n",
            "Epoch:  6 , Batch:  6799 , loss avg over log interval:  0.49207765340805054\n",
            "Epoch:  6 , Batch:  6899 , loss avg over log interval:  0.5350137519836425\n",
            "Epoch:  6 , Batch:  6999 , loss avg over log interval:  0.5071508195996285\n",
            "saving model\n",
            "Epoch:  6 , Batch:  7099 , loss avg over log interval:  0.552610220760107\n",
            "Epoch:  6 , Batch:  7199 , loss avg over log interval:  0.510702136605978\n",
            "saving model\n",
            "Epoch:  6 , Batch:  7299 , loss avg over log interval:  0.5160849940776825\n",
            "Epoch:  6 , Batch:  7399 , loss avg over log interval:  0.5371840766072273\n",
            "Epoch:  6 , Batch:  7499 , loss avg over log interval:  0.5450641034543514\n",
            "saving model\n",
            "Epoch:  6 , avg total loss:  0.526764887583792\n",
            "iter   0\n",
            "accuracy so far =  0.859375\n",
            "iter   100\n",
            "accuracy so far =  0.468904702970297\n",
            "iter   200\n",
            "accuracy so far =  0.46260883084577115\n",
            "iter   300\n",
            "accuracy so far =  0.520141196013289\n",
            "iter   400\n",
            "accuracy so far =  0.49610349127182046\n",
            "iter   500\n",
            "accuracy so far =  0.5221744011976048\n",
            "iter   600\n",
            "accuracy so far =  0.49701019134775376\n",
            "iter   700\n",
            "accuracy so far =  0.4839960770328103\n",
            "iter   800\n",
            "accuracy so far =  0.4747191011235955\n",
            "iter   900\n",
            "accuracy so far =  0.4742647058823529\n",
            "iter   1000\n",
            "accuracy so far =  0.498001998001998\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SRUAFpyTZBNA"
      },
      "source": [
        "# Fuse by Averaging"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Gux6kQStZHqo",
        "outputId": "576828ef-01bb-49b4-eab4-5b9aa4e3048a"
      },
      "source": [
        "amdef test_average():\n",
        "    spatial.eval()\n",
        "    temporal.eval()\n",
        "    correct = 0\n",
        "\n",
        "    with torch.no_grad():\n",
        "        for iter, data in enumerate(test_loader, 0):\n",
        "            videos = data[\"videos\"]\n",
        "            labels = torch.tensor(data[\"labels\"])\n",
        "            indexes = data[\"indexes\"]\n",
        "            \n",
        "            SG3I = getSG3I(videos)\n",
        "            SG3I = SG3I.permute(0,3,1,2)\n",
        "            SG3I = SG3I.to(device)\n",
        "\n",
        "            videos = videos.type(torch.FloatTensor)\n",
        "            videos = videos.to(device)\n",
        "\n",
        "            # spatial\n",
        "            spatial_input = videos[:,0,:,:].permute(0,3,2,1)\n",
        "            spatial_output = spatial(spatial_input)\n",
        "\n",
        "            # temporal        \n",
        "            temporal_output = temporal(SG3I) \n",
        "\n",
        "            # average\n",
        "            averaged_output = (spatial_output + temporal_output) / 2\n",
        "            averaged_output = averaged_output.to(device)\n",
        "            labels = labels.to(device)\n",
        "\n",
        "            # calculate number of correct predictions in batch for accuracy\n",
        "            correct += sum(torch.argmax(averaged_output,1) == labels).item()\n",
        "            if iter % 100 == 0:\n",
        "                print (\"iter  \", iter)\n",
        "                print(\"accuracy so far = \", correct / ((iter + 1) * len(labels)))\n",
        "\n",
        "    # Log\n",
        "    accuracy = correct / len(test_loader.dataset)\n",
        "    print('Accuracy: ', accuracy)\n",
        "\n",
        "# Device\n",
        "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
        "torch.random.manual_seed(1)\n",
        "\n",
        "# load trained spatial\n",
        "spatial = SpatialStream()\n",
        "spatial.to(device)\n",
        "spatial_network_path = '/content/drive/Shared drives/CIS680 Final Project/models/spatial/spatial_epoch10'\n",
        "checkpoint_spatial = torch.load(spatial_network_path)\n",
        "spatial.load_state_dict(checkpoint_spatial['model_state_dict'])\n",
        "\n",
        "# load trained temporal\n",
        "temporal = TemporalStream()\n",
        "temporal.to(device)\n",
        "temporal_network_path = '/content/drive/Shared drives/CIS680 Final Project/models/temporal_SG3I/temporal_epoch14'\n",
        "checkpoint_temporal = torch.load(temporal_network_path)\n",
        "temporal.load_state_dict(checkpoint_temporal['model_state_dict'])\n",
        "\n",
        "test_average()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torchvision/io/video.py:117: UserWarning: The pts_unit 'pts' gives wrong results and will be removed in a follow-up version. Please use pts_unit 'sec'.\n",
            "  + \"follow-up version. Please use pts_unit 'sec'.\"\n",
            "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/container.py:117: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
            "  input = module(input)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "iter   0\n",
            "accuracy so far =  0.953125\n",
            "iter   100\n",
            "accuracy so far =  0.5430074257425742\n",
            "iter   200\n",
            "accuracy so far =  0.4724813432835821\n",
            "iter   300\n",
            "accuracy so far =  0.5403862126245847\n",
            "iter   400\n",
            "accuracy so far =  0.538692331670823\n",
            "iter   500\n",
            "accuracy so far =  0.5510541417165669\n",
            "iter   600\n",
            "accuracy so far =  0.5205386855241264\n",
            "iter   700\n",
            "accuracy so far =  0.4791592368045649\n",
            "iter   800\n",
            "accuracy so far =  0.4636001872659176\n",
            "iter   900\n",
            "accuracy so far =  0.4543042452830189\n",
            "iter   1000\n",
            "accuracy so far =  0.4825643106893107\n",
            "iter   1100\n",
            "accuracy so far =  0.4816643960036331\n",
            "iter   1200\n",
            "accuracy so far =  0.5004553497085762\n",
            "iter   1300\n",
            "accuracy so far =  0.5149284204458109\n",
            "iter   1400\n",
            "accuracy so far =  0.5268112062812277\n",
            "iter   1500\n",
            "accuracy so far =  0.5203822451698867\n",
            "iter   1600\n",
            "accuracy so far =  0.5392723297938788\n",
            "iter   1700\n",
            "accuracy so far =  0.5268132716049383\n",
            "iter   1800\n",
            "accuracy so far =  0.5222098833981121\n",
            "iter   1900\n",
            "accuracy so far =  0.5115153208837454\n",
            "iter   2000\n",
            "accuracy so far =  0.515523488255872\n",
            "iter   2100\n",
            "accuracy so far =  0.5102778438838649\n",
            "iter   2200\n",
            "accuracy so far =  0.5071487392094502\n",
            "Accuracy:  0.5057662492124061\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fy2dqXCRNr7g"
      },
      "source": [
        "# Plotting"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 803
        },
        "id": "pdoRNQIsH3QV",
        "outputId": "7b789010-cf5a-46dd-ad32-53a12da40bdc"
      },
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "EPOCH_SAVE_PREFIX = '/content/drive/Shared drives/CIS680 Final Project/models/fused_full_adam/'\n",
        "\n",
        "# Device\n",
        "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
        "torch.random.manual_seed(1)\n",
        "\n",
        "# Load params\n",
        "last_epoch = 5\n",
        "\n",
        "# load data\n",
        "network_path = EPOCH_SAVE_PREFIX + 'fused_epoch' + str(last_epoch)\n",
        "checkpoint = torch.load(network_path)\n",
        "train_loss_list = checkpoint['train_total_loss_list']\n",
        "epoch_loss_list = checkpoint['epoch_total_loss_list']\n",
        "train_counter = checkpoint['train_counter']\n",
        "test_loss_list = checkpoint['test_loss_list']\n",
        "accuracy_list = checkpoint['accuracy_list']\n",
        "epoch_list = np.arange(last_epoch+1)\n",
        "\n",
        "# plots\n",
        "fig = plt.figure()\n",
        "plt.plot(epoch_loss_list, color='blue')\n",
        "plt.legend(['FuseNet Train Loss'], loc='upper right')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Total Loss')\n",
        "\n",
        "fig = plt.figure()\n",
        "plt.plot(test_loss_list, color='green')\n",
        "plt.legend(['FuseNet Validation Loss'], loc='upper right')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Total Loss')\n",
        "\n",
        "fig = plt.figure()\n",
        "plt.plot(accuracy_list, color='red')\n",
        "plt.legend(['FuseNet Validation Accuracy'], loc='lower right')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Accuracy')\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxV1bn/8c8DQUDmqUoBDVbtr5iEAAkOWMUJHLG32morerVecaKAYgvWCoixtZWKRbFolaq1F+t1qm2togjiUAsJRQSxai0WKMogooggw/P7Y52QwQwnw87OOef7fr32Kyf77LPPcxzyPWuvvdYyd0dERDJXi7gLEBGReCkIREQynIJARCTDKQhERDKcgkBEJMNlxV1AXXXv3t2zs7PjLkNEJKWUlJRsdPceVT2XckGQnZ1NcXFx3GWIiKQUM3uvuud0aUhEJMMpCEREMpyCQEQkw6VcH4GI1Gznzp2sWbOG7du3x12KxKBNmzb07t2bVq1aJf0aBYFImlmzZg0dOnQgOzsbM4u7HGlC7s6mTZtYs2YNffv2Tfp1ujQkkma2b99Ot27dFAIZyMzo1q1bnVuDCgKRNKQQyFz1+XefMUHwz3/CuHGwc2fclYiINC8ZEwQrV8Ivfwn33Rd3JSLpr2XLluTn5+/dVq1a1WjnNjPGjx+/9/dp06YxZcqUGl+zYMECXnnllS/s/81vfrO3xn322Yfc3Fzy8/OZOHFiUrVMmjSJ5557LunaFyxYwOmnn5708U0lss5iM2sDLARaJ97nEXefXOmY1sADwCBgE3COu6+Kop7TToMjjoCpU+H886FNmyjeRUQA2rZty9KlSyM5d+vWrXnssce49tpr6d69e1KvWbBgAe3bt+eoo46qsP+iiy7ioosuAsKsBfPnz//COXfv3k3Lli2rPO/UqVPr8QmanyhbBDuA4929P5APnGxmR1Q65mJgs7sfDEwHfhZVMWZQVARr1sDdd0f1LiJSnezsbDZu3AhAcXExQ4cOBeCFF17Y+618wIABfPLJJwDccsstFBYWkpeXx+TJZd8hs7KyGDVqFNOnT//Ce2zYsIGzzjqLwsJCCgsLefnll1m1ahWzZs1i+vTp5Ofn8+KLL9Zaa/v27Rk/fjz9+/fnr3/9K1OnTqWwsJCcnBxGjRpF6cqOF154IY888sjezzd58mQGDhxIbm4ub775ZtL/bObMmUNubi45OTlMmDABCAF04YUXkpOTQ25u7t7PO2PGDPr160deXh7nnntu0u9Rk8haBB7+SW1N/NoqsVVeF/NMYEri8SPAHWZmHtH6mSecAMcdBzfdBBdfDO3aRfEuIs3HuHHQ2F/M8/PhtttqPuazzz4jPz8fgL59+/L4449Xe+y0adOYOXMmQ4YMYevWrbRp04a5c+fy9ttvs2jRItydESNGsHDhQo455hgArrzySvLy8vjhD39Y4Vxjx47lqquu4uijj+bf//43w4cPZ+XKlVx22WW0b9+ea665JqnP+Omnn3L44Yfzi1/8AoB+/foxadIkAM4//3z+9Kc/ccYZZ3zhdd27d2fJkiXceeedTJs2jXvuuafW9/rPf/7DhAkTKCkpoUuXLgwbNownnniCPn36sHbtWpYvXw7ARx99BMDNN9/Mv/71L1q3br13X0NF2kdgZi3NbCmwHnjW3f9W6ZBewGoAd98FbAG6VXGeUWZWbGbFGzZsaFBNRUWwfj3cfnuDTiMiNSi9NLR06dIaQwBgyJAhXH311cyYMYOPPvqIrKws5s6dy9y5cxkwYAADBw7kzTff5O233977mo4dO3LBBRcwY8aMCud67rnnGD16NPn5+YwYMYKPP/6YrVu3Vn7LWrVs2ZKzzjpr7+/z58/n8MMPJzc3l+eff54VK1ZU+bpvfvObAAwaNCjpfpHFixczdOhQevToQVZWFueddx4LFy7koIMO4t133+X73/8+Tz/9NB07dgQgLy+P8847jwcffJCsrMb5Lh/pgDJ33w3km1ln4HEzy3H35fU4z93A3QAFBQUNai0cdRSceir8/Odw+eXQqVNDzibSvNX2zb0pZWVlsWfPHoAK97lPnDiR0047jaeeeoohQ4bwzDPP4O5ce+21XHrppdWeb9y4cQwcOHDvNX6APXv28Oqrr9KmgZ2Abdq02dsvsH37dq644gqKi4vp06cPU6ZMqfY+/datWwMhSHbt2tWgGrp06cJrr73GM888w6xZs3j44YeZPXs2f/7zn1m4cCF//OMfuemmm3j99dcbHAhNcteQu38EzAdOrvTUWqAPgJllAZ0IncaRKiqCzZvh1lujficRKZWdnU1JSQkAjz766N79//znP8nNzWXChAkUFhby5ptvMnz4cGbPnr332/zatWtZv359hfN17dqVb3/729x777179w0bNozbyzX3SzusO3TosLfvoa5K/+h3796drVu37u0TaCyDBw/mhRdeYOPGjezevZs5c+Zw7LHHsnHjRvbs2cNZZ51FUVERS5YsYc+ePaxevZrjjjuOn/3sZ2zZsqVeLZ7KIgsCM+uRaAlgZm2Bk4DKvSdPAv+deHw28HxU/QPlDRgAZ58dgiDRdyUiEZs8eTJjx46loKCgwl04t912Gzk5OeTl5dGqVStOOeUUhg0bxne/+12OPPJIcnNzOfvss6v8Qz5+/Pi9HdAQOlKLi4vJy8ujX79+zJo1C4AzzjiDxx9/POnO4vI6d+7MJZdcQk5ODsOHD6ewsLCe/wSCefPm0bt3773bqlWruPnmmznuuOPo378/gwYN4swzz2Tt2rUMHTqU/Px8Ro4cyU9/+lN2797NyJEjyc3NZcCAAYwZM4bOnTs3qB4Ai+rvrpnlAfcDLQmB87C7TzWzqUCxuz+ZuMX0t8AA4EPgXHd/t6bzFhQUeGMsTPPGG5CTA9dcEy4TiaSLlStX8rWvfS3uMiRGVf03YGYl7l5Q1fFR3jW0jPAHvvL+SeUebwe+FVUNNenXD0aOhDvugKuugp4946hCRCR+GTOyuCpTpoQpJ266Ke5KRETik9FBcNBBYTzB3XdDI46AF4ldE3S1STNVn3/3GR0EAD/+MbRoATfeGHclIo2jTZs2bNq0SWGQgUrXI6jr7bMZvzBN795hPMHtt8OECXDooXFXJNIwvXv3Zs2aNTR08KWkptIVyuoisruGotJYdw2Vt3499O0LI0bAnDmNemoRkWahpruGMv7SEMCXvgRjx8JDD8GyZXFXIyLStBQECT/4QZhuYtKk2o8VEUknCoKELl3C4LI//AEWLYq7GhGRpqMgKGfsWOjeHa6/Pu5KRESajoKgnA4dYOJEmDsXFi6MuxoRkaahIKjkiivgy1+G666DFLuhSkSkXhQElbRtGwaZvfQSPPNM3NWIiERPQVCFiy+G7OwQCGoViEi6UxBUYZ99YPJkKCmBJ56IuxoRkWgpCKoxciR89avhDqLdu+OuRkQkOgqCamRlwQ03wIoV8Pvfx12NiEh0FAQ1+Na3oH//cJlo5864qxERiYaCoAal01O/8w7cf3/c1YiIRENBUIvTT4fBg2HqVNixI+5qREQan4KgFmZhKcvVq8NKZiIi6UZBkIQTToChQ0MgfPpp3NWIiDQuBUESzKCoCD74AO64I+5qREQal4IgSUOGwCmnwM9+Blu2xF2NiEjjURDUQVERbN4M06fHXYmISONRENTBwIFw1llw662waVPc1YiINA4FQR3dcANs3Qo//3nclYiINA4FQR0ddhicdx7cfjusWxd3NSIiDacgqIcpU+Dzz+EnP4m7EhGRhlMQ1MNXvgLf+x7cdRe8917c1YiINIyCoJ6uvz6ML7jxxrgrERFpmMiCwMz6mNl8M3vDzFaY2dgqjhlqZlvMbGlimxRVPY2tTx+4/HK47z54++24qxERqb8oWwS7gPHu3g84ArjSzPpVcdyL7p6f2KZGWE+ju/ZaaN06TFMtIpKqIgsCd1/n7ksSjz8BVgK9onq/OOy3H4wZAw89BK+/Hnc1IiL10yR9BGaWDQwA/lbF00ea2Wtm9hczO6ya148ys2IzK96wYUOEldbdD34AHTrApJS5qCUiUlHkQWBm7YFHgXHu/nGlp5cAB7p7f+B2oMql4t39bncvcPeCHj16RFtwHXXtCtdcExa5X7w47mpEROou0iAws1aEEPiduz9W+Xl3/9jdtyYePwW0MrPuUdYUhbFjoVu3cCeRiEiqifKuIQPuBVa6+63VHLN/4jjMbHCinpSbxadjR5g4EZ55Bl58Me5qRETqJsoWwRDgfOD4creHnmpml5nZZYljzgaWm9lrwAzgXHf3CGuKzBVXQM+ecN11kJqfQEQyVVZUJ3b3lwCr5Zg7gLRY6mXffUMIjB4Nzz4Lw4bFXZGISHI0srgRXXIJHHigWgUikloUBI1on33C4LLiYvjDH+KuRkQkOQqCRnb++XDooeEOot27465GRKR2CoJGlpUVFq9ZvhwefjjuakREaqcgiMC3vw15eeEy0a5dcVcjIlIzBUEEWrQI01O//Tbcf3/c1YiI1ExBEJEzzoDBg2HqVNixI+5qRESqpyCIiBkUFcG//w2//nXc1YiIVE9BEKETT4Rjj4WbboJt2+KuRkSkagqCCJW2Ct5/H2bOjLsaEZGqKQgidvTRcPLJcPPN8HHlSbhFRJoBBUETKCqCDz+E6dPjrkRE5IsUBE1g0CD45jfhF7+ATSk3ybaIpDsFQROZOhW2boVbbom7EhGRihQETeSww+C734UZM0LnsYhIc6EgaEJTpsDnn8NPfhJ3JSIiZRQETejgg+Gii+Cuu8JAMxGR5kBB0MRKF7i/8cZ46xARKaUgaGIHHACXXQa/+U2YlE5EJG4Kghhce21YzeyGG+KuREREQRCL/feHMWPgf/83LGAjIhInBUFMfvhD6NABJk2KuxIRyXQKgph07Qrjx8Pjj4fF7kVE4qIgiNG4cdCtW9mdRCIicVAQxKhjR5gwAZ5+Gl56Ke5qRCRTKQhiduWVofP4uuvAPe5qRCQTKQhitu++IQQWLoTnnou7GhHJRAqCZuCSS8JAM7UKRCQOtQaBmQ0xs3aJxyPN7FYzOzD60jJH69YweTIsXgxPPhl3NSKSaZJpEfwK2GZm/YHxwD+BByKtKgNdcAEccki4g2jPnrirEZFMkkwQ7HJ3B84E7nD3mUCHaMvKPFlZYcqJ11+Hhx+OuxoRySTJBMEnZnYtMBL4s5m1AFrV9iIz62Nm883sDTNbYWZjqzjGzGyGmb1jZsvMbGDdP0L6OOccyM0Nl4l27Yq7GhHJFMkEwTnADuBid38f6A0ks+DiLmC8u/cDjgCuNLN+lY45BTgksY0iXIbKWC1ahOmp33oLHtDFNxFpIkm1CIBfuvuLZnYokA/Mqe1F7r7O3ZckHn8CrAR6VTrsTOABD14FOptZzzp9gjQzYgQUFobLRDt2xF2NiGSCZIJgIdDazHoBc4Hzgfvq8iZmlg0MAP5W6alewOpyv6/hi2GBmY0ys2IzK96wYUNd3jrlmEFRUVjB7J574q5GRDJBMkFg7r4N+CZwp7t/C8hJ9g3MrD3wKDDO3T+uT5Hufre7F7h7QY8ePepzipRy0klwzDEhELZti7saEUl3SQWBmR0JnAf8uQ6vw8xaEULgd+7+WBWHrAX6lPu9d2JfRittFbz/Ptx5Z9zViEi6S+YP+jjgWuBxd19hZgcB82t7kZkZcC+w0t1vreawJ4ELEncPHQFscfd1Sdae1r7+dRg+HG6+GT6uVztKRCQ5tQaBu7/g7iOAmWbW3t3fdfcxSZx7CKE/4XgzW5rYTjWzy8zsssQxTwHvAu8AvwauqOfnSEtFRbBpE9x2W9yViEg6M69lchszyyWMJO4KGLABuMDdV0Rf3hcVFBR4cQat5PJf/wXPPw//+ldYzEZEpD7MrMTdC6p6LplLQ3cBV7v7ge5+AGGaiV83ZoFSvRtvhE8+gVuSGbkhIlIPyQRBO3ff2yfg7guAdpFVJBXk5MB3vgMzZoTOYxGRxpZMELxrZtebWXZi+zHhur40kSlTwuCyn/407kpEJB0lEwTfA3oAjxFuBe0OXBRlUVLRIYfAhRfCrFmwenWth4uI1Ekydw1tdvcx7j7Q3Qe5+zhCv4E0oUmTws8bb4y3DhFJP/VdoezIRq1CanXAAXDppTB7NrzzTtzViEg60VKVKeRHP4J99gkT0omINJas6p6oYW0AI4n1CKTx7b8/fP/74VbSiRPhsMPirkhE0kG1A8rMrMZpJNz9uEgqqkWmDSirbNMm6Ns3TEz36KNxVyMiqaKmAWXVtgji+kMvNevWDa6+OlweKimBQYPirkhEUp36CFLQ1VeH6Sauvz7uSkQkHSgIUlDHjjBhAvzlL/Dyy3FXIyKpTkGQokaPhv32g+uug1rmDRQRqVF97hoCoHQ9YonHvvuGEBgzBubNgxNPjLsiEUlV9b1ryN39+GhKqlmm3zVU3o4dcOih4bbSV18NK5uJiFRFdw2lqdatw9QT//M/8Mc/wogRcVckIqmo1oVpAMwsB+gHtCnd5+4PRFhXtdQiqGjnTujXL1wq+vvfoYV6fUSkCg1amMbMJgO3J7bjgJ8D+u7ZTLRqFcYULFsG//d/cVcjIqkome+PZwMnAO+7+0VAf6BTpFVJnZx7bljAZtIk2LUr7mpEJNUkEwSfufseYJeZdQTWA32iLUvqokULmDoV3noLfvvbuKsRkVSTTBAUm1lnwjrFJcAS4K+RViV19o1vQEFBuEz0+edxVyMiqSSZhWmucPeP3H0WcBLw34lLRNKMmEFREbz3HtxzT9zViEgqSaazeF7pY3df5e7Lyu+T5mPYMDj66BAI27bFXY2IpIpqg8DM2phZV6C7mXUxs66JLRvo1VQFSvLM4KabYN06+NWv4q5GRFJFTS2CSwl9Av+P0C9Qktj+ANwRfWlSH8ccE1oGN98Mn3wSdzUikgqqDQJ3/6W79wWucfe+5bb+7q4gaMaKimDjRrjttrgrEZFUkMxdQ3eZ2RgzeySxjTYzLVXZjBUWwplnwrRp8OGHcVcjIs1dMkFwJzAo8bP0sa5AN3M33hguDU2bFnclItLc1TQNdZa77wIK3b1/uaeeN7PXoi9NGiI3N4w4nj4d8vLCYxGRqtTUIliU+LnbzL5SutPMDgJ2R1qVNIrp08Oaxt/5Tli3QAPNRKQqNQVB6ez21wDzzWyBmS0AngfG13ZiM5ttZuvNbHk1zw81sy1mtjSxTapr8VKz/faD+fPhqqvg9tvDHUWrV8ddlYg0NzUFQQ8zuxrIB+4iBMDzhKkmBiRx7vuAk2s55kV3z09sU5M4p9RRq1Zw661hZtI33oABA2Du3LirEpHmpKYgaAm0BzoQ+hIssWUl9tXI3RcCumelmTj7bCguhp494eSTwyR1e/bEXZWINAfVdhYD65rgW/qRiY7n/xDGK6yI+P0y2qGHhiUtL78cJk+GV16BBx+E7t3jrkxE4pRMH0FUlgAHJu5Iuh14otpCzEaZWbGZFW/YsCHistJbu3Zw//1w112h/2DgQFi0qPbXiUj6qikITojyjd39Y3ffmnj8FNDKzKr8burud7t7gbsX9OjRI8qyMoIZjBoVWgQtW4aJ6mbOhCRWLRWRNFTTFBORXt83s/3NzBKPBydq2RTle0pFgwZBSUmYm2j0aDjvPNi6Ne6qRKSpRbbUuZnNISxg81UzW2NmF5vZZWZ2WeKQs4HliT6CGcC57vpO2tS6doUnnwyzlv7+9zB4MKxcGXdVItKULNX+9hYUFHhxcXHcZaSlefPC4LNt2+Dee+Gcc+KuSEQai5mVuHtBVc9F1iKQ1HPCCfD3v0N+fpiSQqORRTKDgkAq6NVLo5FFMo2CQL5Ao5FFMouCQKql0cgimUFBIDUqHY08cmQYjXzqqWH1MxFJHwoCqZVGI4ukNwWBJKV0NPLLL0OLFhqNLJJOFARSJwUFsGSJRiOLpBMFgdRZ5dHIhx+u0cgiqUxBIPXSogX86EfhttING6CwMISCiKQeBYE0SOlo5P79NRpZJFUpCKTBevWCBQs0GlkkVSkIpFFoNLJI6lIQSKPSaGSR1KMgkEan0cgiqUVBIJHQaGSR1KEgkMhoNLJIalAQSOQ0GlmkeVMQSJPQaGSR5ktBIE1Go5FFmicFgTQ5jUYWaV4UBBKLyqORjz1Wo5FF4qIgkNiUH428YkW4xfTZZ+OuSiTzKAgkdqWjkfffH4YP12hkkaamIJBmQaORReKjIJBmQ6ORReKhIJBmparRyHfeqdHIIlFSEEizVH408pVXhktGGo0sEg0FgTRb5UcjP/SQRiOLREVBIM2aRiOLRE9BIClBo5FFohNZEJjZbDNbb2bLq3nezGyGmb1jZsvMbGBUtUh60GhkkWhE2SK4Dzi5hudPAQ5JbKOAX0VYi6QJjUYWaXyRBYG7LwQ+rOGQM4EHPHgV6GxmPaOqR9KLRiOLNJ44+wh6AeUb9msS+77AzEaZWbGZFW/YsKFJipPmT6ORRRpHSnQWu/vd7l7g7gU9evSIuxxpRkpHI8+aVTYaed482Lkz7spEUkecQbAW6FPu996JfSJ1YgaXXlo2GvnEE6FjRzjyyHB30YMPwj/+oUtHItXJivG9nwRGm9lDwOHAFndfF2M9kuIKCmDZMnj6aVi8OMxTNHt2uMMIoFOncExhIQweHH726hWCRCSTmUc0iYuZzQGGAt2BD4DJQCsAd59lZgbcQbizaBtwkbsX13begoICLy6u9TARAHbvDqORS4Nh8eIQFqWXjnr2rBgMBQVhRLNIujGzEncvqPK5qIIgKgoCaajt2+G11yqGw5tvlj1/8MEVw2HAANh33/jqFWkMNQVBnJeGRGLRpk2Yt+jww8v2bdkCJSVlwfDiizBnTniuZUvIyakYDjk5kKX/eyRNqEUgUo3336/Yali0CDZvDs+1bRtaCuXD4eCD1d8gzZcuDYk0And4992yYFi8OLQiPvssPN+lS+hjKA2GwkL48pfjrVmklIJAJCK7dsEbb1RsNbz+euikhnBXUuXO6M6d461ZMpOCQKQJbdsGS5dWvKz09ttlzx96aMVwyM8Pl5pEoqTOYpEmtO++cNRRYSu1eXOYG6k0HJ5/Hn73u/BcVhbk5la8pNSvnzqjpemoRSASk7VrK7YaFi8Ody9BCJOBAyuGw0EHqTNa6k8tApFmqFevsH3jG+H3PXvgnXcqhsPMmbBjR3i+a9eKl5QKC8PsqyINpRaBSDO2cycsX16x1bB8edm8SX36hEAYMgROOimMb1CrQaqizmKRNPLpp2HZzvJ3Kr37bniuZ88w6d6wYeGnWgxSSpeGRNJIu3Zw9NFhK7V6dVipbe5ceOop+O1vw/68vBAKJ50EX/+67k6SqqlFIJJm9uwJLYbSYHjppXCJqXVrOOaYEArDhoU7lVqkxIok0hh0aUgkg336KSxcGELh2WfDWs8AX/pSWSiceKJGQac7BYGI7LV2LTz3XFkwlK7+mpNTFgzHHKMZV9ONgkBEqrRnT1ifoTQUXnwx3K66zz6hD6K0fyE/X5eRUp2CQESSsm1b6FOYOzdsr78e9nfvXnY30kknQe/e8dYpdacgEJF6WbcuXEYq7Xj+4IOw/2tfKwuFY4+F9u3jrVNqpyAQkQZzD4PZSlsLCxeG1d5atSob0DZsWFinoWXLuKuVyhQEItLotm8Pl5FKWwtLl4b9XbtWvIx0wAHx1imBgkBEIvfBBzBvXlnH83/+E/Z/9atlrYWhQ6FDh1jLzFgKAhFpUu5hwZ7S1sKCBWElt6wsOPLIstZCQYEuIzUVBYGIxGrHDnjllbLWwpIlISy6dIHjjw/BMGwYZGfHXWn6UhCISLOycWPZZaS5c2HNmrD/4IPLWgvHHQedOsVbZzpREIhIs+UO//hHWWth/vwwLUbLlnDEEWX9C4WFWrWtIRQEIpIyPv8cXn21rLVQXBzColOnsstIJ50EX/lK3JWmFgWBiKSsTZvCGs+lHc/vvRf2t20bblXt0qVuPzt1yswOagWBiKQF97Cc57PPhsV4Nm+GDz/84s9t26o/h1kIg7oGSJcuYS2IVF0BTgvTiEhaMINDDglbTXbsCKFQXVBU/rl6ddnvu3ZVf95WreoXIF26hIn8misFgYikndatwzKddV2q0x22bk0+QNatC+s7bN4MW7bUfO527ep3Katjx+hnflUQiIgkmIWRzx061H1qjF27QhgkEyCbN8Nbb5X9vn179edt0QI6dw7BcPnlMH58wz5jVRQEIiKNICsLunULW1199llyl7Lq2sJJVqRBYGYnA78EWgL3uPvNlZ6/ELgFWJvYdYe73xNlTSIizU3btmGLa7nQyILAzFoCM4GTgDXAYjN70t3fqHTo7919dFR1iIhIzaLsghgMvOPu77r758BDwJkRvp+IiNRDlEHQC1hd7vc1iX2VnWVmy8zsETPrU9WJzGyUmRWbWfGG0pW2RUSkUcS9HPUfgWx3zwOeBe6v6iB3v9vdC9y9oEePHk1aoIhIuosyCNYC5b/h96asUxgAd9/k7jsSv94DDIqwHhERqUKUQbAYOMTM+prZPsC5wJPlDzCznuV+HQGsjLAeERGpQmR3Dbn7LjMbDTxDuH10truvMLOpQLG7PwmMMbMRwC7gQ+DCqOoREZGqadI5EZEMkFazj5rZBuC9er68O7CxEctJBfrMmUGfOTM05DMf6O5V3m2TckHQEGZWXF0ipit95sygz5wZovrMcd8+KiIiMVMQiIhkuEwLgrvjLiAG+syZQZ85M0TymTOqj0BERL4o01oEIiJSiYJARCTDZUwQmNnJZvYPM3vHzCbGXU/UzGy2ma03s+Vx19JUzKyPmc03szfMbIWZjY27pqiZWRszW2RmryU+8w1x19QUzKylmf3dzP4Udy1NwcxWmdnrZrbUzBp9RG1G9BEkFsl5i3KL5ADfqWKRnLRhZscAW4EH3D0n7nqaQmLuqp7uvsTMOgAlwDfS/N+zAe3cfauZtQJeAsa6+6sxlxYpM7saKAA6uvvpcdcTNTNbBRS4eyQD6DKlRZBxi+S4+0LC/E0Zw93XufuSxONPCJMYVrUGRtrwYGvi14L3itcAAAM8SURBVFaJLa2/3ZlZb+A0wozF0ggyJQiSXSRH0oSZZQMDgL/FW0n0EpdJlgLrgWfdPd0/823AD4E9cRfShByYa2YlZjaqsU+eKUEgGcTM2gOPAuPc/eO464mau+9293zCmh+DzSxtLwWa2enAencvibuWJna0uw8ETgGuTFz6bTSZEgS1LpIj6SFxnfxR4Hfu/ljc9TQld/8ImA+cHHctERoCjEhcM38ION7MHoy3pOi5+9rEz/XA44TL3Y0mU4Kg1kVyJPUlOk7vBVa6+61x19MUzKyHmXVOPG5LuCHizXirio67X+vuvd09m/D/8fPuPjLmsiJlZu0SNz9gZu2AYUCj3g2YEUHg7ruA0kVyVgIPu/uKeKuKlpnNAf4KfNXM1pjZxXHX1ASGAOcTviUuTWynxl1UxHoC881sGeELz7PunhG3VGaQ/YCXzOw1YBHwZ3d/ujHfICNuHxURkeplRItARESqpyAQEclwCgIRkQynIBARyXAKAhGRDKcgEEkws93lbjtd2piz1JpZdibNBCupJSvuAkSakc8SUzWIZBS1CERqkZgL/ueJ+eAXmdnBif3ZZva8mS0zs3lmdkBi/35m9nhijYDXzOyoxKlamtmvE+sGzE2MBMbMxiTWUFhmZg/F9DElgykIRMq0rXRp6Jxyz21x91zgDsLslwC3A/e7ex7wO2BGYv8M4AV37w8MBEpHsR8CzHT3w4CPgLMS+ycCAxLnuSyqDydSHY0sFkkws63u3r6K/auA49393cSkdu+7ezcz20hYCGdnYv86d+9uZhuA3u6+o9w5sgnTPxyS+H0C0Mrdi8zsacIiQk8AT5RbX0CkSahFIJIcr+ZxXewo93g3ZX10pwEzCa2HxWamvjtpUgoCkeScU+7nXxOPXyHMgAlwHvBi4vE84HLYu2hMp+pOamYtgD7uPh+YAHQCvtAqEYmSvnmIlGmbWOmr1NPuXnoLaZfEDJ87gO8k9n0f+I2Z/QDYAFyU2D8WuDsx4+tuQiisq+Y9WwIPJsLCgBmJdQVEmoz6CERqEfXC4SJx06UhEZEMpxaBiEiGU4tARCTDKQhERDKcgkBEJMMpCEREMpyCQEQkw/1/PlDIbDfZS5UAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXgUZdr3/e9JEhLCTgioBAgIClkgCQFERBFmEAVFtsjibhp93eBGZ8S53Zjn1efWcURRVGRRxw0igqKAogJuo0jYhISwDCKbSkC2QEK28/mjm75DCFkgnUrS5+c4+kh31dXVZ7H0L1VX1XWJqmKMMcaUVx2nCzDGGFOzWHAYY4ypEAsOY4wxFWLBYYwxpkIsOIwxxlRIoNMFVIXmzZtrZGSk02UYY0yNsXr16v2qGl7SOr8IjsjISFJTU50uwxhjagwR+eVM6+xUlTHGmAqx4DDGGFMhFhzGGGMqxC/6OIypqfLy8ti9ezc5OTlOl2JqqZCQECIiIggKCir3eyw4jKnGdu/eTcOGDYmMjEREnC7H1DKqyoEDB9i9ezft2rUr9/vsVJUx1VhOTg5hYWEWGsYnRISwsLAKH9FacBhTzVloGF86m39fFhxnkJ2XzbP/fpblPy93uhRjjKlWLDjOICggiOe+f47nfnjO6VKMcVRAQABxcXHex44dOypt2yLCAw884H397LPP8sQTT5T6nhUrVvDvf//7tOU7duwgIiKCwsLCU5bHxcWxcuXKEre1Y8cOYmJiAEhNTeX+++8vsV1kZCT79+8vta6nnnrqlNeXXnppqe3L69Zbb2XevHmVsq3KYsFxBoF1Ark17lYWb13M7iO7nS7HGMfUq1ePdevWeR+VOXxPcHAw8+fPL/NLuagzBUdkZCRt2rThm2++8S7LyMjg6NGj9OzZs8ztJiYmMnXq1HLXUVzx4CipxtrCgqMUd8TfQaEW8vra150uxZhqpehv4KmpqfTt2xeAr776yntkEh8fz9GjRwH4xz/+Qffu3enSpQuPP/64dzuBgYGMGzeOKVOmnPYZmZmZDB8+nO7du9O9e3e+++47duzYwauvvsqUKVOIi4s7JSQARo8ezZw5c7yv58yZw6hRo9ixYwd9+vQhISGBhISEEr/UV6xYweDBgwE4cOAAAwYMIDo6muTkZIrOlHr99dfTrVs3oqOjee211wCYNGkS2dnZxMXFMXbsWAAaNGgAuK9c+stf/kJMTAyxsbHMnTvX+3l9+/ZlxIgRdOrUibFjx1LeGVlzcnK47bbbiI2NJT4+nuXL3afU09LS6NGjB3FxcXTp0oWtW7dy7NgxBg0aRNeuXYmJifF+/rmwy3FLcWGzC+nfrj+z1s7ivy//b+qI5axxzoRPJ7Dut3WVus248+J4fuDzpbY5+YUI0K5dOxYsWHDGts8++yzTpk2jd+/eZGVlERISwtKlS9m6dSs//vgjqsp1113H119/zeWXXw7APffcQ5cuXfjrX/96yrbGjx/Pf/3Xf3HZZZexc+dOrrrqKjZt2sRdd91FgwYNePDBB0/7/KSkJOLi4njxxRcJDAxk7ty5vP/++7Ro0YLPP/+ckJAQtm7dyujRo0sdv27y5MlcdtllPPbYYyxatIhZs2Z5182ePZtmzZqRnZ1N9+7dGT58OP/zP//DSy+9xLp1p//9zJ8/n3Xr1rF+/Xr2799P9+7dvfu+du1a0tLSuOCCC+jduzffffcdl112WSl/G27Tpk1DRNiwYQMZGRkMGDCALVu28OqrrzJ+/HjGjh1Lbm4uBQUFLF68mAsuuIBFixYBcPjw4TK3XxYLjjK4ElyM+mAUX2z/ggEXDnC6HGOq3MlTVeXRu3dvJk6cyNixYxk2bBgREREsXbqUpUuXEh8fD0BWVhZbt271fnk2atSIm2++malTp1KvXj3vtr744gvS09O9r48cOUJWVlapn9+yZUtiYmL48ssvadmyJYGBgcTExHD48GHuvfde1q1bR0BAAFu2bCl1O19//TXz588HYNCgQTRt2tS7burUqd7w3LVrF1u3biUsLOyM2/r2228ZPXo0AQEBtGzZkiuuuIJVq1bRqFEjevToQUREBIC3/6g8wfHtt99y3333AdCpUyfatm3Lli1b6NWrF08++SS7d+9m2LBhdOzYkdjYWB544AEeeughBg8eTJ8+fcrcflksOMpwfafrCasXxow1Myw4jKPKOjKoSoGBgd5O6KL3AEyaNIlBgwaxePFievfuzWeffYaq8vDDD3PnnXeecXsTJkwgISGB2267zbussLCQH374gZCQkArVdvJ0VcuWLRk9ejQAU6ZMoWXLlqxfv57CwsIKb/OkFStW8MUXX/D9998TGhpK3759z+mu/uDgYO/zgIAA8vPzz3pbAGPGjKFnz54sWrSIa665hunTp9OvXz/WrFnD4sWLeeSRR+jfvz+PPfbYOX2OnXspQ3BgMLd0vYWPMj5i37F9TpdjTLUQGRnJ6tWrAfjggw+8y//zn/8QGxvLQw89RPfu3cnIyOCqq65i9uzZ3qOFPXv2sG/fqf+XmjVrRlJS0imnhAYMGMCLL77ofX3yqKdhw4bevpOSDBs2jMWLFzN37lxGjRoFuE/PnH/++dSpU4e33nqLgoKCUvfv8ssv59133wVgyZIlHDx40Ludpk2bEhoaSkZGBj/88IP3PUFBQeTl5Z22rT59+jB37lwKCgrIzMzk66+/pkePHqV+fln69OnDO++8A8CWLVvYuXMnF198Mdu3b6d9+/bcf//9DBkyhJ9++om9e/cSGhrKjTfeyF/+8hfWrFlzTp8NFhzlkpyQTF5hHm+ue9PpUoypFh5//HHGjx9PYmIiAQEB3uXPP/88MTExdOnShaCgIK6++moGDBjAmDFj6NWrF7GxsYwYMaLEL/4HHnjglKurpk6dSmpqKl26dCEqKopXX30VgGuvvZYFCxaU2DkO0KRJE3r16kXLli1p3749AHfffTdvvvkmXbt2JSMjg/r165e5f19//TXR0dHMnz+fNm3aADBw4EDy8/Pp3LkzkyZN4pJLLvG+Z9y4cXTp0sXbOX7S0KFD6dKlC127dqVfv34888wznHfeeWX9EZ/izjvvJCIigoiICHr16sXdd99NYWEhsbGx3HDDDbzxxhsEBweTkpJCTEwMcXFxbNy4kZtvvpkNGzZ4O8wnT57MI488UqHPLomUtxe/JktMTNRzncjpstmXkXk8k4x7MuxOXlNlNm3aROfOnZ0uw9RyJf07E5HVqppYUns74ignV4KLLQe28M3O03/DMcYYf2LBUU4jo0fSOLgxM9bMcLoUY4xxlAVHOYUGhTI2dizz0udxMPug0+UYP+IPp5ONc87m35cFRwUkJySTk5/D2z+97XQpxk+EhIRw4MABCw/jEyfn46jo5cnWOV7Rbb2WSG5BLuvvWm+d5MbnbAZA42tnmgGwtM5xuwGwglwJLu5adBer9q6iR6tzuxbbmLIEBQVVaGY2Y6qCnaqqoNGxowkNCmXGauskN8b4JwuOCmoU3IhR0aN4b+N7HD1x5rtXjTGmtrLgOAuubi6O5R1jzsY5ZTc2xphaxoLjLPRs1ZPo8Gi7p8MY45csOM6CiOBKcLFq7yrW/7be6XKMMaZKWXCcpZu63kRwQDAz18x0uhRjjKlSPg0OERkoIptFZJuITCphfbCIzPWsXykikcXWtxGRLBF5sMiyJiIyT0QyRGSTiPTy5T6cSbN6zRgeNZy3N7xNdl62EyUYY4wjfBYcIhIATAOuBqKA0SISVazZHcBBVe0ATAGeLrb+OWBJsWUvAJ+qaiegK7CpsmsvL1eCi0M5h5iXPs+pEowxpsr58oijB7BNVberai4wBxhSrM0Q4OQkF/OA/uK5HVtErgd+BtJONhaRxsDlwCwAVc1V1UM+3IdSXdH2Cjo062Cd5MYYv+LL4GgF7CryerdnWYltVDUfOAyEiUgD4CFgcrH27YBM4HURWSsiM0WkxBlZRGSciKSKSGpmZua5703Jn0FyfDLf7PyGjP0ZPvkMY4ypbqpr5/gTwBRVLT4zfSCQALyiqvHAMeC0vhMAVX1NVRNVNTE8PNxnhd4adyuBdQKtk9wY4zd8GRx7gNZFXkd4lpXYRkQCgcbAAaAn8IyI7AAmAH8TkXtxH7XsVtWVnvfPwx0kjmnZoCXXXXwdb65/k9yCXCdLMcaYKuHL4FgFdBSRdiJSFxgFLCzWZiFwi+f5CGCZuvVR1UhVjQSeB55S1ZdU9Tdgl4hc7HlPfyDdh/tQLq4EF/uP7+ejjI+cLsUYY3zOZ8Hh6bO4F/gM95VPKaqaJiJ/F5HrPM1m4e7T2AZM5AynnYq5D3hHRH4C4oCnKr/6ivlz+z/TpnEb6yQ3xvgFm4+jkkxeMZknvnqC7fdvp11TGwbbGFOzlTYfR3XtHK9xbo+/nTpSh1lrZzldijHG+JQFRyVp3bg1AzsM5PV1r5NfmO90OcYY4zMWHJXIleBi79G9LNla/GZ3Y4ypPSw4KtGgjoM4r8F51klujKnVLDgqUVBAELfF3cairYvYc6T4LSvGGFM7WHBUsjvi76BQC3l93etOl2KMMT5hwVHJLmx2If3a9WPW2lkUaqHT5RhjTKWz4PABV4KLHYd28OX2L50uxRhjKp0Fhw8M7TSUsHph1klujKmVLDh8IDgwmJu73syHGR+Secw3Q7obY4xTLDh8JDkhmbzCPN5c/2bZjY0xpgax4PCRqPAoLm19KTPXzMQfxgMzxvgPCw4fciW42HxgM9/s/MbpUowxptJYcPjQyKiRNApuZLMDGmNqFQsOH6pftz5jY8fyfvr7HMw+6HQ5xhhTKSw4fMyV4CInP4d3NrzjdCnGGFMpLDh8LP78eBLOT2DGmhnWSW6MqRUsOKqAK8HFT7//xKq9q5wuxRhjzpkFRxUYEzuG0KBQ6yQ3xtQKFhxVoFFwI26IvoH3Nr5HVm6W0+UYY8w5seCoIq4EF1m5WczZOMfpUowx5pxYcFSRSyIuITo82gY+NMbUeBYcVURESE5I5sc9P/LT7z85XY4xxpw1C44qdFOXm6gbUJcZq+2owxhTc1lwVKGw0DCGdx7O2xveJjsv2+lyjDHmrFhwVDFXgotDOYf4YNMHTpdijDFnxYKjivWN7EuHZh2sk9wYU2P5NDhEZKCIbBaRbSIyqYT1wSIy17N+pYhEFlvfRkSyROTBYssDRGStiHziy/p9QURIjk/m61++ZvP+zU6XY4wxFeaz4BCRAGAacDUQBYwWkahize4ADqpqB2AK8HSx9c8BS0rY/HhgU+VWXHVuibuFwDqBdie5MaZG8uURRw9gm6puV9VcYA4wpFibIcDJuVXnAf1FRABE5HrgZyCt6BtEJAIYBNTYb93zGpzHtRddy5vr3yS3INfpcowxpkJ8GRytgF1FXu/2LCuxjarmA4eBMBFpADwETC5hu88DfwUKS/twERknIqkikpqZmXl2e+BDrgQXmcczWbh5odOlGGNMhVTXzvEngCmqesrATiIyGNinqqvL2oCqvqaqiaqaGB4e7qMyz96ACwfQpnEb6yQ3xtQ4gT7c9h6gdZHXEZ5lJbXZLSKBQGPgANATGCEizwBNgEIRycF9hHKdiFwDhACNRORtVb3Rh/vhEwF1Arg97nYmfzWZHYd2ENkk0umSjDGmXHx5xLEK6Cgi7USkLjAKKH5eZiFwi+f5CGCZuvVR1UhVjcR9auopVX1JVR9W1QjP8lGe9jUuNE66Lf42AGatmeVwJcYYU34+Cw5Pn8W9wGe4r4BKUdU0Efm7iFznaTYLd5/GNmAicNolu7VZm8ZtGNhhILPXzSa/MN/pcowxplzEH6YzTUxM1NTUVKfLKNGCTQsYljKMj0d/zOCLBjtdjjHGACAiq1U1saR11bVz3G8MvmgwLeu3tE5yY0yNYcHhsKCAIG6Lu41FWxax9+hep8sxxpgyWXBUA3ck3EGBFvD62tedLsUYY8pkwVENdGjWgSsjr2TW2lkUaqn3NRpjjOMsOKoJV4KLnw/9zJfbv3S6FGOMKZUFRzUxtPNQmtVrxsy1NXYILmOMn7DgqCZCAkO4ucvNLNi0gMxj1W9sLWOMOcmCoxpxdXORV5jHv9b/y+lSjDHmjCw4qpGo8CgubX0pM9bMwB9uzDTG1EwWHNVMcnwymw9s5tud3zpdijHGlMiCo5pJik6iUXAj6yQ3xlRbZQaHiPQWkfqe5zeKyHMi0tb3pfmn+nXrMyZmDO+nvc+hnENOl2OMMacpzxHHK8BxEekKPAD8B7DeWx9ydXORnZ/NOz+943QpxhhzmvIER766e2qHAC+p6jSgoW/L8m8J5yeQcH6CdZIbY6ql8gTHURF5GLgRWCQidYAg35ZlkuOTWf/7elL3Vs/h4I0x/qs8wXEDcAK4Q1V/wz0F7D98WpVhTOwYQoNCbbh1Y0y1U64jDuAFVf1GRC4C4oD3fFuWaRzSmKToJN7b+B5ZuVlOl2OMMV7lCY6vgWARaQUsBW4C3vBlUcbNleAiKzeLuRvnOl2KMcZ4lSc4RFWPA8OAl1V1JBDj27IMQK+IXkSFR9npKmNMtVKu4BCRXsBYYFEF3mfOkYjgSnCxcs9KNvy+welyjDEGKF8ATAAeBhaoapqItAeW+7Ysc9KNXW6kbkBdO+owxlQbZQaHqn6lqtcB00SkgapuV9X7q6A2AzQPbc6wzsN466e3yM7LdrocY4wp15AjsSKyFkgD0kVktYhE+740c5IrwcWhnEPM3zTf6VKMMaZcp6qmAxNVta2qtsE97IidN6lCfSP7cmHTC+10lTGmWihPcNRXVW+fhqquAOr7rCJzmjpSh+SEZL765Su2HNjidDnGGD9XnuDYLiKPikik5/EIsN3XhZlT3Rp3KwESwMw1Nty6McZZ5QmO24FwYD7wAdAcuM2XRZnTndfgPK69+FreWPcGuQW5TpdjjPFj5bmq6qCq3q+qCaraTVUn4O73KJOIDBSRzSKyTUQmlbA+WETmetavFJHIYuvbiEiWiDzoed1aRJaLSLqIpInI+HLtZS3hSnCReTyTjzd/7HQpxhg/drY38vUqq4GIBADTgKuBKGC0iEQVa3YHcFBVOwBTgKeLrX8OWFLkdT7wgKpGAZcA95SwzVrrqguvonWj1tZJboxxlC/vAO8BbPPc95ELzME9p0dRQ4A3Pc/nAf1FRABE5HrgZ9yXAQOgqr+q6hrP86PAJqCVD/ehWgmoE8Dt8bez9D9L2XFoh9PlGGP81BmDQ0QSzvDoRvnm42gF7Cryejenf8l726hqPnAYCBORBsBDwORS6osE4oGVZ1g/TkRSRSQ1MzOzHOXWDLfH3w7A7LWzHa7EGOOvAktZ989S1mVUdiHFPAFMUdUszwHIKTzB8gEwQVWPlLQBVX0NeA0gMTGx1kyj16ZxG67qcBWz187msSseI7BOaX+FxhhT+c74raOqV57jtvcArYu8jvAsK6nNbhEJBBoDB4CewAgReQZoAhSKSI6qviQiQbhD4x1V9ctbqV0JLoanDOfTbZ8y+KLBTpdjjPEzvuzjWAV0FJF2IlIXGAUsLNZmIXCL5/kIYJm69VHVSFWNBJ4HnvKEhgCzgE2q+pwPa6/Wrr3oWlrWb2n3dBhjHOGz4PD0WdwLfIa7EzvFM7ru30XkOk+zWbj7NLYBE4HTLtktpjfuiaT6icg6z+MaH+1CtRUUEMStcbfyyZZP+PXor06XY4zxM6Jaa07/n1FiYqKmpqY6XUal2vbHNjq+2JEn+z3J3/r8zelyjDG1jIisVtXEktadsY9DRBJK2+jJy2KNMzo068CVkVcyc81MJl02iTpic2sZY6rG2V5VpUC/Sq7FVFByQjJj549l2c/L+FP7PzldjjHGT/jyqirjY8M6D6NZvWbMXDPTgsMYU2XKdROAiMTgHjYk5OQyVf2Xr4oy5RMSGMJNXW7ildRX2H98P81DmztdkjHGD5RnBsDHgRc9jyuBZ4DrSn2TqTKuBBe5Bbn8a73luDGmapSnR3UE0B/4TVVvA7rivlHPVAPRLaLpFdGLGWtm4A9XyBljnFee4MhW1UIgX0QaAfs49Y5w4zBXgouM/Rl8t+s7p0sxxviB8gRHqog0wT3P+GpgDfC9T6syFZIUnUTDug1tuHVjTJUoz0ROd6vqIVV9FfgzcIvnlJWpJurXrc+Y2DG8n/Y+h3IOOV2OMaaWK0/n+Jcnn6vqDlX9qegyUz24Elxk52fz7oZ3nS7FGFPLlTYfR4iINAOai0hTEWnmeUTiR5Mn1RTdLuhG/Hnx1klujPG50o447sTdp9EJd7/Gas/jI+Al35dmKsqV4GLdb+tY/etqp0sxxtRiZwwOVX1BVdsBD6pquyKPrqpqwVENjYkdQ73AesxYbZ3kxhjfKc9VVdNF5H4Rmed53OuZTMlUM41DGpMUncS7G98lKzfL6XKMMbVUeYLjZaCb5+fJ56/4sihz9lwJLrJys0hJS3G6FGNMLVVa5/jJcay6q+otqrrM87gN6F415ZmKurT1pXRu3tnu6TDG+ExpRxw/en4WiMiFJxeKSHugwKdVmbMmIrgSXPyw+wc27tvodDnGmFqotOAQz88HgeUiskJEVgDLgAd8XZg5ezd1vYm6AXWtk9wY4xOlBUe4iEwE4oDpuANjGe6hR+KroDZzlpqHNmdop6G89dNb5OTnOF2OMaaWKS04AoAGQEPc83aI5xHoWWaqMVeCi4M5B5m/ab7TpRhjapnSJnL6VVX/XmWVmEp1Zbsrad+0PTPWzGBM7BinyzHG1CLl6eMwNVAdqUNyfDIrdqxg64GtTpdjjKlFSguO/lVWhfGJW+NuJUACmLlmptOlGGNqkdKGHPmjKgsxle/8hudz7cXX8sb6N8gtyHW6HGNMLVGeO8dNDZYcn8y+Y/v4ePPHTpdijKklLDhquYEdBhLRKIKZa+10lTGmclhw1HIBdQK4Pe52Ptv2Gb8c+sXpcowxtYBPg0NEBorIZhHZJiKTSlgfLCJzPetXeiaJKrq+jYhkiciD5d2mOd3t8bcDMHvtbIcrMcbUBj4LDhEJAKYBVwNRwGgRiSrW7A7goKp2AKYATxdb/xywpILbNMW0bdKWqzpcxex1sykotGHGjDHnxpdHHD2Abaq6XVVzgTnAkGJthgBvep7PA/qLiACIyPXAz0BaBbdpSpAcn8zuI7v5dNunTpdijKnhfBkcrYBdRV7v5vS5yr1tVDUfOAyEiUgD4CFg8lls05Tg2ouvpUX9FjbcujHmnFXXzvEngCmqetbT2InIOBFJFZHUzMzMyqushqobUJdbu97KJ1s+4dejvzpdjjGmBvNlcOwBWhd5HeFZVmIbz8RRjYEDQE/gGRHZAUwA/iYi95ZzmwCo6muqmqiqieHh4ee+N7VAckIyBVrAG+vecLoUY0wN5svgWAV0FJF2IlIXGAUsLNZmIXCL5/kIYJm69VHVSFWNBJ4HnlLVl8q5TXMGHcM60jeyLzPXzqRQC50uxxhTQ/ksODx9FvcCnwGbgBRVTRORv4vIdZ5ms3D3aWwDJgKlXl57pm36ah9qI1eCi+0Ht7P85+VOl2KMqaFEVZ2uwecSExM1NTXV6TKqhZz8HC745wUMuHAAc0bMcbocY0w1JSKrVTWxpHXVtXPc+EhIYAg3dbmJBRkL2H98v9PlGGNqIAsOP+Tq5iK3IJe31r/ldCnGmBrIgsMPxbSI4ZKIS5ixZgb+cKrSGFO5LDj8lCvBxab9m/j3rn87XYoxpoax4PBTSdFJNKzb0O4kN8ZUmAWHn2pQtwGjY0aTkpbCoZxDTpdjjKlBLDj8mKubi+z8bN7b8J7TpRhjahALDj/W7fxuxJ0XZ6erjDEVYsHhx0QEV4KLtb+tZfXe1U6XY4ypISw4/NyY2DHUC6xnRx3GmHKz4PBzTUKaMDJ6JO9ueJes3LMexd4Y40csOAyuBBdHc4/yftr7TpdijKkBLDgMvVv3pnPzzna6yhhTLhYcBhEhOSGZ73d/T9o+G6XeGFM6Cw4DwM1db6ZuQF076jDGlMmCwwDQPLQ5QzsN5a2f3iInP8fpcowx1ZgFh/FKTkjmj+w/GL9kvF1hZYw5IwsO49WvXT/u63EfM9bMIGpaFB9lfOR0ScaYasiCw3jVkTpMvXoq393+HU1CmnD93OsZOncouw7vcro0Y0w1YsFhTtOrdS9Wj1vN0396ms+2fUbUy1G88MMLFBQWOF2aMaYasOAwJQoKCOKvvf9K2t1p9GnThwmfTaDHzB42ppUxxoLDlK5d03YsGrOIlBEp7D26lx4zezDh0wkcPXHU6dKMMQ6x4DBlEhFGRo8k454M7up2F1NXTqXztM58mPGh06UZYxxgwWHKrXFIY6YNmsb3d3xPWGgYQ+cOZcicIew8vNPp0owxVciCw1RYz4iepLpS+cef/8EX278galoUU76fQn5hvtOlGWOqgAWHOStBAUE8eOmDpN2dRt/IvkxcOpEeM3qwas8qp0szxviYBYc5J5FNIvl49MfMGzmP37J+o+fMnty/5H6OnDjidGnGGB+x4DDnTEQYHjWcTfds4p7u9/DSjy/ReVpn5m+aj6o6XZ4xfut43nGfbNenwSEiA0Vks4hsE5FJJawPFpG5nvUrRSTSs7yHiKzzPNaLyNAi7/kvEUkTkY0i8p6IhPhyH0z5NQ5pzIvXvMgPyT8QHhrO8JThXDfnOn459IvTpRnjN3Yd3sWU76fQa1YvEqYn+OSXN58Fh4gEANOAq4EoYLSIRBVrdgdwUFU7AFOApz3LNwKJqhoHDASmi0igiLQC7vesiwECgFG+2gdzdnq06kHquFT+OeCfLPt5GVEvR/HPf//TOs+N8ZE9R/bwwg8v0Ht2b9o834aJSyeSk5/DLV1v8cn/u8BK3+L/6gFsU9XtACIyBxgCpBdpMwR4wvN8HvCSiIiqFj2+CgGKRmYgUE9E8oBQYK9vyjfnIrBOIBN7TWR45+Hct+Q+Hvz8Qd7e8DbTB0+nR6seTpdnTI239+hePkj/gJT0FL7d+S0AXVt25cl+TzIyaiQdwzr67LN9GRytgKKj4+0Gep6pjarmi8hhIAzYLyI9gdlAWy1I0+QAAA/nSURBVOAmVc0H9ojIs8BOIBtYqqpLS/pwERkHjANo06ZNpe2UqZi2Tdry0aiP+DDjQ+5bch+XzLyEu7vfzZP9nqRxSGOnyzOmRvkt6zdvWHzzyzcoSmyLWP7Plf+HkVEjubj5xVVShy+D45yo6kogWkQ6A2+KyBKgHu6jlHbAIeB9EblRVd8u4f2vAa8BJCYmWg+tg0SEoZ2H0r99fx5d9igv/vgi8zfNZ+rVUxneeTgi4nSJxlRbv2f9zvxN80lJT+GrHV+hKFHhUTzR9wlGRo2kc3jnKq/Jl8GxB2hd5HWEZ1lJbXaLSCDQGDhQtIGqbhKRLCAGd2D8rKqZACIyH7gUOC04TPXTKLgRL1z9Ajd2uZE7P7mTke+PZFDHQbx0zUtENol0ujxjqo3MY5nesFixYwWFWkin5p149PJHSYpOIrpFtKP1+TI4VgEdRaQd7oAYBYwp1mYhcAvwPTACWKaq6nnPLs/pq7ZAJ2AH7s7wS0QkFPepqv5Aqg/3wfhA91bd+dH1Iy+ufJFHlz9K9MvRPHHFE0y4ZAJBAUFOl2eMI/Yf38+CTQtISU9h+c/LKdACLgq7iL9d9jeSopOIaRFTbY7OxZfX2YvINcDzuL/wZ6vqkyLydyBVVRd6LqV9C4gH/gBGqep2EbkJmATkAYXA31X1Q882JwM3APnAWiBZVU+UVkdiYqKmplq+VEe7Du/iviX38dHmj+jSsgvTB0/nkohLnC7LmCpx4PgBPsz4kJT0FL7c/iUFWkCHZh1IikoiKTqJLi27OBYWIrJaVRNLXOcPN2hZcFR/JzvP9xzZw12Jd/FU/6doEtLE6bKMqXQHsw96w+KL7V+QX5hP+6btuSH6BkZGjSTuvLhqcWRhwWHBUSMcPXGUx5Y/xtQfp9KifgteGPgCI6NGVov/RMaci0M5h/go4yNS0lP4/D+fk1eYR7sm7UiKdh9ZxJ8XX+3+nVtwWHDUKKv3rmbcJ+NY8+saru5wNdOumUa7pu2cLsuYCjmcc5iFmxeSkp7CZ9s+I68wj7aN23rDotv53apdWBRlwWHBUePkF+Yz7cdpPLL8EQoKC3j8iseZ2GuidZ6bau3IiSN8vPljUtJT+HTbp+QW5NK6UWtvWHS/oHu1DouiLDgsOGqs3Ud2c/+S+1mQsYDYFrFMHzydXq17OV2WMV5HTxzlky2fkJKewpKtSzhRcIJWDVt5w6JHqx7UkZo3nqwFhwVHjbdw80LuXXwvu47s4s5ud/J/+/9fmtZr6nRZxk9l5WaxaMsiUtJTWLx1MTn5OVzQ8AJGRo0kKTqJSyIuqZFhUZQFhwVHrZCVm8Xjyx/n+ZXPEx4azvMDn+eG6BtqzKG/qdmO5R5j8dbFpKSnsGjLIrLzszmvwXnesLi09aU1PiyKsuCw4KhV1vy6hjs/uZPUvalcdeFVvDzoZdo3be90WaYWOp53nCVbl5CSnsInWz7heN5xWtZvyYioESRFJ9G7dW8C6gQ4XaZPWHBYcNQ6BYUFvLzqZf572X+TV5jH41c8zgO9HrDOc3POsvOy+XTbp6Skp/Dx5o85lneM8NBwb1j0adOn1oZFURYcFhy11p4jexj/6Xg+2PQB0eHRTB88nd5tejtdlqlhcvJz+GzbZ6Skp7Bw80KycrNoHtqc4Z2HkxSdxOVtLyewTrUdE9YnLDgsOGq9jzd/zL1L7mXn4Z24Elw8/aenrfPclOpE/gmW/mcpKekpfJTxEUdzj9KsXjNvWPSN7Ot3YVGUBYcFh1/Iys1i8orJTPlhCmGhYUy5agqjY0Zb57nxyi3I5fP/fE5KegofZnzIkRNHaBrSlGGdh5EUncSVkVfa6U4PCw4LDr+y7rd13PnJnfy450f+3P7PvDLoFS5sdqHTZRkHHMs9RnpmOhv2beCbnd+wYNMCDp84TJOQJgztNJSk6CT6tetH3YC6Tpda7VhwWHD4nYLCAl5NfZW/LfsbuQW5PHr5ozx46YP2BVFL5Rfms+2PbWz4fQMb9nkev29g+8HtqGfm6cbBjbm+0/UkRSfxp/Z/sn8LZbDgsODwW3uP7mXCpxN4P/19osKjeHXQq/Rp28fpssxZUlX2Ht3Lhn0b2Lhvozcg0jPTOVHgnl2hjtShY7OOxLaMJSY8htiWscS2iKV90/Z+cTVUZbHgsODwe4u2LOKexffwy+FfSI5P5uk/P02zes2cLsuU4nDOYdIy0047ijiYc9Db5vwG53uDIbZFLLEtY+ncvDP1guo5WHntYMFhwWFwn++e/NVknvv+OZrVa8ZzVz3H2Nix1nnusNyCXDbv3+wNhpMhsfPwTm+bhnUbEtMixhsOsS1iiWkRQ1homIOV124WHBYcpoj1v63nzk/uZOWelfRv159XBr1Cx7COTpdV66kqvxz+xRsOJ081ZezPIL8wH4DAOoF0at7pf0PCExRtG7e1gK9iFhwWHKaYgsICXlv9Gg9/+TDH8o7RqmErwuuH06J+C8JDi/0sttxOg5TtwPEDp/RBnAyKo7lHvW3aNG5zSjjEtojl4uYXW6d1NWHBYcFhzuDXo78ybdU0dh3ZReaxTPYd20fmcffPnPycEt9TP6g+Leq3+N9QCT09XIq+Dg4MruK9qjrZedls2r/ptH6IX7N+9bZpGtL0tH6I6PBoGoc0drByUxYLDgsOU0GqSlZuljdEiodKSctzC3JL3Faj4EanBkopQdM8tHm1/I27oLCA7Qe3n3YEsfWPrRRqIQDBAcFEhUeddjXTBQ0vsNNMNVBpweG/99MbUwoRoWFwQxoGNyzXyLuqypETR7yBcqaw2XFoB6v2rCLzeKb3vH5xTUKanBY0JZ0yC68fTvPQ5pU6LIaq8vux392nmYocRaTtSyM7P9v9Z4NwYbMLiWkRQ1J0kvcookOzDn49RIc/sSMOYxygqhzKOXRKsJR2ZLP/+H7vb/bFNavXrFx9M+H1wwmrF+a9lyErN4u0fWmnXc20//h+77Zb1G9xWj9EVHgU9evWr5I/J+McO+IwppoREZrWa0rTek25mIvLbF+ohfyR/cdpweINm+Pun+mZ6WQez+TA8QPeO6ZP+VyEsNAw6gXWY9eRXd7loUGhxLSIYcjFQ7yXusa2jKVF/RaVut+mdrDgMKYGqCN1aB7anOahzekc3rnM9vmF+fyR/UfJRzHHMsnKy+KiZhd5jyLaNW1Xq2avM75lwWFMLRRYJ9B75Zcxlc1+xTDGGFMhFhzGGGMqxKfBISIDRWSziGwTkUklrA8Wkbme9StFJNKzvIeIrPM81ovI0CLvaSIi80QkQ0Q2iUgvX+6DMcaYU/ksOEQkAJgGXA1EAaNFJKpYszuAg6raAZgCPO1ZvhFIVNU4YCAwXURO9se8AHyqqp2ArsAmX+2DMcaY0/nyiKMHsE1Vt6tqLjAHGFKszRDgTc/zeUB/ERFVPa6qJ++OCgH3dYUi0hi4HJgFoKq5qnrIh/tgjDGmGF8GRytgV5HXuz3LSmzjCYrDQBiAiPQUkTRgA3CXZ307IBN4XUTWishMESnxTiQRGSciqSKSmpmZWZn7ZYwxfq3ado6r6kpVjQa6Aw+LSAjuy4cTgFdUNR44BpzWd+J5/2uqmqiqieHh4VVWtzHG1Ha+DI49QOsiryM8y0ps4+nDaAwcKNpAVTcBWUAM7qOW3aq60rN6Hu4gMcYYU0V8eQPgKqCjiLTDHRCjgDHF2iwEbgG+B0YAy1RVPe/Zpar5ItIW6ATsUNX9IrJLRC5W1c1AfyC9rEJWr169X0R+Ocv9aA7sL7NV7WL7XPv52/6C7XNFtT3TCp8Fh+dL/17gMyAAmK2qaSLydyBVVRfi7uR+S0S2AX/gDheAy4BJIpIHFAJ3q+rJnb8PeEdE6gLbgdvKUctZn6sSkdQzDfRVW9k+137+tr9g+1yp2/WH0XHPhf1j8w/+ts/+tr9g+1yZqm3nuDHGmOrJgqNsrzldgANsn2s/f9tfsH2uNHaqyhhjTIXYEYcxxpgKseAwxhhTIRYcZ1DWyL61kYjMFpF9IrLR6Vqqgoi0FpHlIpIuImkiMt7pmnxNREJE5EfPqNNpIjLZ6ZqqiogEeIYq+sTpWqqCiOwQkQ2eUcZTK3Xb1sdxOs/IvluAP+O+W30VMFpVy7zZsCYTkctx36X/L1WNcboeXxOR84HzVXWNiDQEVgPX1+a/ZxERoL6qZolIEPAtMF5Vf3C4NJ8TkYlAItBIVQc7XY+vicgO3KOMV/pNj3bEUbLyjOxb66jq17hvxPQLqvqrqq7xPD+Ke4j+4gNx1irqluV5GeR51PrfHkUkAhgEzHS6ltrAgqNk5RnZ19QinknE4oGVpbes+TynbNYB+4DPi4z9Vps9D/wV90gU/kKBpSKyWkTGVeaGLTiM3xORBsAHwARVPeJ0Pb6mqgWeSdIigB4iUqtPS4rIYGCfqq52upYqdpmqJuCeTO8ez6noSmHBUbLyjOxragHPef4PgHdUdb7T9VQlzyRoy3HPslmb9Qau85zznwP0E5G3nS3J91R1j+fnPmAB7lPwlcKCo2TekX09gymOwj2Sr6lFPB3Fs4BNqvqc0/VUBREJF5Emnuf1cF8AkuFsVb6lqg+raoSqRuL+v7xMVW90uCyfEpH6ngs+8Ex2NwD3lNyVwoKjBJ7ZBk+O7LsJSFHVNGer8j0ReQ/3EPcXi8huEbnD6Zp8rDdwE+7fQNd5Htc4XZSPnQ8sF5GfcP+C9Lmq+sXlqX6mJfCtiKwHfgQWqeqnlbVxuxzXGGNMhdgRhzHGmAqx4DDGGFMhFhzGGGMqxILDGGNMhVhwGGOMqRALDmPOkogUFLmMd11ljqIsIpH+MkqxqXkCnS7AmBos2zN0hzF+xY44jKlknnkQnvHMhfCjiHTwLI8UkWUi8pOIfCkibTzLW4rIAs8cGetF5FLPpgJEZIZn3oylnju9EZH7PXOI/CQicxzaTePHLDiMOXv1ip2quqHIusOqGgu8hHtkVoAXgTdVtQvwDjDVs3wq8JWqdgUSgJOjFHQEpqlqNHAIGO5ZPgmI92znLl/tnDFnYneOG3OWRCRLVRuUsHwH0E9Vt3sGUfxNVcNEZD/uiaPyPMt/VdXmIpIJRKjqiSLbiMQ9HEhHz+uHgCBV/f9F5FPcE259CHxYZH4NY6qEHXEY4xt6hucVcaLI8wL+t09yEDAN99HJKhGxvkpTpSw4jPGNG4r8/N7z/N+4R2cFGAt843n+JfD/gXeSpcZn2qiI1AFaq+py4CGgMXDaUY8xvmS/qRhz9up5ZtI76VNVPXlJblPPCLQngNGeZfcBr4vIX4BM4DbP8vHAa57RiAtwh8ivZ/jMAOBtT7gIMNUzr4YxVcb6OIypZJ4+jkRV3e90Lcb4gp2qMsYYUyF2xGGMMaZC7IjDGGNMhVhwGGOMqRALDmOMMRViwWGMMaZCLDiMMcZUyP8D4i2WsHIW2s8AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU9fX/8dchQUHFumEUAw3fb7UqEBYDKiiCflUoCCpoQdSCWkRKi3XFfltbl/ar1lZ/KKCpuyLgAi4VF0ARFxCCYi2CgholLjWAolGRJef3x2cmhDghE8jNJDPv5+MxD+femblzBmHOfD7n3s8xd0dERKSqJqkOQEREGiYlCBERSUgJQkREElKCEBGRhJQgREQkoexUB1BX9tlnH8/Ly0t1GCIijcrixYtXu3vLRI+lTYLIy8ujqKgo1WGIiDQqZvZhdY9piklERBJSghARkYSUIEREJCElCBERSUgJQkREElKCEBGRhJQgREQkISUIEZHG7Mkn4d57Izm0EoSISGN1661w8slQWAibN9f54ZUgREQam82b4aKL4Ne/hpNOglmzICurzt9GCUJEpDH59ls47TS46SYYOxYefRR22SWSt0qbtZhERNLe55/DgAGwcCHcfHNIEBFSghARaQzeeQf69oXPPoPp00PtIWJKECIiDd28eSEhNG0Kc+dCt2718raqQYiINGQPPgjHHw85ObBgQb0lB1CCEBFpmNzhL3+BYcPgyCPh1Vehbdt6DUFTTCIiDc3GjXDBBXDnnSFB3Hkn7LxzvYehEYSISEPy1VfQv39ICr//Pdx/f0qSA2gEISLScJSUwM9+BsuWhQRxzjkpDUcJQkSkIViyBPr1g7IymDkzFKZTTFNMIiKp9vTTcPTRYbmMl19uEMkBlCBERFKrsDCsp/STn4TTWDt0SHVEFZQgRERSobwcxo2D88+HE08MF8O1apXqqLYSaYIwsz5m9o6ZrTSzcQkeH25mpWa2JHY7L7a/k5nNN7OlZvYvM/t5lHGKiNSr9evhjDPg+uth1Ch4/HFo0SLVUf1AZEVqM8sCJgDHAyXAIjN7wt3frvLUae4+psq+b4Gz3X2FmbUCFpvZs+7+ZVTxiojUizVrYOBAeOUVuOEGuOQSMEt1VAlFeRZTN2Clu78PYGZTgYFA1QTxA+7+bqX7n5jZ50BLQAlCRBqvlSvDaawffQQPPRSW7W7AopxiOgBYVWm7JLavqkGxaaRHzKx11QfNrBuwE/BegsdGmlmRmRWVlpbWVdwiInVv/vywZMbatTBnToNPDpD6IvWTQJ675wOzgK0aq5rZ/sD9wAh3L6/6YncvdPcCdy9o2bJlvQQsIlJrjzwCvXvDHnuERNGjR6ojSkqUCeJjoPKIIDe2r4K7r3H372ObdwCHxR8zs92Bp4D/dfcFEcYpIhINd7jxxjBaOOywkBwOPDDVUSUtygSxCDjQzNqa2U7AEOCJyk+IjRDiBgDLYvt3AmYA97n7IxHGKCISjU2b4Fe/gksvDQlizhzYZ59UR1UrkRWp3X2TmY0BngWygLvcfamZXQ0UufsTwG/MbACwCVgLDI+9/HSgJ7C3mcX3DXf3JVHFKyJSZ8rKYMgQeOopuOwy+L//gyapntGvPXP3VMdQJwoKCryoqCjVYYhIpvvkk7Aa65tvwoQJ4TqHBszMFrt7QaLHtFifiEhd+fe/w2msa9fCk0+G+41Y4xvziIg0RLNnh7OTNm2Cl15q9MkBlCBERHbc3XdD377Qpg289hp07pzqiOqEEoSIyPZyhyuvDI19evcOS3W3/sH1vo2WahAiIttjwwY477zQEnTECLj9dmjaNNVR1SmNIEREauuLL8IS3fffD9dcE9qDpllyAI0gRERqp7g4FKBXrgwJ4swzUx1RZJQgRESStWhRuMZhwwaYNQuOOSbVEUVKU0wiIsl4/PGQEHbZBV59Ne2TAyhBiIjUbPx4OOUUaN8+9I0+5JBUR1QvlCBERKqzeTP89rcwdiwMGABz50JOTqqjqjdKECIiiXz7LQweDDffHBLEo4+G6aUMoiK1iEhV//lPGDEsWrQlQWQgJQgRkcqWLw+nsX72GcyYAQMHpjqilFGCEBGJe/FFOPlk2GmnUG/o1i3VEaWUahAiIgCTJ8Pxx8N++4UzlTI8OYAShIhkOnf485/DFdHdu4drHNq2TXVUDYKmmEQkc23cGDq+3XVXSBB33AE775zqqBoMjSBEJDOtWwf9+oXk8Ic/wH33KTlUoRGEiGSeVatCcli2LKzEes45qY6oQVKCEJHM8sYbITl88w3MnBkK05KQpphEJHPMnAlHHw3Z2aH7m5LDNilBiEhmuP32cHX0QQeF01g7dEh1RA2eEoSIpLfycrj88nC20oknwrx50KpVqqNqFFSDEJH0tX49nH02PPxwSBC33BKmlyQpGkFkoi+/DE3Wr78e3nwzXCgkkm5Wr4bjjgvJ4YYbYOJEJYdaijRBmFkfM3vHzFaa2bgEjw83s1IzWxK7nVfpsV+Y2YrY7RdRxplxCgvhnntg3Djo1AkOOCCc5vfww6EZu0hjt2IFHHkkLF4MDz0El14KZqmOqtExj+jXo5llAe8CxwMlwCJgqLu/Xek5w4ECdx9T5bV7AUVAAeDAYuAwd6/226ugoMCLiorq+mOkn/JyOPBAyM2FKVPg2WfhmWfguefCyKJJk/APq0+fcOvSJewTaSxefTUUowGeeCIsnyHVMrPF7l6Q6LEo/+V3A1a6+/vuvgGYCiS7bu6JwCx3XxtLCrOAPhHFmVmeew7efx9Gjw6FuhEjYNo0KC2FV16B//1f+P77cGVp165h4bKzzoIHHwxDdpGGatUq+Mtf4NhjYc89w5lKSg47JMoJuQOAVZW2S4DDEzxvkJn1JIw2fuvuq6p57QFRBZpRJk4MLRNPOWXr/dnZ4R9T9+5w9dXw+echmTzzTLg98EAYohcUQN++YXTRrRtkZaXmc4hAmBJ95JGwEuuLL4Z9J5wQtvfZJ7WxpYFUzx08CeS5ez5hlHBvbV5sZiPNrMjMikpLSyMJMK18+CE89RSce25Y735b9t03LF72wAOhu9bChXDVVSGRXHttSCQtW8KQIXDvvaG5ikh9WL8+tP889dQwwh05Mvz9u/pqWLkyTJsqOdSJKEcQHwOtK23nxvZVcPc1lTbvAG6o9NpeVV47t+obuHshUAihBrGjAae9wsLw35Eja/e6Jk3CdFPXrmHqae1amDVry+hi2rTwvE6dtowujjwSmjat2/glc5WXhxHCAw+E5LBuXUgOo0fDsGFw2GEqQkcgyiJ1NmHa6DjCF/4i4Ax3X1rpOfu7+6ex+6cAl7v7EbEi9WKgS+yprxOK1Gurez8VqWuwYQO0bg2HHx4Kd3XFPZwq+8wz8PTToUC4aRPsvjv8z/9sKXa3bl3zsUQqi//dmjw5nFDx8cew225h5HDmmdC7t05brQPbKlJH9qfr7pvMbAzwLJAF3OXuS83saqDI3Z8AfmNmA4BNwFpgeOy1a83sGkJSAbh6W8lBkjBjRqgrjB5dt8c1CyOHTp3CabPr1sHzz4dk8fTTMH16eF67dltGF0cdpWWVpXoffhhOipg8GZYuDUmgTx/429/gpJNgl11SHWHGiGwEUd80gqjBMcdASUk4P7y+Tlt1h7ff3jK6eOmlMJLZdddwpkmfPiFpqHuXrF0brsOZPDn8PYFQ5zrzTDjtNNUUIpSSEYQ0IEuXhvVnrr++fq9pMAsjh3bt4OKLoawsNIKPjy6efDI876CDtowujjkGmjevvxgldb77Dv75z5AUZs4M3d0OPjicBHHGGfrh0ABoBJEJxowJrRRLShrOLzH3cMbJ00+HEcYLL4SzU5o1g169towuDjxQxcd0snlz+JEweXIoNn/1Fey/PwwdGorNnTvr/3c929YIQgki3ZWVhQviBg6E++9PdTTV++67MMqJT0e9807Y37btltFF796hSCmNizssWRLOQJo6FT75BFq0gEGDQlLo3VvX06SQEkQmKyyE888PV0k3pqtKP/hgy2m0c+aE7l877RSavcRHF4ceql+bDdkHH2wpNi9bFk577ts31BX699dUYgOhBJGp3MOQHUKbxcb6Zfr99yHBxUcX//532J+buyVZHHcc/OhHqY1TYM2asDje5Mnh/xmEs9bOPBMGD4a9905tfPIDShCZav78MGq47bYwikgXJSVbRhezZoV57PhSIfHrLjp1arwJsbH59ttwwsHkySGBb9oURndnnhlqC3l5qY5QtkEJIlOdfTY89liY803XufuNG8OibPHRxRtvhP377Re6h/XtG/oO77VXauNMN5s3h+td4sXmeK3rjDNCXaFjRyXoRkIJIhOtXh36PPzyl3DrramOpv589llYZPDpp8N/164Np/Yefni4svugg8Iv2ry8cPaMiqPJc4fXX99yZfNnn4Ur5gcPDknhmGP059kI6TqITHT33eGitAsuSHUk9Wu//cLI6eyzw6/coqItp9Jee+3W3fOys6FNmy0J48c/3vr+AQdoKQcIy8NPnhxu77wTis39+oWk0L9/ODVZ0pJGEOmoclOg+BLIEk6l/egjKC4Otw8/3Pr+J59s/fysrLCGVNXkEd/OzU3fBQlLS7cUm+fPD/t69txSbN5zz9TGJ3VGI4hME28K9Oc/pzqShqV5c/jpT8MtkfXrQ9OZqsmjuDicavvxx1uPQJo0CUkiUfLIywvJpaZl1RuSb7+Fxx8PSeHZZ0OxuX17uO66UGxu0ybVEUo90wgiHQ0cGAq3q1Y1ri+ohm7DhvBnWjV5xLdLSsLoLc4sTFNVnbqK32/TJvWLFm7aFJLf5MlhQceyspD04sXm/PzUxieR0wgik3z0UVjfZtw4JYe6ttNO8N//HW6JbNwYRhmJkscrr4SriDdv3vo1+++fePQRTyBRXEzmHmozkyeHmP7zn3ANyZAhISn07Kk+5AIoQaSfwsLwBVDbpkCy45o23fLlnsimTaHOkagGsnBhaJ25cePWr8nJSZw84tu1Wfr6vfe2FJvffTckvP79Q1L42c9UbJYf0BRTOomqKZDUj82b4dNPExfQ4//dsGHr17RsWX3yyMsLdYVp00JSeO21MO11zDGh2DxoEOyxR71+RGl4NMWUKeJNgTLt1NZ0kZUV5v9zc8PyFFWVl4drDxLVQN56K0wtrl+/9WvMwogyPz8s9z50qLr7SdKUINLJpElh9dMTT0x1JBKFJk3C1cqtWoWe31W5hx8IlZPHhg1wyinhbCSRWlKCSBdLl4ZrHuq7KZA0HGahZpGTE6YZRXaQvknSxW23haLjiBGpjkRE0oQSRDooK4N774XTTw9FSxGROlBjgjCzk8xMiaQhe/BB+PprFadFpE4l88X/c2CFmd1gZgdHHZDUknsoTufnJy5ciohspxoThLufCXQG3gPuMbP5ZjbSzFpEHp3UbMGC0O939Gitvy8idSqpqSN3/wp4BJgK7A+cArxuZr+OMDZJxqRJoQH8sGGpjkRE0kwyNYgBZjYDmAs0Bbq5e1+gI3BxtOHJNq1eHZZkPvvs9O0YJyIpk8x1EIOAm9x9XuWd7v6tmZ0bTViSlLvvhu+/V3FaRCKRTIL4E/BpfMPMmgM57l7s7nOiCkxqUF4ern04+mho1y7V0YhIGkqmBvEwUGmRezbH9tXIzPqY2TtmttLMxm3jeYPMzM2sILbd1MzuNbO3zGyZmV2RzPtllHhToNGjUx2JiKSpZBJEtrtXLCEZu19jowEzywImAH2BQ4GhZnZogue1AMYCr1XafRqws7t3AA4DzjezvCRizRyTJsG++8Kpp6Y6EhFJU8kkiFIzGxDfMLOBwOokXtcNWOnu78eSylRgYILnXQNcD1RehtKBXc0sG2gObAC+SuI9M0O8KdC556opkIhEJpkEMQr4nZl9ZGargMuB85N43QHAqkrbJbF9FcysC9Da3Z+q8tpHgG8ItY+PgBvdfW0S75kZ4k2Bzk/mf4OIyPapsUjt7u8BR5jZbrHtsrp449jyHX8Hhid4uBuh1tEK2BN4ycxmu/v7VY4xEhgJ0CZTGqpv2AB33AH9+oWmMCIiEUlquW8z6we0A5pZ7Gpdd7+6hpd9DFTuTJIb2xfXAmgPzI0dcz/gidh01hnAM+6+EfjczF4BCoCtEoS7FwKFEDrKJfNZGr0ZM0IPYRWnRSRiyVwodxthPaZfA0YoICfz03URcKCZtTWznYAhQEUfTHdf5+77uHueu+cBC4AB7l5EmFY6Nvb+uwJHAMtr88HSlpoCiUg9SaYG0d3dzwa+cPergCOBg2p6kbtvAsYAzwLLgIfcfamZXV256F2NCcBuZraUkGjudvd/JRFrenv77dAU6Pzz1RRIRCKXzBRT/Oyib82sFbCGsB5Tjdx9JjCzyr4rq3lur0r3ywgjFals0qRw1tI556Q6EhHJAMkkiCfNbA/gr8DrhFNQ/xFpVPJDZWVw331w2mlqCiQi9WKbCSJ2ptEcd/8SeNTM/gk0c/d19RKdbPHgg/DVV1p3SUTqzTYnst29nFAPiG9/r+SQApWbAnXvnupoRCRDJFPpnBNbK0ndaFLltddCU6ALLlBTIBGpN8kkiPMJi/N9b2ZfmdnXZqZlL+rTxIlqCiQi9S6ZK6nVWjSV4k2Bzj03JAkRkXpSY4Iws56J9ldtICQRUVMgEUmRZE5zvbTS/WaEdZIWE7vSWSJUXg633x6aArVvn+poRCTDJDPFdFLlbTNrDdwcWUSyxaxZ8N57cM01qY5ERDLQ9qzXUAIcUteBSAITJ6opkIikTDI1iFsIV09DSCidCFdUS5TiTYEuvxx23jnV0YhIBkqmBlFU6f4mYIq7vxJRPBL3j3+EC+RGjkx1JCKSoZJJEI8A6919M4Re02a2i7t/G21oGWzDhpAg+vWDvLxURyMiGSqpK6kJfaHjmgOzowlHAHjssdAUSKe2ikgKJZMgmlVuMxq7v0t0IQkTJ4aRg5oCiUgKJZMgvjGzLvENMzsM+C66kDJcvCnQqFGQlZXqaEQkgyVTg7gQeNjMPiG0HN2P0IJUonDbbWoKJCINQjIXyi0ys4OBn8Z2vePuG6MNK0OVlcG996opkIg0CDVOMZnZr4Bd3f3f7v5vQq/o0dGHloGmTFFTIBFpMJKpQfwy1lEOAHf/AvhldCFlKPdQnO7QQU2BRKRBSCZBZFVuFmRmWcBO0YWUoeJNgUaPVlMgEWkQkilSPwNMM7PbY9vnA09HF1KGmjQJdttNTYFEpMFIJkFcDowERsW2/0U4k0nqypo1MG2amgKJSINS4xSTu5cDrwHFhF4QxwLLog0rw6gpkIg0QNWOIMzsIGBo7LYamAbg7r3rJ7QMUV4ern046ig1BRKRBmVbI4jlhNFCf3c/yt1vATbXT1gZJN4UaLTOHBaRhmVbCeJU4FPgBTP7h5kdR7iSWurSpEnhojg1BRKRBqbaBOHuj7n7EOBg4AXCkhv7mtkkMzshmYObWR8ze8fMVprZuG08b5CZuZkVVNqXb2bzzWypmb1lZs2S/1iNxEcfwZNPhuK0mgKJSAOTTJH6G3d/MNabOhd4g3Bm0zbFrpeYAPQFDgWGmtmhCZ7XAhhLKITH92UDDwCj3L0d0AtIv+U94k2Bzj8/1ZGIiPxArXpSu/sX7l7o7scl8fRuwEp3f9/dNwBTgYEJnncNcD2wvtK+E4B/ufubsfddE29YlDY2bIA77oCf/UxNgUSkQapVgqilA4BVlbZLYvsqxJYRb+3uT1V57UGAm9mzZva6mV0WYZyp8dhj8NlnKk6LSIOVzIVykTCzJsDfgeEJHs4GjgK6At8Cc8xssbvPqXKMkYSL+GjTpk2k8da5SZPUFEhEGrQoRxAfA60rbefG9sW1ANoDc82sGDgCeCJWqC4B5rn76ljv65lAF6qITXcVuHtBy8a0PPbbb8PcuaH2oKZAItJARZkgFgEHmllbM9sJGAI8EX/Q3de5+z7unufuecACYIC7FwHPAh3MbJdYwfoY4O0IY61fagokIo1AZAnC3TcBYwhf9suAh9x9qZldbWYDanjtF4Tpp0XAEuD1BHWKxumbb0JToMGDYd99Ux2NiEi1Iq1BuPtMwvRQ5X1XVvPcXlW2HyCc6ppeHnwwNAVScVpEGrgop5ikKjUFEpFGRAmiPsWbAl1wgZoCiUiDpwRRn+JNgc48M9WRiIjUSAmivsSbAp11lpoCiUijoARRX9QUSEQaGSWI+lC5KVCHDqmORkQkKUoQ9SHeFEijBxFpRJQg6kO8KdCgQamOREQkaUoQUVu1Sk2BRKRRUoKIWmGhmgKJSKOkBBGljRvVFEhEGi0liCjFmwKpOC0ijZASRJQmTgwjhz59Uh2JiEitKUFEZdkyNQUSkUZNCSIqkyZB06ZqCiQijZYSRBTiTYFOO01NgUSk0VKCiMKUKaEpkIrTItKIKUHUtXhToPbtoUePVEcjIrLdlCDq2sKF8MYboaWomgKJSCOmBFHXJk5UUyARSQtKEHVJTYFEJI0oQdSle+5RUyARSRtKEHWlvDxc+9Cjh5oCiUhaUIKoK7Nnh6ZAo0enOhIRkTqhBFFXJk5UUyARSStKEHVBTYFEJA0pQdSFf/xDTYFEJO1EmiDMrI+ZvWNmK81s3DaeN8jM3MwKquxvY2ZlZnZJlHHukI0bQ4Lo21dNgUQkrUSWIMwsC5gA9AUOBYaa2aEJntcCGAu8luAwfweejirGOhFvCqTitIikmShHEN2Ale7+vrtvAKYCAxM87xrgemB95Z1mdjLwAbA0whh33MSJ8OMfqymQiKSdKBPEAcCqStslsX0VzKwL0Nrdn6qyfzfgcuCqbb2BmY00syIzKyotLa2bqGtDTYFEJI2lrEhtZk0IU0gXJ3j4T8BN7l62rWO4e6G7F7h7QcuWLSOIsga33RaaAp17bv2/t4hIxLIjPPbHQOtK27mxfXEtgPbAXAurnu4HPGFmA4DDgcFmdgOwB1BuZuvd/dYI462db74JS2sMHqymQCKSlqJMEIuAA82sLSExDAHOiD/o7uuAfeLbZjYXuMTdi4CjK+3/E1DWoJIDbGkKpOK0iKSpyKaY3H0TMAZ4FlgGPOTuS83s6tgoofFSUyARyQBRjiBw95nAzCr7rqzmub2q2f+nOg9sR8WbAk2YoKZAIpK2dCX19pg0SU2BRCTtKUHU1po1MHVqSA67757qaEREIqMEUVtqCiQiGUIJojbKy8O1Dz16QH5+qqMREYmUEkRtzJ4NK1dq9CAiGUEJojYmTYJ99gkXx4mIpDkliGStWgVPPKGmQCKSMZQgkqWmQCKSYZQgklG5KVDbtqmORkSkXihBJCPeFEjFaRHJIEoQyZg0KTQF6ts31ZGIiNQbJYiaLFsGL7ygpkAiknGUIGqipkAikqGUILblm2/g3nvVFEhEMpISxLZMmQLr1qk4LSIZSQmiOu6hON2uHRx1VKqjERGpd0oQ1Vm0CF5/PbQUVVMgEclAShDVmTgRdt1VTYFEJGNF2nK00VqzBqZNg+HD1RRIANi4cSMlJSWsX78+1aGIbJdmzZqRm5tL06ZNk36NEkQi99wD69erOC0VSkpKaNGiBXl5eZimHKWRcXfWrFlDSUkJbWuxXJCmmKqKNwXq3l1NgaTC+vXr2XvvvZUcpFEyM/bee+9aj4CVIKqaMyc0BRo9OtWRSAOj5CCN2fb8/VWCqGriRDUFEhFBCWJrJSVqCiQNVlZWFp06daq4FRcX19mxzYyLL764YvvGG2/kT3/60zZfM3fuXF599dUf7C8uLiY3N5fy8vKt9nfq1InXXnst4bGKi4tp3749AEVFRfzmN79J+Ly8vDxWr169zbj+8pe/bLXdvXv3bT6/NjZt2kTLli0ZN25cnR2zIVOCqKywUE2BpMFq3rw5S5Ysqbjl5eXV2bF33nlnpk+fXuOXb2XVJYi8vDzatGnDSy+9VLFv+fLlfP311xx++OE1HregoIDx48cnHUdVVRNEohi316xZszjooIN4+OGHcfc6O25VmzZtiuzYtaEEEbdxI9xxB/Tpo6ZAsm0XXgi9etXt7cILtyuUyr+oi4qK6NWrFwAvvvhixUijc+fOfP311wD89a9/pWvXruTn5/PHP/6x4jjZ2dmMHDmSm2666QfvUVpayqBBg+jatStdu3bllVdeobi4mNtuu42bbrqJTp06bZUMAIYOHcrUqVMrtqdOncqQIUMoLi7m6KOPpkuXLnTp0iXhl/fcuXPp378/AGvWrOGEE06gXbt2nHfeeVt9KZ988skcdthhtGvXjsLCQgDGjRvHd999R6dOnRg2bBgAu+22GxDO5Ln00ktp3749HTp0YNq0aRXv16tXLwYPHszBBx/MsGHDqv3ynzJlCmPHjqVNmzbMnz+/Yv8zzzxDly5d6NixI8cddxwAZWVljBgxgg4dOpCfn8+jjz66VTwAjzzyCMOHDwdg+PDhjBo1isMPP5zLLruMhQsXcuSRR9K5c2e6d+/OO++8A8DmzZu55JJLaN++Pfn5+dxyyy08//zznHzyyRXHnTVrFqecckrCz1AbOs017vHH4dNPwyhCpAGKf/EBtG3blhkzZlT73BtvvJEJEybQo0cPysrKaNasGc899xwrVqxg4cKFuDsDBgxg3rx59OzZE4Bf/epX5Ofnc9lll211rLFjx/Lb3/6Wo446io8++ogTTzyRZcuWMWrUKHbbbTcuueSSH7z/6aefTqdOnbjlllvIzs5m2rRpPPzww+y7777MmjWLZs2asWLFCoYOHUpRUVG1n+Oqq67iqKOO4sorr+Spp57izjvvrHjsrrvuYq+99uK7776ja9euDBo0iOuuu45bb72VJUuW/OBY06dPZ8mSJbz55pusXr2arl27Vnz2N954g6VLl9KqVSt69OjBK6+8wlFVlthZv349s2fP5vbbb+fLL79kypQpdO/endLSUn75y18yb9482rZty9q1awG45ppr+NGPfsRbb70FwBdffFHt54wrKSnh1VdfJSsri6+++oqXXnqJ7OxsZs+eze9+9zseffRRCgsLKS4uZsmSJWRnZ7N27Vr23HNPRo8eTWlpKS1btqCLZ98AAAtDSURBVOTuu+/mnHPOqfH9ahJpgjCzPsD/A7KAO9z9umqeNwh4BOjq7kVmdjxwHbATsAG41N2fjzJWJk5UUyBJzs03p+Rt41NMyejRowcXXXQRw4YN49RTTyU3N5fnnnuO5557js6dOwPhF+6KFSsqviR33313zj77bMaPH0/z5s0rjjV79mzefvvtiu2vvvqKsrKybb5/Tk4O7du3Z86cOeTk5JCdnU379u1Zt24dY8aMYcmSJWRlZfHuu+9u8zjz5s1j+vTpAPTr148999yz4rHx48dXJMlVq1axYsUK9t5772qP9fLLLzN06FCysrLIycnhmGOOYdGiRey+++5069aN3NxcgIr6TtUE8c9//pPevXvTvHlzBg0axDXXXMPNN9/MggUL6NmzZ8X1BXvttVfFn1vlUVTl2Ktz2mmnkRXrO7Nu3Tp+8YtfsGLFCsyMjRs3Vhx31KhRZGdnb/V+Z511Fg888AAjRoxg/vz53HfffTW+X00iSxBmlgVMAI4HSoBFZvaEu79d5XktgLFA5erVauAkd//EzNoDzwIHRBVrRVOgP/9ZTYGkUcnOzq4oBlc+x33cuHH069ePmTNn0qNHD5599lncnSuuuILzt1Fju/DCC+nSpQsjRoyo2FdeXs6CBQto1qxZrWKLTzPl5OQwdOhQAG666SZycnJ48803KS8vr/Ux4+bOncvs2bOZP38+u+yyC7169dqhq9x3rnRSSlZWVsIawJQpU3j55Zcraj9r1qzh+edr/7u18ummVWPeddddK+7/4Q9/oHfv3syYMYPi4uKK6cPqjBgxgpNOOolmzZpx2mmnVSSQHRFlDaIbsNLd33f3DcBUYGCC510DXA9U/Em5+xvu/klscynQ3MyiO61ITYGkkcrLy2Px4sUAFXPcAO+99x4dOnTg8ssvp2vXrixfvpwTTzyRu+66q+LX/8cff8znn3++1fH22msvTj/99K2mck444QRuueWWiu34KKZFixYVtY1ETj31VGbOnMm0adMYMmQIEH4V77///jRp0oT777+fzZs3b/Pz9ezZkwcffBCAp59+umKaZt26dey5557ssssuLF++nAULFlS8pmnTphW/tis7+uijmTZtGps3b6a0tJR58+bRrVu3bb5/XHy656OPPqK4uJji4mImTJjAlClTOOKII5g3bx4ffPABQMUU0/HHH8+ECRMqjhGPPScnh2XLllFeXr7NacJ169ZxwAHhd/E999xTsf/444/n9ttvr0hi8fdr1aoVrVq14tprr90qwe+IKBPEAcCqStslVBkFmFkXoLW7P7WN4wwCXnf376s+YGYjzazIzIpKS0u3L8p4U6BBgyAnZ/uOIZIif/zjHxk7diwFBQUVUxMAN998c0URs2nTpvTt25cTTjiBM844gyOPPJIOHTowePDghF/wF1988VZnM40fP56ioiLy8/M59NBDue222wA46aSTmDFjRsIiNcAee+zBkUceSU5ODv/1X/8FwOjRo7n33nvp2LEjy5cv3+oXc3Wfb968ebRr147p06fTpk0bAPr06cOmTZs45JBDGDduHEcccUTFa0aOHEl+fn5FkTrulFNOIT8/n44dO3Lsscdyww03sN9++9X0RwzAjBkzOPbYY7caaQwcOJAnn3yS3XffncLCQk499VQ6duzIz3/+cwB+//vf88UXX9C+fXs6duzICy+8AMB1111H//796d69O/vvv3+173nZZZdxxRVX0Llz561GNOeddx5t2rSp+CzxBAowbNgwWrduzSGHHJLU56qJRXWqlpkNBvq4+3mx7bOAw919TGy7CfA8MNzdi81sLnCJuxdVOkY74AngBHd/b1vvV1BQ4NsqdlXrk0/gootgzBj1fZBqLVu2rM7+0YlEZcyYMXTu3Jlzq5kNSfT32MwWu3tBoudHWaT+GGhdaTs3ti+uBdAemBubk9sPeMLMBsQK1bnADODsmpLDDmnVCioVkkREGqPDDjuMXXfdlb/97W91dswoE8Qi4EAza0tIDEOAM+IPuvs6YJ/4duURhJntATwFjHP3VyKMUUQkLcRrUXUpshqEu28CxhDOQFoGPOTuS83sajMbUMPLxwA/Aa40syWx275RxSqSjCivnBWJ2vb8/Y2sBlHftrsGIZKEDz74gBYtWmjJb2mU4v0gvv766x/0g0hVDUIkbeTm5lJSUsJ2ny0nkmLxjnK1oQQhkoSmTZvWqhOXSDrQYn0iIpKQEoSIiCSkBCEiIgmlzVlMZlYKfLgDh9iHsEhgJsm0z5xpnxf0mTPFjnzmH7t7y0QPpE2C2FFmVlTdqV7pKtM+c6Z9XtBnzhRRfWZNMYmISEJKECIikpASxBaZ2Gs00z5zpn1e0GfOFJF8ZtUgREQkIY0gREQkISUIERFJKOMThJn1MbN3zGylmY1LdTxRM7O7zOxzM/t3qmOpL2bW2sxeMLO3zWypmY1NdUxRM7NmZrbQzN6MfearUh1TfTCzLDN7w8z+mepY6ouZFZvZW7G2CHW6pHVG1yDMLAt4Fzie0DN7ETDU3d9OaWARMrOeQBlwn7u3T3U89cHM9gf2d/fXzawFsBg4Oc3/Pxuwq7uXmVlT4GVgrLsvSHFokTKzi4ACYHd375/qeOqDmRUDBe5e5xcHZvoIohuw0t3fd/cNwFRgYIpjipS7zwPWpjqO+uTun7r767H7XxMaWB2Q2qii5UFZbLNp7JbWvwZjbYr7AXekOpZ0kekJ4gBgVaXtEtL8iyPTmVke0Bl4LbWRRC823bIE+ByY5e7p/plvBi4DylMdSD1z4DkzW2xmI+vywJmeICSDmNluwKPAhe7+VarjiZq7b3b3TkAu0M3M0nZK0cz6A5+7e903Zm74jnL3LkBf4FexaeQ6kekJ4mOgdaXt3Ng+STOxefhHgcnuPj3V8dQnd/8SeAHok+pYItQDGBCbj58KHGtmD6Q2pPrh7h/H/vs5MIMwdV4nMj1BLAIONLO2ZrYTMAR4IsUxSR2LFWzvBJa5+99THU99MLOWZrZH7H5zwokYy1MbVXTc/Qp3z3X3PMK/4+fd/cwUhxU5M9s1duIFZrYrcAJQZ2coZnSCcPdNwBjgWULh8iF3X5raqKJlZlOA+cBPzazEzM5NdUz1oAdwFuFX5ZLY7WepDipi+wMvmNm/CD+EZrl7xpz6mUFygJfN7E1gIfCUuz9TVwfP6NNcRUSkehk9ghARkeopQYiISEJKECIikpAShIiIJKQEISIiCSlBiNTAzDZXOj12SV2u+mtmeZm0sq40LtmpDkCkEfgutmSFSEbRCEJkO8XW4b8hthb/QjP7SWx/npk9b2b/MrM5ZtYmtj/HzGbEejS8aWbdY4fKMrN/xPo2PBe78hkz+02sh8W/zGxqij6mZDAlCJGaNa8yxfTzSo+tc/cOwK2E1UQBbgHudfd8YDIwPrZ/PPCiu3cEugDxq/YPBCa4ezvgS2BQbP84oHPsOKOi+nAi1dGV1CI1MLMyd98twf5i4Fh3fz+2GOBn7r63ma0mNCjaGNv/qbvvY2alQK67f1/pGHmEZTAOjG1fDjR192vN7BlCc6fHgMcq9XcQqRcaQYjsGK/mfm18X+n+ZrbUBvsBEwijjUVmppqh1CslCJEd8/NK/50fu/8qYUVRgGHAS7H7c4ALoKKZz4+qO6iZNQFau/sLwOXAj4AfjGJEoqRfJCI1ax7rzBb3jLvHT3XdM7Zi6vfA0Ni+XwN3m9mlQCkwIrZ/LFAYW0F3MyFZfFrNe2YBD8SSiAHjY30dROqNahAi2ynKZvEiDYGmmEREJCGNIEREJCGNIEREJCElCBERSUgJQkREElKCEBGRhJQgREQkof8PwHEGzPENoNwAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    }
  ]
}