clip_rnn_image_captioning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "clip_rnn_image_captioning.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "zJ66DKu72U_4"
      },
      "source": [
        "!pip install ftfy regex tqdm\n",
        "!pip install git+https://github.com/openai/CLIP.git"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z_TkEgHjw00K"
      },
      "source": [
        "import torch\n",
        "import clip\n",
        "from PIL import Image\n",
        "\n",
        "device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\" # If using GPU then use mixed precision training.\n",
        "clip_model, preprocess = clip.load(\"ViT-B/32\", device=device)\n",
        "\n",
        "image = preprocess(Image.open(\"clip.png\")).unsqueeze(0).to(device)\n",
        "text = clip.tokenize([\"a diagram\", \"a dog\", \"a cat\"]).to(device)\n",
        "\n",
        "with torch.no_grad():\n",
        "    image_features = clip_model.encode_image(image)\n",
        "    text_features = clip_model.encode_text(text)\n",
        "    \n",
        "    logits_per_image, logits_per_text = clip_model(image, text)\n",
        "    probs = logits_per_image.softmax(dim=-1).cpu().numpy()\n",
        "print(\"Label probs:\", probs)  # prints: [[0.9927937  0.00421068 0.00299572]]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y4-tGI_Wdyw2"
      },
      "source": [
        "!pip install -r requirements.txt\n",
        "!pip install requests"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xc030jlBeNTX"
      },
      "source": [
        "!wget https://ivc.ischool.utexas.edu/VizWiz_final/images/train.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SaENe1znm0TD"
      },
      "source": [
        "!mkdir images\n",
        "!mv \"/content/train.zip\" \"/content/images/\"\n",
        "!unzip /content/images/train.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eiYJU_H8m87u"
      },
      "source": [
        "!mv \"/content/train\" \"/content/images/\"\n",
        "!rm /content/images/train.zip"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M1enIWVZn3Yw"
      },
      "source": [
        "!wget https://ivc.ischool.utexas.edu/VizWiz_final/images/val.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VwsnSGwroIEd"
      },
      "source": [
        "!mv \"/content/val.zip\" \"/content/images/\"\n",
        "!unzip /content/images/val.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O2BKyvBpoPV9"
      },
      "source": [
        "!mv \"/content/val\" \"/content/images/\"\n",
        "!rm /content/images/val.zip"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_DNuL2oFoVSX"
      },
      "source": [
        "!wget https://ivc.ischool.utexas.edu/VizWiz_final/images/test.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d1q45xdToVpi"
      },
      "source": [
        "!mv \"/content/test.zip\" \"/content/images/\"\n",
        "!unzip /content/images/test.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GG535_vroV6L"
      },
      "source": [
        "!mv \"/content/test\" \"/content/images/\"\n",
        "!rm /content/images/test.zip"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DFKCg4Vp0gmI"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torchvision.models as models\n",
        "\n",
        "import torch.nn.functional as F\n",
        "import numpy as np\n",
        "import math\n",
        "\n",
        "device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\n",
        "\n",
        "class DecoderRNN(nn.Module):\n",
        "    def __init__(self, embed_size, hidden_size, vocab_size):\n",
        "        ''' Initialize the layers of this model.'''\n",
        "        super().__init__()\n",
        "\n",
        "        # Keep track of hidden_size for initialization of hidden state\n",
        "        self.hidden_size = hidden_size\n",
        "\n",
        "        # Embedding layer that turns words into a vector of a specified size\n",
        "        self.word_embeddings = nn.Embedding(vocab_size, embed_size)\n",
        "\n",
        "        # The LSTM takes embedded word vectors (of a specified size) as input\n",
        "        # and outputs hidden states of size hidden_dim\n",
        "        self.lstm = nn.LSTM(input_size=embed_size, \\\n",
        "                            hidden_size=hidden_size,  # LSTM hidden units\n",
        "                            num_layers=1,  # number of LSTM layer\n",
        "                            bias=True,  # use bias weights b_ih and b_hh\n",
        "                            batch_first=True,  # input & output will have batch size as 1st dimension\n",
        "                            dropout=0,  # Not applying dropout\n",
        "                            bidirectional=False,  # unidirectional LSTM\n",
        "                            )\n",
        "\n",
        "        # The linear layer that maps the hidden state output dimension\n",
        "        # to the number of words we want as output, vocab_size\n",
        "        self.linear = nn.Linear(hidden_size, vocab_size)\n",
        "\n",
        "        # initialize the hidden state\n",
        "        # self.hidden = self.init_hidden()\n",
        "\n",
        "    def init_hidden(self, batch_size):\n",
        "        \"\"\" At the start of training, we need to initialize a hidden state;\n",
        "        there will be none because the hidden state is formed based on previously seen data.\n",
        "        So, this function defines a hidden state with all zeroes\n",
        "        The axes semantics are (num_layers, batch_size, hidden_dim)\n",
        "        \"\"\"\n",
        "        return (torch.zeros((1, batch_size, self.hidden_size), device=device), \\\n",
        "                torch.zeros((1, batch_size, self.hidden_size), device=device))\n",
        "\n",
        "    def forward(self, features, captions):\n",
        "        \"\"\" Define the feedforward behavior of the model \"\"\"\n",
        "\n",
        "        # Discard the <end> word to avoid predicting when <end> is the input of the RNN\n",
        "        captions = captions[:, :-1]\n",
        "\n",
        "        # Initialize the hidden state\n",
        "        self.batch_size = features.shape[0]  # features is of shape (batch_size, embed_size)\n",
        "        self.hidden = self.init_hidden(self.batch_size)\n",
        "        \n",
        "        # Create embedded word vectors for each word in the captions\n",
        "        embeddings = self.word_embeddings(\n",
        "            captions)  # embeddings new shape : (batch_size, captions length - 1, embed_size)\n",
        "\n",
        "        # Stack the features and captions\n",
        "        embeddings = torch.cat((features.unsqueeze(1), embeddings),\n",
        "                               dim=1)  # embeddings new shape : (batch_size, caption length, embed_size)\n",
        "\n",
        "        # Get the output and hidden state by passing the lstm over our word embeddings\n",
        "        # the lstm takes in our embeddings and hidden state\n",
        "        lstm_out, self.hidden = self.lstm(embeddings,\n",
        "                                          self.hidden)  # lstm_out shape : (batch_size, caption length, hidden_size)\n",
        "\n",
        "        # Fully connected layer\n",
        "        outputs = self.linear(lstm_out)  # outputs shape : (batch_size, caption length, vocab_size)\n",
        "\n",
        "        return outputs\n",
        "\n",
        "    ## Greedy search\n",
        "    def sample(self, inputs):\n",
        "        \" accepts pre-processed image tensor (inputs) and returns predicted sentence (list of tensor ids of length max_len) \"\n",
        "\n",
        "        output = []\n",
        "        batch_size = inputs.shape[0]  # batch_size is 1 at inference, inputs shape : (1, 1, embed_size)\n",
        "        hidden = self.init_hidden(batch_size)  # Get initial hidden state of the LSTM\n",
        "\n",
        "        while True:\n",
        "            lstm_out, hidden = self.lstm(inputs, hidden)  # lstm_out shape : (1, 1, hidden_size)\n",
        "            outputs = self.linear(lstm_out)  # outputs shape : (1, 1, vocab_size)\n",
        "            outputs = outputs.squeeze(1)  # outputs shape : (1, vocab_size)\n",
        "            _, max_indice = torch.max(outputs, dim=1)  # predict the most likely next word, max_indice shape : (1)\n",
        "\n",
        "            output.append(max_indice.cpu().numpy()[0].item())  # storing the word predicted\n",
        "\n",
        "            if (max_indice == 1):\n",
        "                # We predicted the <end> word, so there is no further prediction to do\n",
        "                break\n",
        "\n",
        "            ## Prepare to embed the last predicted word to be the new input of the lstm\n",
        "            inputs = self.word_embeddings(max_indice)  # inputs shape : (1, embed_size)\n",
        "            inputs = inputs.unsqueeze(1)  # inputs shape : (1, 1, embed_size)\n",
        "\n",
        "        return output\n",
        "\n",
        "    ## Beam search implementation (Attempt)\n",
        "    def beam_search_sample(self, inputs, beam=3):\n",
        "        output = []\n",
        "        batch_size = inputs.shape[0]  # batch_size is 1 at inference, inputs shape : (1, 1, embed_size)\n",
        "        hidden = self.init_hidden(batch_size)  # Get initial hidden state of the LSTM\n",
        "\n",
        "        # sequences[0][0] : index of start word\n",
        "        # sequences[0][1] : probability of the word predicted\n",
        "        # sequences[0][2] : hidden state related of the last word\n",
        "        sequences = [[[torch.Tensor([0])], 1.0, hidden]]\n",
        "        max_len = 20\n",
        "\n",
        "        ## Step 1\n",
        "        # Predict the first word <start>\n",
        "        outputs, hidden = DecoderRNN.get_outputs(self, inputs, hidden)\n",
        "        _, max_indice = torch.max(outputs, dim=1)  # predict the most likely next word, max_indice shape : (1)\n",
        "        output.append(max_indice.cpu().numpy()[0].item())  # storing the word predicted\n",
        "        # inputs = DecoderRNN.get_next_word_input(self, max_indice)\n",
        "\n",
        "        l = 0\n",
        "        while len(sequences[0][0]) < max_len:\n",
        "            print(\"l:\", l)\n",
        "            l += 1\n",
        "            temp = []\n",
        "            for seq in sequences:\n",
        "                #                 print(\"seq[0]: \", seq[0])\n",
        "                inputs = seq[0][-1]  # last word index in seq\n",
        "                inputs = inputs.type(torch.cuda.LongTensor)\n",
        "                print(\"inputs : \", inputs)\n",
        "                # Embed the input word\n",
        "                inputs = self.word_embeddings(inputs)  # inputs shape : (1, embed_size)\n",
        "                inputs = inputs.unsqueeze(1)  # inputs shape : (1, 1, embed_size)\n",
        "\n",
        "                # retrieve the hidden state\n",
        "                hidden = seq[2]\n",
        "\n",
        "                preds, hidden = DecoderRNN.get_outputs(self, inputs, hidden)\n",
        "\n",
        "                # Getting the top <beam_index>(n) predictions\n",
        "                softmax_score = F.log_softmax(outputs, dim=1)  # Define a function to sort the cumulative score\n",
        "                sorted_score, indices = torch.sort(-softmax_score, dim=1)\n",
        "                word_preds = indices[0][:beam]\n",
        "                best_scores = sorted_score[0][:beam]\n",
        "\n",
        "                # Creating a new list so as to put them via the model again\n",
        "                for i, w in enumerate(word_preds):\n",
        "                    #                     print(\"seq[0]: \", seq[0][0][:].cpu().numpy().item())\n",
        "                    next_cap, prob = seq[0][0].cpu().numpy().tolist(), seq[1]\n",
        "\n",
        "                    next_cap.append(w)\n",
        "                    print(\"next_cap : \", next_cap)\n",
        "                    prob * best_scores[i].cpu().item()\n",
        "                    temp.append([next_cap, prob])\n",
        "\n",
        "            sequences = temp\n",
        "            # Order according to proba\n",
        "            ordered = sorted(sequences, key=lambda tup: tup[1])\n",
        "\n",
        "            # Getting the top words\n",
        "            sequences = ordered[:beam]\n",
        "            print(\"sequences: \", sequences)\n",
        "\n",
        "    def get_outputs(self, inputs, hidden):\n",
        "        lstm_out, hidden = self.lstm(inputs, hidden)  # lstm_out shape : (1, 1, hidden_size)\n",
        "        outputs = self.linear(lstm_out)  # outputs shape : (1, 1, vocab_size)\n",
        "        outputs = outputs.squeeze(1)  # outputs shape : (1, vocab_size)\n",
        "\n",
        "        return outputs, hidden\n",
        "\n",
        "    def get_next_word_input(self, max_indice):\n",
        "        ## Prepare to embed the last predicted word to be the new input of the lstm\n",
        "        inputs = self.word_embeddings(max_indice)  # inputs shape : (1, embed_size)\n",
        "        inputs = inputs.unsqueeze(1)  # inputs shape : (1, 1, embed_size)\n",
        "\n",
        "        return inputs"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3ZT3kYrF8lh7"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.utils.data as data\n",
        "from data_loader import get_loader\n",
        "from torchvision import transforms\n",
        "#Malikeh: from models import EncoderCNN, DecoderRNN\n",
        "import math\n",
        "import utils\n",
        "import json\n",
        "\n",
        "def validate(clip_model, decoder, criterion, data_loader, vocab_size, epoch, device='cpu', save_captions=False, linear_layer = None):\n",
        "    with torch.no_grad():\n",
        "        #Malikeh: encoder.eval()\n",
        "        if linear_layer != None:\n",
        "          linear_layer.eval()\n",
        "        decoder.eval()\n",
        "        val_loss = 0\n",
        "\n",
        "        total_step = len(data_loader.dataset.paths) #number of images in val dataset\n",
        "        predicted_captions = []\n",
        "        print(\"Running Validation...\")\n",
        "        for batch in data_loader:\n",
        "            # Obtain the batch.\n",
        "            images, captions, img_id = batch #next(iter(data_loader))\n",
        "            # Move batch of images and captions to GPU if CUDA is available.\n",
        "            images = images.to(device)\n",
        "            captions = captions.to(device)\n",
        "\n",
        "            # Pass the inputs through the CNN-RNN model.\n",
        "            features = clip_model.encode_image(images)\n",
        "            if linear_layer != None:\n",
        "                features = linear_layer(features)\n",
        "            outputs = decoder(features, captions)\n",
        "\n",
        "            # Calculate the batch loss.\n",
        "            loss = criterion(outputs.contiguous().view(-1, vocab_size), captions.view(-1))\n",
        "            val_loss += loss\n",
        "\n",
        "            if save_captions:\n",
        "                pred = decoder.sample(features.unsqueeze(1))\n",
        "                caption = utils.clean_sentence(pred, data_loader)\n",
        "                predicted_captions.append({\"image_id\": int(img_id), \"caption\": str(caption)})\n",
        "\n",
        "        if save_captions:\n",
        "            with open('val_captions.json', 'w') as fp:\n",
        "                json.dump(predicted_captions, fp)\n",
        "\n",
        "        val_loss /= total_step\n",
        "        print(\"Validation Loss for epoch \" + str(epoch) + ': ' + str(float(val_loss)))\n",
        "        return val_loss"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ea99-ggxw4Tr"
      },
      "source": [
        "###Clip Without Linear Layer\n",
        "import nltk\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torchvision import transforms\n",
        "import sys\n",
        "from data_loader import get_loader\n",
        "import math\n",
        "import ssl\n",
        "import torch.utils.data as data\n",
        "import numpy as np\n",
        "import os\n",
        "#Malikeh:from models import EncoderCNN, DecoderRNN\n",
        "import utils\n",
        "#Malikeh:import validation\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    ssl._create_default_https_context = ssl._create_unverified_context\n",
        "    nltk.download('punkt')\n",
        "\n",
        "    batch_size = 32            # batch size\n",
        "    vocab_threshold = 6        # minimum word count threshold\n",
        "    vocab_from_file = False    # if True, load existing vocab file\n",
        "    embed_size = 512           # dimensionality of image and word embeddings\n",
        "    hidden_size = 512          # number of features in hidden state of the RNN decoder\n",
        "    num_epochs = 1             # number of training epochs (1 for testing)\n",
        "    save_every = 1             # determines frequency of saving model weights\n",
        "    print_every = 200          # determines window for printing average loss\n",
        "    log_file = 'training_log.txt'       # name of file with saved training loss and perplexity\n",
        "\n",
        "    transform_train = transforms.Compose([\n",
        "        transforms.Resize(256),                          # smaller edge of image resized to 256\n",
        "        transforms.RandomCrop(224),                      # get 224x224 crop from random location\n",
        "        transforms.RandomHorizontalFlip(),               # horizontally flip image with probability=0.5\n",
        "        transforms.ToTensor(),                           # convert the PIL Image to a tensor\n",
        "        transforms.Normalize((0.485, 0.456, 0.406),      # normalize image for pre-trained model\n",
        "                             (0.229, 0.224, 0.225))])\n",
        "\n",
        "    # Build data loader.\n",
        "    data_loader = get_loader(transform=transform_train, mode='train', batch_size=batch_size, vocab_threshold=vocab_threshold,\n",
        "                             vocab_from_file=vocab_from_file)\n",
        "    val_data_loader = get_loader(transform=transform_train, mode='val')\n",
        "\n",
        "    # The size of the vocabulary.\n",
        "    vocab_size = len(data_loader.dataset.vocab)\n",
        "    print(vocab_size)\n",
        "\n",
        "    # Initialize the encoder and decoder.\n",
        "    #Malikeh:encoder = EncoderCNN(embed_size)\n",
        "    decoder = DecoderRNN(embed_size, hidden_size, vocab_size)\n",
        "\n",
        "    # Move models to GPU if CUDA is available.\n",
        "    device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\n",
        "    #Malikeh:encoder.to(device)\n",
        "    decoder.to(device)\n",
        "\n",
        "    # Define the loss function.\n",
        "    criterion = nn.CrossEntropyLoss().cuda() if torch.cuda.is_available() else nn.CrossEntropyLoss()\n",
        "\n",
        "    params = list(decoder.parameters()) #Malikeh + list(encoder.embed.parameters())\n",
        "\n",
        "    optimizer = torch.optim.Adam(params, lr=0.001, betas=(0.9, 0.999), eps=1e-08)\n",
        "    # optimizer = torch.optim.Adam(params, lr=0.01, betas=(0.9, 0.999), eps=1e-08)\n",
        "    #optimizer = torch.optim.RMSprop(params, lr=0.01, alpha=0.99, eps=1e-08)\n",
        "\n",
        "    # Set the total number of training steps per epoch.\n",
        "    total_step = math.ceil(len(data_loader.dataset.caption_lengths) / data_loader.batch_sampler.batch_size)\n",
        "    #total_step = 1\n",
        "\n",
        "    # Open the training log file.\n",
        "    f = open(log_file, 'w')\n",
        "\n",
        "    # Collect losses in these arrays\n",
        "    training_loss_per_epoch = []\n",
        "    val_loss_per_epoch = []\n",
        "\n",
        "    for epoch in range(1, num_epochs + 1):\n",
        "        avg_batch_loss = 0\n",
        "        avg_training_loss = 0\n",
        "\n",
        "        #Malikeh:encoder.train()\n",
        "        decoder.train()\n",
        "\n",
        "        for i_step in range(1, total_step + 1):\n",
        "\n",
        "            # Randomly sample a caption length, and sample indices with that length.\n",
        "            indices = data_loader.dataset.get_train_indices()\n",
        "            # Create and assign a batch sampler to retrieve a batch with the sampled indices.\n",
        "            new_sampler = data.sampler.SubsetRandomSampler(indices=indices)\n",
        "            data_loader.batch_sampler.sampler = new_sampler\n",
        "\n",
        "            # Obtain the batch.\n",
        "            images, captions = next(iter(data_loader))\n",
        "\n",
        "            # Move batch of images and captions to GPU if CUDA is available.\n",
        "            images = images.to(device)\n",
        "            captions = captions.to(device)\n",
        "\n",
        "            # Zero the gradients.\n",
        "            decoder.zero_grad()\n",
        "            #Malikeh:encoder.zero_grad()\n",
        "\n",
        "            # Pass the inputs through the CNN-RNN model.\n",
        "            #Malikeh: features = encoder(images)\n",
        "            with torch.no_grad():\n",
        "              features = clip_model.encode_image(images)\n",
        "            outputs = decoder(features, captions)\n",
        "\n",
        "            # Calculate the batch loss.\n",
        "            #         print(\"outputs.shape: \", outputs.shape)\n",
        "            loss = criterion(outputs.contiguous().view(-1, vocab_size), captions.view(-1))\n",
        "            avg_batch_loss += loss\n",
        "            avg_training_loss += loss\n",
        "\n",
        "            # Backward pass.\n",
        "            loss.backward()\n",
        "\n",
        "            # Update the parameters in the optimizer.\n",
        "            optimizer.step()\n",
        "\n",
        "            # Get training statistics.\n",
        "            stats = 'Epoch [%d/%d], Step [%d/%d], Loss: %.4f, Perplexity: %5.4f' % (\n",
        "            epoch, num_epochs, i_step, total_step, loss.item(), np.exp(loss.item()))\n",
        "\n",
        "            # Print training statistics (on same line).\n",
        "            print('\\r' + stats, end=\"\")\n",
        "            sys.stdout.flush()\n",
        "\n",
        "            # Print training statistics to file.\n",
        "            f.write(stats + '\\n')\n",
        "            f.flush()\n",
        "\n",
        "            # Print training statistics (on different line).\n",
        "            if i_step % print_every == 0:\n",
        "                print('\\r' + stats)\n",
        "\n",
        "        # Save the weights.\n",
        "        if epoch % save_every == 0:\n",
        "            torch.save(decoder.state_dict(), os.path.join('./models', 'decoder-%d.pkl' % epoch))\n",
        "            #Malikeh:torch.save(encoder.state_dict(), os.path.join('./models', 'encoder-%d.pkl' % epoch))\n",
        "\n",
        "        avg_batch_loss /= total_step\n",
        "        training_loss_per_epoch.append(avg_batch_loss)\n",
        "\n",
        "        \n",
        "        print(\"\\nTraining Loss for epoch \" + str(epoch) + ': ' + str(float(avg_batch_loss)))\n",
        "        val_loss = validate(clip_model, decoder, criterion, val_data_loader, vocab_size, epoch, device = device, save_captions=False, linear_layer = None)\n",
        "        val_loss_per_epoch.append(val_loss)\n",
        "\n",
        "\n",
        "    # Close the training log file.\n",
        "    f.close()\n",
        "\n",
        "    utils.plotLosses(training_loss_per_epoch,\n",
        "                     val_loss_per_epoch,\n",
        "                     'Cross Entropy Loss (per Epoch)')\n",
        "\n",
        "\n",
        "    # test_data_loader = get_loader(transform=transform_train, mode='test')\n",
        "    # test(clip_model, decoder, test_data_loader, device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ymYX8aFw-Tky"
      },
      "source": [
        "import torch\n",
        "from torch import nn\n",
        "import torchvision\n",
        "\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "class Attention(nn.Module):\n",
        "    \"\"\"\n",
        "    Attention Network.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, encoder_dim, decoder_dim, attention_dim):\n",
        "        \"\"\"\n",
        "        :param encoder_dim: feature size of encoded images\n",
        "        :param decoder_dim: size of decoder's RNN\n",
        "        :param attention_dim: size of the attention network\n",
        "        \"\"\"\n",
        "        super(Attention, self).__init__()\n",
        "        self.encoder_att = nn.Linear(encoder_dim, attention_dim)  # linear layer to transform encoded image\n",
        "        self.decoder_att = nn.Linear(decoder_dim, attention_dim)  # linear layer to transform decoder's output\n",
        "        self.full_att = nn.Linear(attention_dim, 1)  # linear layer to calculate values to be softmax-ed\n",
        "        self.relu = nn.ReLU()\n",
        "        self.softmax = nn.Softmax(dim=1)  # softmax layer to calculate weights\n",
        "\n",
        "    def forward(self, encoder_out, decoder_hidden):\n",
        "        \"\"\"\n",
        "        Forward propagation.\n",
        "        :param encoder_out: encoded images, a tensor of dimension (batch_size, num_pixels, encoder_dim)\n",
        "        :param decoder_hidden: previous decoder output, a tensor of dimension (batch_size, decoder_dim)\n",
        "        :return: attention weighted encoding, weights\n",
        "        \"\"\"\n",
        "        att1 = self.encoder_att(encoder_out.cuda().float())  # (batch_size, num_pixels, attention_dim)\n",
        "        att2 = self.decoder_att(decoder_hidden.float())  # (batch_size, attention_dim)\n",
        "        att = self.full_att(self.relu(att1 + att2.unsqueeze(1)).float()).squeeze(2)  # (batch_size, num_pixels)\n",
        "        alpha = self.softmax(att)  # (batch_size, num_pixels)\n",
        "        attention_weighted_encoding = (encoder_out * alpha.unsqueeze(2)).sum(dim=1)  # (batch_size, encoder_dim)\n",
        "\n",
        "        return attention_weighted_encoding, alpha\n",
        "\n",
        "\n",
        "class DecoderWithAttention(nn.Module):\n",
        "    \"\"\"\n",
        "    Decoder.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, attention_dim, embed_dim, decoder_dim, vocab_size, encoder_dim=768, dropout=0.5):\n",
        "        \"\"\"\n",
        "        :param attention_dim: size of attention network\n",
        "        :param embed_dim: embedding size\n",
        "        :param decoder_dim: size of decoder's RNN\n",
        "        :param vocab_size: size of vocabulary\n",
        "        :param encoder_dim: feature size of encoded images\n",
        "        :param dropout: dropout\n",
        "        \"\"\"\n",
        "        super(DecoderWithAttention, self).__init__()\n",
        "\n",
        "        self.encoder_dim = encoder_dim\n",
        "        self.attention_dim = attention_dim\n",
        "        self.embed_dim = embed_dim\n",
        "        self.decoder_dim = decoder_dim\n",
        "        self.vocab_size = vocab_size\n",
        "        self.dropout = dropout\n",
        "\n",
        "        self.attention = Attention(encoder_dim, decoder_dim, attention_dim)  # attention network\n",
        "\n",
        "        self.embedding = nn.Embedding(vocab_size, embed_dim)  # embedding layer\n",
        "        self.dropout = nn.Dropout(p=self.dropout)\n",
        "        self.decode_step = nn.LSTMCell(embed_dim + encoder_dim, decoder_dim, bias=True)  # decoding LSTMCell\n",
        "        self.init_h = nn.Linear(encoder_dim, decoder_dim)  # linear layer to find initial hidden state of LSTMCell\n",
        "        self.init_c = nn.Linear(encoder_dim, decoder_dim)  # linear layer to find initial cell state of LSTMCell\n",
        "        self.f_beta = nn.Linear(decoder_dim, encoder_dim)  # linear layer to create a sigmoid-activated gate\n",
        "        self.sigmoid = nn.Sigmoid()\n",
        "        self.fc = nn.Linear(decoder_dim, vocab_size)  # linear layer to find scores over vocabulary\n",
        "        self.init_weights()  # initialize some layers with the uniform distribution\n",
        "\n",
        "    def init_weights(self):\n",
        "        \"\"\"\n",
        "        Initializes some parameters with values from the uniform distribution, for easier convergence.\n",
        "        \"\"\"\n",
        "        self.embedding.weight.data.uniform_(-0.1, 0.1)\n",
        "        self.fc.bias.data.fill_(0)\n",
        "        self.fc.weight.data.uniform_(-0.1, 0.1)\n",
        "\n",
        "    def init_hidden_state(self, encoder_out):\n",
        "        \"\"\"\n",
        "        Creates the initial hidden and cell states for the decoder's LSTM based on the encoded images.\n",
        "        :param encoder_out: encoded images, a tensor of dimension (batch_size, num_pixels, encoder_dim)\n",
        "        :return: hidden state, cell state\n",
        "        \"\"\"\n",
        "        mean_encoder_out = encoder_out.mean(dim=1)\n",
        "        h = self.init_h(mean_encoder_out.float())  # (batch_size, decoder_dim)\n",
        "        c = self.init_c(mean_encoder_out.float())\n",
        "        return h, c  \n",
        "\n",
        "    def forward(self, encoder_out, encoded_captions, caption_lengths):\n",
        "        \"\"\"\n",
        "        Forward propagation.\n",
        "        :param encoder_out: encoded images, a tensor of dimension (batch_size, enc_image_size, enc_image_size, encoder_dim)\n",
        "        :param encoded_captions: encoded captions, a tensor of dimension (batch_size, max_caption_length)\n",
        "        :param caption_lengths: caption lengths, a tensor of dimension (batch_size, 1)\n",
        "        :return: scores for vocabulary, sorted encoded captions, decode lengths, weights, sort indices\n",
        "        \"\"\"\n",
        "\n",
        "        batch_size = encoder_out.size(0)\n",
        "        encoder_dim = encoder_out.size(-1)\n",
        "        vocab_size = self.vocab_size\n",
        "\n",
        "        # Flatten image\n",
        "        encoder_out = encoder_out.view(batch_size, -1, encoder_dim)  # (batch_size, num_pixels, encoder_dim)\n",
        "        num_pixels = encoder_out.size(1)\n",
        "\n",
        "        # Sort input data by decreasing lengths; why? apparent below\n",
        "        caption_lengths, sort_ind = caption_lengths.squeeze(1).sort(dim=0, descending=True)\n",
        "        encoder_out = encoder_out[sort_ind]\n",
        "        encoded_captions = encoded_captions[sort_ind]\n",
        "\n",
        "        # Embedding\n",
        "        embeddings = self.embedding(encoded_captions)  # (batch_size, max_caption_length, embed_dim)\n",
        "\n",
        "        # Initialize LSTM state\n",
        "        h, c = self.init_hidden_state(encoder_out)  # (batch_size, decoder_dim)\n",
        "\n",
        "        # We won't decode at the <end> position, since we've finished generating as soon as we generate <end>\n",
        "        # So, decoding lengths are actual lengths - 1\n",
        "        decode_lengths = (caption_lengths - 1).tolist()\n",
        "\n",
        "        # Create tensors to hold word predicion scores and alphas\n",
        "        predictions = torch.zeros(batch_size, max(decode_lengths), vocab_size).to(device)\n",
        "        alphas = torch.zeros(batch_size, max(decode_lengths), num_pixels).to(device)\n",
        "\n",
        "        # At each time-step, decode by\n",
        "        # attention-weighing the encoder's output based on the decoder's previous hidden state output\n",
        "        # then generate a new word in the decoder with the previous word and the attention weighted encoding\n",
        "        for t in range(max(decode_lengths)):\n",
        "            batch_size_t = sum([l > t for l in decode_lengths])\n",
        "            attention_weighted_encoding, alpha = self.attention(encoder_out[:batch_size_t],\n",
        "                                                                h[:batch_size_t])\n",
        "            gate = self.sigmoid(self.f_beta(h[:batch_size_t]))  # gating scalar, (batch_size_t, encoder_dim)\n",
        "            attention_weighted_encoding = gate * attention_weighted_encoding\n",
        "            h, c = self.decode_step(\n",
        "                torch.cat([embeddings[:batch_size_t, t, :], attention_weighted_encoding], dim=1),\n",
        "                (h[:batch_size_t], c[:batch_size_t]))  # (batch_size_t, decoder_dim)\n",
        "            \n",
        "            preds = self.fc(self.dropout(h))  # (batch_size_t, vocab_size)\n",
        "            predictions[:batch_size_t, t, :] = preds\n",
        "            alphas[:batch_size_t, t, :] = alpha\n",
        "\n",
        "        return predictions, encoded_captions, decode_lengths, alphas, sort_ind"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OWIt8Chru7W-"
      },
      "source": [
        "def get_clip_features(x):\n",
        "  x1 = clip_model.visual.conv1(x.half()) #torch.Size([1, 768, 7, 7])\n",
        "  x2 = x1.reshape(x1.shape[0], x1.shape[1], -1) #torch.Size([1, 768, 49])\n",
        "  x3 = x2.permute(0, 2, 1) #torch.Size([1, 768, 49])\n",
        "  x4 = torch.cat([clip_model.visual.class_embedding.to(x3.dtype) + torch.zeros(x3.shape[0], 1, x3.shape[-1], dtype=x3.dtype, device=x3.device), x3], dim=1)\n",
        "  #torch.Size([1, 50, 768])\n",
        "  x5 = x4 + clip_model.visual.positional_embedding.to(x4.dtype) #torch.Size([1, 50, 768])\n",
        "  x6 = clip_model.visual.ln_pre(x5) #torch.Size([1, 50, 768])\n",
        "  x7 = x6.permute(1, 0, 2)  # NLD -> LND #torch.Size([50, 1, 768])\n",
        "  x8 = clip_model.visual.transformer(x7) #torch.Size([50, 1, 768])\n",
        "  x9 = x8.permute(1, 0, 2)  # LND -> NLD torch.Size([1, 50, 768])\n",
        "  return x9"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5r3WQdYQfxSM"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.utils.data as data\n",
        "from data_loader import get_loader\n",
        "from torchvision import transforms\n",
        "#Malikeh: from models import EncoderCNN, DecoderRNN\n",
        "import math\n",
        "import utils\n",
        "import json\n",
        "\n",
        "def validate(clip_model, decoder, criterion, data_loader, vocab_size, epoch, device='cpu', save_captions=False, linear_layer = None):\n",
        "    with torch.no_grad():\n",
        "        #Malikeh: encoder.eval()\n",
        "        if linear_layer != None:\n",
        "          linear_layer.eval()\n",
        "        decoder.eval()\n",
        "        val_loss = 0\n",
        "\n",
        "        total_step = len(data_loader.dataset.paths) #number of images in val dataset\n",
        "        predicted_captions = []\n",
        "        print(\"Running Validation...\")\n",
        "        for batch in data_loader:\n",
        "            # Obtain the batch.\n",
        "            images, captions, img_id = batch #next(iter(data_loader))\n",
        "            # Move batch of images and captions to GPU if CUDA is available.\n",
        "            images = images.to(device)\n",
        "            captions = captions.to(device)\n",
        "\n",
        "            # Pass the inputs through the CNN-RNN model.\n",
        "            features = get_clip_features(images)\n",
        "            if linear_layer != None:\n",
        "                features = linear_layer(features)\n",
        "            #outputs = decoder(features, captions)\n",
        "            caplens = torch.tensor([captions.shape[1]]).reshape(1,1)\n",
        "            caplens = caplens.to(device)\n",
        "\n",
        "            scores, caps_sorted, decode_lengths, alphas, sort_ind = decoder(features, captions, caplens)\n",
        "\n",
        "            # Since we decoded starting with <start>, the targets are all words after <start>, up to <end>\n",
        "            targets = caps_sorted[:, 1:]\n",
        "\n",
        "            # Remove timesteps that we didn't decode at, or are pads\n",
        "            # pack_padded_sequence is an easy trick to do this\n",
        "            scores_copy = scores.clone()\n",
        "            scores = torch.nn.utils.rnn.pack_padded_sequence(scores, decode_lengths, batch_first=True)\n",
        "            targets = torch.nn.utils.rnn.pack_padded_sequence(targets, decode_lengths, batch_first=True)\n",
        "\n",
        "\n",
        "            # Calculate the batch loss.\n",
        "            loss = criterion(scores.data, targets.data)\n",
        "            #loss = criterion(outputs.contiguous().view(-1, vocab_size), captions.view(-1))\n",
        "            val_loss += loss\n",
        "\n",
        "        val_loss /= total_step\n",
        "        print(\"Validation Loss for epoch \" + str(epoch) + ': ' + str(float(val_loss)))\n",
        "        return val_loss"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "NlcwyqIYeJ7Z",
        "outputId": "bfb02d32-c8ad-4801-f767-81763229caba"
      },
      "source": [
        "###Clip With Attention\n",
        "import nltk\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torchvision import transforms\n",
        "import sys\n",
        "from data_loader import get_loader\n",
        "import math\n",
        "import ssl\n",
        "import torch.utils.data as data\n",
        "import numpy as np\n",
        "import os\n",
        "#Malikeh:from models import EncoderCNN, DecoderRNN\n",
        "import utils\n",
        "#Malikeh:import validation\n",
        "\n",
        "def clip_gradient(optimizer, grad_clip):\n",
        "    \"\"\"\n",
        "    Clips gradients computed during backpropagation to avoid explosion of gradients.\n",
        "    :param optimizer: optimizer with the gradients to be clipped\n",
        "    :param grad_clip: clip value\n",
        "    \"\"\"\n",
        "    for group in optimizer.param_groups:\n",
        "        for param in group['params']:\n",
        "            if param.grad is not None:\n",
        "                param.grad.data.clamp_(-grad_clip, grad_clip)\n",
        "                \n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    ssl._create_default_https_context = ssl._create_unverified_context\n",
        "    nltk.download('punkt')\n",
        "\n",
        "    batch_size = 32            # batch size\n",
        "    vocab_threshold = 6        # minimum word count threshold\n",
        "    vocab_from_file = False    # if True, load existing vocab file\n",
        "    embed_size = 512           # dimensionality of image and word embeddings\n",
        "    hidden_size = 512          # number of image_features in hidden state of the RNN decoder\n",
        "    num_epochs = 5             # number of training epochs (1 for testing)\n",
        "    save_every = 1             # determines frequency of saving model weights\n",
        "    print_every = 200          # determines window for printing average loss\n",
        "    log_file = 'training_log.txt'       # name of file with saved training loss and perplexity\n",
        "\n",
        "    attention_dim = 512\n",
        "    dropout = 0.5\n",
        "    alpha_c = 1.\n",
        "    encoder_dim=512\n",
        "    grad_clip = 5.\n",
        "    fine_tune_encoder = True\n",
        "\n",
        "    transform_train = transforms.Compose([\n",
        "        transforms.Resize(256),                          # smaller edge of image resized to 256\n",
        "        transforms.RandomCrop(224),                      # get 224x224 crop from random location\n",
        "        transforms.RandomHorizontalFlip(),               # horizontally flip image with probability=0.5\n",
        "        transforms.ToTensor(),                           # convert the PIL Image to a tensor\n",
        "        transforms.Normalize((0.485, 0.456, 0.406),      # normalize image for pre-trained model\n",
        "                             (0.229, 0.224, 0.225))])\n",
        "\n",
        "    # Build data loader.\n",
        "    data_loader = get_loader(transform=transform_train, mode='train', batch_size=batch_size, vocab_threshold=vocab_threshold,\n",
        "                             vocab_from_file=vocab_from_file)\n",
        "    val_data_loader = get_loader(transform=transform_train, mode='val')\n",
        "\n",
        "    # The size of the vocabulary.\n",
        "    vocab_size = len(data_loader.dataset.vocab)\n",
        "\n",
        "    # Initialize the encoder and decoder.\n",
        "    # encoder = Encoder() #att-enc\n",
        "    # encoder.fine_tune(fine_tune_encoder)\n",
        "    #Malikeh: decoder = DecoderRNN(embed_size, hidden_size, vocab_size)\n",
        "    decoder = DecoderWithAttention(attention_dim=attention_dim,\n",
        "                                   embed_dim=embed_size,\n",
        "                                   decoder_dim=hidden_size,\n",
        "                                   vocab_size=vocab_size,\n",
        "                                   dropout=dropout)\n",
        "\n",
        "    # Move models to GPU if CUDA is available.\n",
        "    device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\n",
        "    #encoder.to(device) #att-enc\n",
        "    decoder.to(device)\n",
        "\n",
        "    # Define the loss function.\n",
        "    criterion = nn.CrossEntropyLoss().cuda() if torch.cuda.is_available() else nn.CrossEntropyLoss()\n",
        "\n",
        "    params = list(decoder.parameters()) #+ list(encoder.parameters()) #att-enc\n",
        "\n",
        "    #Malikeh: optimizer = torch.optim.Adam(params, lr=0.001, betas=(0.9, 0.999), eps=1e-08)\n",
        "    optimizer = torch.optim.Adam(params, lr=0.001, betas=(0.9, 0.999), eps=1e-08)\n",
        "    # optimizer = torch.optim.Adam(params, lr=0.01, betas=(0.9, 0.999), eps=1e-08)\n",
        "    # optimizer = torch.optim.RMSprop(params, lr=0.01, alpha=0.99, eps=1e-08)\n",
        "\n",
        "    # Set the total number of training steps per epoch.\n",
        "    total_step = math.ceil(len(data_loader.dataset.caption_lengths) / data_loader.batch_sampler.batch_size)\n",
        "    #total_step = 1\n",
        "\n",
        "    # Open the training log file.\n",
        "    f = open(log_file, 'w')\n",
        "\n",
        "    # Collect losses in these arrays\n",
        "    training_loss_per_epoch = []\n",
        "    val_loss_per_epoch = []\n",
        "\n",
        "    for epoch in range(1, num_epochs + 1):\n",
        "        avg_batch_loss = 0\n",
        "\n",
        "        #encoder.train() #att-enc\n",
        "        decoder.train()\n",
        "\n",
        "        for i_step in range(1, total_step + 1):\n",
        "\n",
        "            # Randomly sample a caption length, and sample indices with that length.\n",
        "            indices = data_loader.dataset.get_train_indices()\n",
        "            # Create and assign a batch sampler to retrieve a batch with the sampled indices.\n",
        "            new_sampler = data.sampler.SubsetRandomSampler(indices=indices)\n",
        "            data_loader.batch_sampler.sampler = new_sampler\n",
        "\n",
        "            # Obtain the batch.\n",
        "            images, captions = next(iter(data_loader))\n",
        "            \n",
        "\n",
        "            # Move batch of images and captions to GPU if CUDA is available.\n",
        "            images = images.to(device)\n",
        "            captions = captions.to(device)\n",
        "\n",
        "            # Zero the gradients.\n",
        "            decoder.zero_grad()\n",
        "            # encoder.zero_grad()\n",
        "\n",
        "            # Pass the inputs through the CNN-RNN model.\n",
        "            # image_features = encoder(images)\n",
        "            with torch.no_grad():\n",
        "              image_features = get_clip_features(images)\n",
        "              #image_features = clip_model.encode_image(images)\n",
        "            #Malikeh: outputs = decoder(image_features, captions)\n",
        "            caplens = [data_loader.dataset.caption_lengths[index] for index in indices] \n",
        "            caplens = torch.tensor(caplens).reshape(len(indices),1)\n",
        "            caplens = caplens.to(device)\n",
        "\n",
        "            scores, caps_sorted, decode_lengths, alphas, sort_ind = decoder(image_features, captions, caplens)\n",
        "\n",
        "\n",
        "            # Since we decoded starting with <start>, the targets are all words after <start>, up to <end>\n",
        "            targets = caps_sorted[:, 1:]\n",
        "\n",
        "            # Remove timesteps that we didn't decode at, or are pads\n",
        "            # pack_padded_sequence is an easy trick to do this\n",
        "            scores_copy = scores.clone()\n",
        "            scores = torch.nn.utils.rnn.pack_padded_sequence(scores, decode_lengths, batch_first=True)\n",
        "            targets = torch.nn.utils.rnn.pack_padded_sequence(targets, decode_lengths, batch_first=True)\n",
        "\n",
        "            # Calculate the batch loss.\n",
        "            #         print(\"outputs.shape: \", outputs.shape)\n",
        " \n",
        "            loss = criterion(scores.data, targets.data)\n",
        "            \n",
        "            avg_batch_loss += loss\n",
        "\n",
        "            loss += alpha_c * ((1. - alphas.sum(dim=1)) ** 2).mean()\n",
        "\n",
        "            # Backward pass.\n",
        "            loss.backward()\n",
        "\n",
        "            clip_gradient(optimizer, grad_clip)\n",
        "            \n",
        "\n",
        "            # Update the parameters in the optimizer.\n",
        "            optimizer.step()\n",
        "\n",
        "            # Get training statistics.\n",
        "            stats = 'Epoch [%d/%d], Step [%d/%d], Loss: %.4f' % (\n",
        "            epoch, num_epochs, i_step, total_step, loss.item())\n",
        "\n",
        "            # Print training statistics (on same line).\n",
        "            print('\\r' + stats, end=\"\")\n",
        "            sys.stdout.flush()\n",
        "\n",
        "            # Print training statistics to file.\n",
        "            f.write(stats + '\\n')\n",
        "            f.flush()\n",
        "\n",
        "            # Print training statistics (on different line).\n",
        "            if i_step % print_every == 0:\n",
        "                print('\\r' + stats)\n",
        "            \n",
        "\n",
        "        # Save the weights.\n",
        "        if epoch % save_every == 0:\n",
        "            torch.save(decoder.state_dict(), os.path.join('./models', 'decoder-%d.pkl' % epoch))\n",
        "            #Malikeh:torch.save(encoder.state_dict(), os.path.join('./models', 'encoder-%d.pkl' % epoch))\n",
        "\n",
        "        avg_batch_loss /= total_step\n",
        "        training_loss_per_epoch.append(avg_batch_loss)\n",
        "\n",
        "        \n",
        "        print(\"\\nTraining Loss for epoch \" + str(epoch) + ': ' + str(float(avg_batch_loss)))\n",
        "        val_loss = validate(clip_model, decoder, criterion, val_data_loader, vocab_size, epoch, device = device, save_captions=False, linear_layer = None)\n",
        "        val_loss_per_epoch.append(val_loss)\n",
        "\n",
        "\n",
        "    # Close the training log file.\n",
        "    f.close()\n",
        "\n",
        "    utils.plotLosses(training_loss_per_epoch,\n",
        "                     val_loss_per_epoch,\n",
        "                     'Cross Entropy Loss (per Epoch)')\n",
        "\n",
        "\n",
        "    # test_data_loader = get_loader(transform=transform_train, mode='test')\n",
        "    # test(clip_model, decoder, test_data_loader, device)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "loading annotations into memory...\n",
            "Done (t=0.64s)\n",
            "creating index...\n",
            "index created! imgs = 23431, anns = 100575\n",
            "[0/100575] Tokenizing captions...\n",
            "[100000/100575] Tokenizing captions...\n",
            "loading annotations into memory...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r  0%|          | 0/100575 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Done (t=0.24s)\n",
            "creating index...\n",
            "index created! imgs = 23431, anns = 100575\n",
            "Obtaining caption lengths...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 100575/100575 [00:10<00:00, 9354.98it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Vocabulary successfully loaded from vocab.pkl file!\n",
            "loading annotations into memory...\n",
            "Done (t=0.20s)\n",
            "creating index...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "  3%|▎         | 934/33145 [00:00<00:03, 9338.76it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "index created! imgs = 7750, anns = 33145\n",
            "Obtaining caption lengths...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 33145/33145 [00:03<00:00, 9450.70it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [1/5], Step [200/3143], Loss: 4.7409\n",
            "Epoch [1/5], Step [400/3143], Loss: 4.8517\n",
            "Epoch [1/5], Step [600/3143], Loss: 4.7153\n",
            "Epoch [1/5], Step [800/3143], Loss: 4.3040\n",
            "Epoch [1/5], Step [1000/3143], Loss: 4.8538\n",
            "Epoch [1/5], Step [1200/3143], Loss: 4.0638\n",
            "Epoch [1/5], Step [1400/3143], Loss: 4.7034\n",
            "Epoch [1/5], Step [1600/3143], Loss: 4.2157\n",
            "Epoch [1/5], Step [1800/3143], Loss: 4.6249\n",
            "Epoch [1/5], Step [2000/3143], Loss: 4.4693\n",
            "Epoch [1/5], Step [2200/3143], Loss: 4.1417\n",
            "Epoch [1/5], Step [2400/3143], Loss: 4.0952\n",
            "Epoch [1/5], Step [2600/3143], Loss: 4.2739\n",
            "Epoch [1/5], Step [2800/3143], Loss: 4.3693\n",
            "Epoch [1/5], Step [3000/3143], Loss: 3.9678\n",
            "Epoch [1/5], Step [3143/3143], Loss: 4.1132\n",
            "Training Loss for epoch 1: 3.8724825382232666\n",
            "Running Validation...\n",
            "Validation Loss for epoch 1: 4.651616096496582\n",
            "Epoch [2/5], Step [200/3143], Loss: 3.7792\n",
            "Epoch [2/5], Step [400/3143], Loss: 4.0902\n",
            "Epoch [2/5], Step [600/3143], Loss: 3.6627\n",
            "Epoch [2/5], Step [800/3143], Loss: 3.7513\n",
            "Epoch [2/5], Step [1000/3143], Loss: 4.2731\n",
            "Epoch [2/5], Step [1200/3143], Loss: 3.7292\n",
            "Epoch [2/5], Step [1400/3143], Loss: 4.0366\n",
            "Epoch [2/5], Step [1600/3143], Loss: 3.9708\n",
            "Epoch [2/5], Step [1800/3143], Loss: 3.4851\n",
            "Epoch [2/5], Step [2000/3143], Loss: 3.6514\n",
            "Epoch [2/5], Step [2200/3143], Loss: 4.0710\n",
            "Epoch [2/5], Step [2400/3143], Loss: 3.4409\n",
            "Epoch [2/5], Step [2600/3143], Loss: 4.0037\n",
            "Epoch [2/5], Step [2800/3143], Loss: 3.7322\n",
            "Epoch [2/5], Step [3000/3143], Loss: 4.0506\n",
            "Epoch [2/5], Step [3143/3143], Loss: 3.6916\n",
            "Training Loss for epoch 2: 3.2557592391967773\n",
            "Running Validation...\n",
            "Validation Loss for epoch 2: 4.650144100189209\n",
            "Epoch [3/5], Step [200/3143], Loss: 3.7858\n",
            "Epoch [3/5], Step [400/3143], Loss: 3.6788\n",
            "Epoch [3/5], Step [600/3143], Loss: 3.5691\n",
            "Epoch [3/5], Step [800/3143], Loss: 3.8178\n",
            "Epoch [3/5], Step [1000/3143], Loss: 3.7697\n",
            "Epoch [3/5], Step [1200/3143], Loss: 3.8784\n",
            "Epoch [3/5], Step [1400/3143], Loss: 3.3156\n",
            "Epoch [3/5], Step [1600/3143], Loss: 3.6895\n",
            "Epoch [3/5], Step [1800/3143], Loss: 3.5150\n",
            "Epoch [3/5], Step [2000/3143], Loss: 3.5123\n",
            "Epoch [3/5], Step [2200/3143], Loss: 3.9954\n",
            "Epoch [3/5], Step [2400/3143], Loss: 3.9180\n",
            "Epoch [3/5], Step [2600/3143], Loss: 3.5604\n",
            "Epoch [3/5], Step [2800/3143], Loss: 3.6350\n",
            "Epoch [3/5], Step [3000/3143], Loss: 3.6434\n",
            "Epoch [3/5], Step [3143/3143], Loss: 3.5227\n",
            "Training Loss for epoch 3: 3.0523741245269775\n",
            "Running Validation...\n",
            "Validation Loss for epoch 3: 4.408127784729004\n",
            "Epoch [4/5], Step [200/3143], Loss: 3.5532\n",
            "Epoch [4/5], Step [400/3143], Loss: 3.5206\n",
            "Epoch [4/5], Step [600/3143], Loss: 3.4521\n",
            "Epoch [4/5], Step [800/3143], Loss: 3.2871\n",
            "Epoch [4/5], Step [1000/3143], Loss: 3.3661\n",
            "Epoch [4/5], Step [1200/3143], Loss: 3.4460\n",
            "Epoch [4/5], Step [1400/3143], Loss: 3.4313\n",
            "Epoch [4/5], Step [1600/3143], Loss: 3.7750\n",
            "Epoch [4/5], Step [1800/3143], Loss: 3.6470\n",
            "Epoch [4/5], Step [2000/3143], Loss: 3.3601\n",
            "Epoch [4/5], Step [2200/3143], Loss: 3.6185\n",
            "Epoch [4/5], Step [2400/3143], Loss: 3.3269\n",
            "Epoch [4/5], Step [2600/3143], Loss: 3.3619\n",
            "Epoch [4/5], Step [2800/3143], Loss: 3.3262\n",
            "Epoch [4/5], Step [3000/3143], Loss: 3.4191\n",
            "Epoch [4/5], Step [3143/3143], Loss: 3.6353\n",
            "Training Loss for epoch 4: 2.9163501262664795\n",
            "Running Validation...\n",
            "Validation Loss for epoch 4: 4.456430435180664\n",
            "Epoch [5/5], Step [200/3143], Loss: 3.1417\n",
            "Epoch [5/5], Step [400/3143], Loss: 3.8200\n",
            "Epoch [5/5], Step [600/3143], Loss: 3.3572\n",
            "Epoch [5/5], Step [800/3143], Loss: 3.5046\n",
            "Epoch [5/5], Step [1000/3143], Loss: 3.5043\n",
            "Epoch [5/5], Step [1200/3143], Loss: 3.2457\n",
            "Epoch [5/5], Step [1400/3143], Loss: 3.3165\n",
            "Epoch [5/5], Step [1600/3143], Loss: 3.4888\n",
            "Epoch [5/5], Step [1800/3143], Loss: 3.2251\n",
            "Epoch [5/5], Step [2000/3143], Loss: 3.2397\n",
            "Epoch [5/5], Step [2200/3143], Loss: 3.5427\n",
            "Epoch [5/5], Step [2400/3143], Loss: 3.0406\n",
            "Epoch [5/5], Step [2600/3143], Loss: 3.5680\n",
            "Epoch [5/5], Step [2800/3143], Loss: 3.4652\n",
            "Epoch [5/5], Step [3000/3143], Loss: 3.0333\n",
            "Epoch [5/5], Step [3143/3143], Loss: 3.4057\n",
            "Training Loss for epoch 5: 2.8139476776123047\n",
            "Running Validation...\n",
            "Validation Loss for epoch 5: 4.51342248916626\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3xUdbr48c+TQhpJSKOEFooUAaVEpKgUdS8qAoKKBRWvyoruRa+uq+6uBda9V11+LtfeXVbXwqLsYlt1lWIBJCAgIAJSBAKkAIFAElKe3x/nJEzCTApkMiHzvF+veeXMOd8555mT5DzzLec7oqoYY4wxVYUEOgBjjDGNkyUIY4wxXlmCMMYY45UlCGOMMV5ZgjDGGOOVJQhjjDFeWYIwpgkRka9FpF+g4/AXEVkoIjf72NZKRH4QkYiGjqupsgQRpETkGhHJEJF8EdktIh+LyDkBjGebiBS48ZQ/nq7la31eNBqaiEwWka8CdOxLgUOq+l0DHe8vInK0yu9sdUMc2xtV3QssAKYEKoamxhJEEBKRu4BZwP8ArYAOwLPAWB/lwxootEtVtbnH41f1sdMGjD/QbgVe98eOqzmHj1f5nZ3pj+PXwd+AXwY4hibDEkSQEZF4YAZwu6q+p6qHVbVYVd9X1XvcMg+LyFwReUNEDgKTRSRVROaLyD4R2Swit3jsc6BbGzkoIntF5Al3faS7j1wROSAiy0Wk1QnEPFlEvhKRmSKyX0S2ishF7rY/AucCT3vWOkREReR2EdkEbHLX3eLGvs99L6kex1ARmSYiW0QkR0T+JCIhItLMLd/Ho2xLETkiIil1fB9D3HOQ5/4cUuU9bhGRQ+77u9Zd31VEFrmvyRGRd3zsuxkwEljksa789/iOu9+VInKmx/ZUEXlXRLLdY07z8tqKv4E6vtc095xOEZFMt5b6a4/tESIyy92W6S5HeGwfKyKr3L+pn0RklMfuO4rTlHZIRD4VkWSPbcuAziLSsS7xGh9U1R5B9ABGASVAWDVlHgaKgXE4HyKigMU4tYxIoC+QDYx0yy8BrnOXmwOD3OVfAu8D0UAoMACI83HMbcAFPrZNduO5xd3PVCATEHf7QuDmKq9R4DMg0Y1/JJAD9AcigKeAxVXKL3DLdwA2lu/Tfd+PeZS9A3i/mli/8rI+EdgPXAeEAVe7z5OAGOAg0N0t2wbo5S6/BfzO/T1EAuf4OG4v4LCP3+PlQDjwa2CruxwCrAAeBJoBnYEtwH/4+hvwcsy/AI/4iCfNPadvue+vj/s3c4G7fQawFGgJpADfAH9wtw0E8oAL3WO3BXp4/K5/Arq5v9eFwKNVjr0GGBPo/7Wm8LAaRPBJAnJUtaSGcktU9R+qWgYkA0OBe1W1UFVXAS8D17tli4GuIpKsqvmqutRjfRLQVVVLVXWFqh6s5pj/cGsa5Y9bPLZtV9WXVLUUmI1zEa2pNvK/qrpPVQuAa4FXVXWlqhYB9wODRSTNo/xjbvmfcZrgrnbXzwauFhFxn19H3ZtyLgE2qerrqlqiqm8BG4BL3e1lQG8RiVLV3aq6zl1fDHQEUt1z76t/owVwyMv6Fao6V1WLgSdwkswg4CwgRVVnqOpRVd0CvARc5fHair8B9xx68+sqv7PZVbZPV6eW+j3wGsfO6bXADFXNUtVsYDrOeQW4Ced39Zl77F2qusFjn6+p6kY3pjk4H1g8HXLPhzlJliCCTy6QXIt2+R0ey6nAPlX1vABtx/lkB84/dDdgg9t0Mtpd/zrwCfC224zwuIiEV3PMcarawuPxkse2PeULqnrEXWxex/ew3WMf+Tjnoq2P8tvd16Cqy4AjwHAR6QF0BebXcOyqKh3f4xhtVfUwMBGnD2G3iHzoHgfgN4AA34rIOhH5Tx/73w/Eellf8Z7cZL/TjaUjkOp5cQd+S+Wku4OazazyO7vB1/HxOKccfz48t7XHqSX4ssdj+QjH/x3EAgdqEbupgSWI4LMEKMJpOqiO5zS/mUCiiHhegDoAuwBUdZOqXo3TXPAYMFdEYtTp25iuqqcDQ4DRHKt11CdfUxJXfQ8V7dIiEoNTu9nlUaa9x3IH9zXlZgOTcD7lzlXVwjrGWOn4HscoP4efqOqFODWjDTif5lHVPap6i6qm4jTZPSsiXb3sf7PztqRtlfUV70lEQoB2biw7gK1VLu6xqnqxx2vrY6pnX+e06vnw3LYD6HIiB3M/+HQFAjaaqimxBBFkVDUPp935GREZJyLRIhIuIheJyOM+XrMDp434f8XpeD4Dp9bwBoCITBKRFPcTavkntzIRGSEifUQkFKeNvRinKaW+7cVpQ6/OW8CNItLX7Qz9H2CZqm7zKHOPiCSISHucfgbPDuE3gMtwksRfaziWuOep4gF8BHQTZ3hxmIhMBE4HPhBn/P5YN2kVAfm450lErhCRdu5+9+NctI87h6p6FPg3MKzKpgEiMt69cN7p7n8p8C1wSETuFZEoEQkVkd4iclYN762uHnD/xnoBN3LsnL4F/F5EUtxO5gdx/56AV3B+V+eLM1CgrUeNqiYDgW2qWrW2Zk6AJYggpKr/D7gL+D1Ox+EO4FfAP6p52dU4HY+ZwDzgIVX9t7ttFLBORPKB/wOuctuHWwNzcZLDDzgjbKpru39fKo+pn1fLt/R/wOXijHB60lsBN9YHgHeB3TifUK+qUuyfOB23q4APcS5U5a/fAazEuUB/WUM8Q4CCKo88nBrU3ThNW78BRqtqDs7/4V0453YfzkV+qruvs4Bl7rmdD9zh9hd48wLH2vE939NEjnWQj3drdqVuPH1xOq5zcPqV4mt4b1X9psrvLKfK9kU4tZvPcZqjPnXXPwJk4HQof49zbh8BUNVvcZLJn3HO2yKOr335ci3wfB3fg/GhfBSIMUFNRBQ4TVU3V1PmVSBTVX/fcJHVjYh8DfxKVb8TkYdxBghMCkAcabgjpmoxIKK+jtkSJ5n0O4EmQONFsNxAZMxJcS9444FGPY2Fqg4NdAyBoqpZQM9Ax9GUWBOTMTUQkT8Aa4E/qerWQMdjTEOxJiZjjDFeWQ3CGGOMV02qDyI5OVnT0tICHYYxxpwyVqxYkaOqXucVa1IJIi0tjYyMjECHYYwxpwwR8XnPiDUxGWOM8coShDHGGK8sQRhjjPHKEoQxxhivLEEYY4zxyhKEMcYYryxBGGOM8apJ3QdxwhY9DqXFUPGNku7PGp9z7HmdXyv18No6HqfejivQsge06l05FmNMk2IJAuCrWVB8ONBRnHqSu0Gv8dB7AqR0C3Q0xph6ZgkC4HeZx68rn8SwYjLD6p7Xpaz7vNZlOX57XY5Tb6/1WC4rge1fw7p5sOgxWPQotOoDvS9zEkZiJ4wxp74mNZtrenq62lQbDezgblj/D1j7Huz81lmX2t+pVfS6DOKrfkWyMaYxEZEVqprudZslCFNvDvzs1CrWvgu73e+M7zDYSRanj4XmLQMbnzHmOJYgTMPL/cmpVax9F7J/AAmBtHOh93joOQaiEwMdoTEGSxAm0Pauh3Vusti3BULCoMtIp7+ix8UQGR/oCI0JWtUlCOukNv7X6nTnMeJ3TtPT2nedpqhNt0JoBJx2oVOz6DYKmsUEOlpjjMsShGk4IpDa13lcOAN2LneaodbNgw0fQHi0kyR6T4CuF0B4ZKAjNiaoWROTCbyyUtj+jdMMtf6fcCQXIuKgxyVOM1SXERAaHugojWmSrA/CnDpKS2DrIqdm8cP7UJQHUQlOx3bv8U5Hd0hooKM0psmwBGFOTSVF8NMXTp/Fho+cu91jWjpDZntPgPZnQ4hNJ2bMybBOanNqCouA7hc5j6NHYNOnTrL47nVY/hLEtXVuxus93rk5z+aFMqZeWQ3CnHqKDsGPHzvNUJv/DWXFkJDmzgs13iYRNKYOqqtB+L1+LiKhIvKdiHzgZdtkEckWkVXu42aPbTeIyCb3cYO/4zSnkIhYOONKuOZtuGcTjH0GEjvD1/8Hz58DzwyEhY9C9sZAR2rMKc3vNQgRuQtIB+JUdXSVbZOBdFX9VZX1iUCG+zoFVgADVHV/dceyGkSQO5zjjIJa+54zmSDqTiLo1iwS0gIdoTEnprjQGd13JMf5Oz+yz1k+kus8DwmFS/7fCe06YH0QItIOuAT4I3BXHV76H8BnqrrP3c9nwCjgrXoP0jQdMclw1k3Oo2ISwXfh8+nOo+0ApxnKJhE0gaQKhXnuBd+9wHte7Cutd5eP5nvfl4RAVKLfPvz4u5N6FvAbILaaMhNE5DxgI/DfqroDaAvs8Ciz011nTO3EtYFBU53H/u3OzXjr3oNPf+c8bBJBU19Ki71f1CuWvXzqLyvxvq+wSIhOhpgk52dSV+eDT3SS84hJdtaXL0e28OtIPr8lCBEZDWSp6goRGe6j2PvAW6paJCK/BGYDI+t4nCnAFIAOHTqcRMSmyUroCOfc6Tw8JxH86Nfw8W/cSQQnQM9LbRLBYKcKRw+7F/XcKhd4d/nIvsqf+gvzfO8vssWxi3pCGrQbUPkCX+nCn9TopprxWx+EiPwvcB1QAkQCccB7qjrJR/lQYJ+qxovI1cBwVf2lu+0FYKGqVtvEZH0Qpk58TSLYewJ0vxgi4wIdoTlZZaVQsL/KBT7XvfjnVGnicS/8pUXe9xUS7v2iXvGJv3zZXR+VCKGN/06CgN8o59Ygfu2lk7qNqu52ly8D7lXVQW4n9Qqgv1t0JU4n9b7qjmMJwpwQ1cqTCObtsEkEG6viAu8XdW9NOYdznOSAj2tcRJxTY/S8qHttynEv/BGxTXL4dKO6UU5EZgAZqjofmCYiY3BqGfuAyQCquk9E/gAsd182o6bkYMwJs0kEG15ZmTONSsEB5yJe6P4sf+7rU7+v746XkGOf4KOToGXP6ptyopOcGzFNtexGOWN8qW4Swd4ToPPw4J5EUBWKj/i+yFf3vDAPn5/swUnK0cnOJ/zjPs0nHf+p38+dtU1ZwJuYGoolCOM3pcXuJILzvEwiOAHSzjl1JxEsLa7jRd5jufSo7/1KKES1cM5TpPvT23Nv26yW1mAsQRhTn3xNIthrnHOfRSAmESwrg6KDtbuoF1R57muMfbmIOI8LeS0u+OXPm2ibfVNjCcIYf/GcRHDTp1BSeOKTCKo6nbB1aaopf16YB1rme9+hEe4FvIaLetXnkfGnxEgcc+IsQRjTEKqbRDClh5dP8l4u+tU22YTU7qLubVt4VIOdBnNqaVSjmIxpssonETzjSudi/8MHTgf31/8HWnqsXLNY9yLuNtukdK/dp/pmsdYRaxqUJQhj/CEqAfpf5zwO50LBPveCHx/cI5/MKcUShDH+FuMOzzTmFGP1VWOMMV5ZgjDGGOOVJQhjjDFeWYIwxhjjlSUIY4wxXlmCMMYY45UlCGOMMV5ZgjDGGOOVJQhjjDFeWYIwxhjjlSUIY4wxXvk9QYhIqIh8JyIfeNl2l4isF5E1IvK5iHT02FYqIqvcx3x/x2mMMaayhpis7w7gByDOy7bvgHRVPSIiU4HHgYnutgJV7dsA8RljjPHCrzUIEWkHXAK87G27qi5Q1SPu06VAO3/GY4wxpvb83cQ0C/gNUM13IVa4CfjY43mkiGSIyFIRGefrRSIyxS2XkZ2dfZLhGmOMKee3BCEio4EsVV1Ri7KTgHTgTx6rO7pfg3cNMEtEunh7raq+qKrpqpqekpJSH6EbY4zBvzWIocAYEdkGvA2MFJE3qhYSkQuA3wFjVLWofL2q7nJ/bgEWAv38GKsxxpgq/JYgVPV+VW2nqmnAVcAXqjrJs4yI9ANewEkOWR7rE0Qkwl1Oxkk26/0VqzHGmOM1+FeOisgMIENV5+M0KTUH/i4iAD+r6higJ/CCiJThJLFHVdUShDHGNCBR1UDHUG/S09M1IyMj0GEYY8wpQ0RWuP29x7E7qY0xxnhlCcIYY4xXliCMMcZ4ZQnCGGOMV5YgjDHGeGUJwhhjjFeWIIwxxnhlCcIYY4xXliCMMcZ4ZQnCGGOMV5YgjDHGeGUJwhhjjFeWIIwxxnhlCcIYY4xXliCMMcZ4ZQnCGGOMV5YgjDHGeGUJwhhjjFd+TxAiEioi34nIB162RYjIOyKyWUSWiUiax7b73fU/ish/+DtOY4wxlTVEDeIO4Acf224C9qtqV+DPwGMAInI6cBXQCxgFPCsioQ0QqzHGGJdfE4SItAMuAV72UWQsMNtdngucLyLirn9bVYtUdSuwGRjoz1iNMcZU5u8axCzgN0CZj+1tgR0AqloC5AFJnutdO911xxGRKSKSISIZ2dnZ9RW3McYEPb8lCBEZDWSp6gp/HQNAVV9U1XRVTU9JSfHnoYwxJqj4swYxFBgjItuAt4GRIvJGlTK7gPYAIhIGxAO5nutd7dx1xhhjGojfEoSq3q+q7VQ1DafD+QtVnVSl2HzgBnf5creMuuuvckc5dQJOA771V6zGGGOOF9bQBxSRGUCGqs4HXgFeF5HNwD6cRIKqrhOROcB6oAS4XVVLGzpWY4wJZuJ8YG8a0tPTNSMjI9BhGGPMKUNEVqhqurdtdie1McYYryxBGGOM8coShDHGGK8sQRhjjPHKEoQxxhivLEEYY4zxyhIE0JSG+hpjTH0J+gRRUlrG1DdW8uGa3YEOxRhjGpWgTxBHikvJyS/i9jdX8sKin6w2YYwxrqBPEHGR4bxx89lc0qcN//vxBh7451pKSn3NTm6MMcGjwediaowiw0N56up+tEuI4oXFW9h9oJCnrulHdDM7PcaY4BX0NYhyISHC/Rf35A/jerPgxywmvrCUrEOFgQ7LGGMCxhJEFdcN6shL16ezOSufy575hk17DwU6JGOMCQhLEF6c37MVc345mKOlZUx47huW/JQb6JCMMabBWYLwoU+7eObdNoSWcZFc/+oy5n23M9AhGWNMg6pVghCRGBEJcZe7icgYEQn3b2iB1y4hmndvHcKAjgn89zurefqLTTYM1hgTNGpbg1gMRIpIW+BT4DrgL/4KqjGJjw5n9n8OZFzfVGZ+upH73/ueYhsGa4wJArUdxymqekREbgKeVdXHRWSVPwNrTCLCQvnzxL60T4zmqS82k5lXyDPX9CM2sslXoowxQay2NQgRkcHAtcCH7rrQGl4QKSLfishqEVknItO9lPmziKxyHxtF5IDHtlKPbfNr+4b8RUS4+xfdeWxCH77enMOVLyxlT54NgzXGNF21rUHcCdwPzFPVdSLSGVhQw2uKgJGqmu/2V3wlIh+r6tLyAqr63+XLIvJfQD+P1xeoat9axtdgJp7VgdbxUdz2xgoue/ZrXp18Fj3bxAU6LGOMqXe1qkGo6iJVHaOqj7md1TmqOq2G16iq5rtPw91HdT28VwNv1SaeQBvWLYW/3zoEVbji+SV8uSk70CEZY0y9q+0opjdFJE5EYoC1wHoRuacWrwt1+yqygM9UdZmPch2BTsAXHqsjRSRDRJaKyLhqjjHFLZeRnd1wF+rTU+OYd/sQ2iVEceNry5mTsaPBjm2MMQ2htn0Qp6vqQWAc8DHOxfy6ml6kqqVuM1E7YKCI9PZR9CpgrqqWeqzrqKrpwDXALBHp4uMYL6pquqqmp6Sk1PLt1I828VH8/dbBDO6SxG/mruGJT3+0YbDGmCajtgki3O1HGAfMV9Viqm8uqkRVD+D0WYzyUeQqqjQvqeou9+cWYCGV+ycajdjIcF6dfBZXDGjHk19s5u45qzlaYsNgjTGnvtomiBeAbUAMsNhtEjpY3QtEJEVEWrjLUcCFwAYv5XoACcASj3UJIhLhLicDQ4H1tYy1wYWHhvD45Wdw94XdeO+7XUx+7VvyCooDHZYxxpyU2nZSP6mqbVX1YrfzeTswooaXtQEWiMgaYDlOH8QHIjJDRMZ4lLsKeFsrt830BDJEZDVOzeNRVW20CQKcYbD/df5pPHHlmSzfto8rnv+GXQcKAh2WMcacMKlNm7mIxAMPAee5qxYBM1Q1z4+x1Vl6erpmZGQEOgy+2ZzDL99YQWR4KK9NPovebeMDHZIxxnglIivc/t7j1LaJ6VXgEHCl+zgIvFY/4TU9Q7om8+7UITQLDeHKF5awYENWoEMyxpg6q22C6KKqD6nqFvcxHejsz8BOdd1axTLvtiF0TonhptnL+duy7YEOyRhj6qS2CaJARM4pfyIiQwFrYK9By7hI3pkymGHdUvjdvLU8+vEGyspsGKwx5tRQ26k2bgX+6vZFAOwHbvBPSE1LTEQYL12fzkPz1/H8op/YdaCAmVecQURYtVNZGWNMwNUqQajqauBMEYlznx8UkTuBNf4MrqkICw3hkXG9aZ8YzaMfb2BvXiEvXj+AFtHNAh2aMcb4VKdvlFPVg+4d1QB3+SGeJktEuHVYF566uh+rdhxg/HPf8HPukUCHZYwxPp3MV45KvUURRC49M5U3bj6b3PyjXPbs16zacaDmFxljTACcTIKw3tYTNLBTIu/dNoToiFCuenEJn67bE+iQjDHmONUmCBE5JCIHvTwOAakNFGOT1CWlOfNuG0r31nH88o0VvPb11kCHZIwxlVSbIFQ1VlXjvDxiVbW2I6CMD8nNI3j7lkFc0LMV099fzx8+WG/DYI0xjcbJNDGZehDVLJTnJw1g8pA0XvlqK7f9bSWFxaU1v9AYY/zMEkQjEBoiPDymFw+MPp1P1u/h6peWkptfFOiwjDFBzhJEI3LTOZ147tr+rM88yPjnvmFrzuFAh2SMCWKWIBqZUb3b8NaUQRwqLGH8s1+TsW1foEMyxgQpSxCNUP8OCcy7bQgtoptxzcvL+HDN7kCHZIwJQpYgGqmOSTG8O3UIZ7SN5/Y3V/Li4p/s+66NMQ3KEkQjlhjTjDduPptL+rThfz7awIP/XEdJqX3ftTGmYdi9DI1cZHgoT13dj3YJUbyweAuZBwp46pp+RDezX50xxr/8VoMQkUgR+VZEVovIOhGZ7qXMZBHJFpFV7uNmj203iMgm9xHUU4uHhAj3X9yTP4ztxYIfs7jqxaVkHSoMdFjGmCbOn01MRcBIVT0T6AuMEpFBXsq9o6p93cfLACKSiPMd2GcDA4GHRCTBj7GeEq4bnMZL16ezaW8+lz3zDZuzDgU6JGNME+a3BKGOfPdpuPuobS/rfwCfqeo+Vd0PfAaM8kOYp5zze7binV8OoqikjPHPfsPSLbmBDskY00T5tZNaREJFZBWQhXPBX+al2AQRWSMic0WkvbuuLbDDo8xOd523Y0wRkQwRycjOzq7X+BurM9q1YN5tQ2gZF8l1ryzjH9/tCnRIxpgmyK8JQlVLVbUv0A4YKCK9qxR5H0hT1TNwagmzT+AYL6pquqqmp6SknHzQp4j2idG8e+sQBnRM4M53VvHMgs02DNYYU68aZJirqh4AFlClmUhVc1W1fNKhl4EB7vIuoL1H0XbuOuMhPjqc2f85kHF9U/nTJz9y/3vfU2zDYI0x9cSfo5hSRKSFuxwFXAhsqFKmjcfTMcAP7vInwC9EJMHtnP6Fu85UEREWyp8n9uW/Rnbl7eU7uGl2BvlFJYEOyxjTBPizBtEGWCAia4DlOH0QH4jIDBEZ45aZ5g6BXQ1MAyYDqOo+4A/u65YDM9x1xgsR4e5fdOexCX34enMOVz6/hD15NgzWGHNypCm1W6enp2tGRkagwwioRRuzue2NFcRFhfPajWfRo3VcoEMyxjRiIrJCVdO9bbOpNpqYYd1SmHPrYMpUueK5JXy1KSfQIRljTlGWIJqgXqnxzLttKG0Topj82rf8PWNHzS8yxpgqLEE0Uaktophz62AGd0ninrlreOKzjTYM1hhTJ5YgmrC4yHBenXwWVwxox5Ofb+Luv6/maIkNgzXG1I5NCdrEhYeG8PjlZ9A+MZonPtvInrxCnps0gPio8ECHZoxp5KwGEQREhGnnn8YTV57J8m37uOL5b9h1oCDQYRljGjlLEEFkfP92zL5xILvzCrnsma9Zuysv0CEZYxoxSxBBZkjXZN6dOoTw0BCufGEJCzZkBTokY0wjZQkiCHVrFcu824bQOSWGm/+awZvLfg50SMaYRsgSRJBqGRfJO1MGc95pyfx23vc89q8NlJXZMFhjzDGWIIJYTEQYL12fzjVnd+C5hT9xxzurKCopDXRYxphGwoa5Brmw0BD+OK43HRKjefTjDezNK+TF6wfQIrpZoEMzxgSY1SAMIsKtw7rw5NX9WLXjAOOf+4afc48EOixjTIBZgjAVxpyZyhs3n01u/lHGP/c1q3ccCHRIxpgAsgRhKhnYKZH3bhtCVLNQJr64hE/X7Ql0SMaYALEEYY7TJaU5700dSvfWcfzyjRX85eutgQ7JGBMAliCMVymxEbx9yyAu6NmKh99fzyMfrLdhsMYEGUsQxqeoZqE8P2kAk4ek8fJXW7n9zZUUFtswWGOChd8ShIhEisi3IrLa/d7p6V7K3CUi60VkjYh8LiIdPbaVisgq9zHfX3Ga6oWGCA+P6cUDo0/nX+v2cM1LS8nNLwp0WMaYBuDPGkQRMFJVzwT6AqNEZFCVMt8B6ap6BjAXeNxjW4Gq9nUfY/wYp6mFm87pxHPX9mdd5kEmPPcNGdv2WZOTMU2c326UU+fry/Ldp+HuQ6uUWeDxdCkwyV/xmJM3qncb3rwlklv+msHlzy8hMaYZw7qlMLx7CuedlkJCjN1cZ0xTIv78GkoRCQVWAF2BZ1T13mrKPg3sUdVH3OclwCqgBHhUVf/h43VTgCkAHTp0GLB9+/b6fRPmOHlHilm4MYuFP2azaGM2+w4fJUSgb/sWDO/ekhHdW9IrNY6QEAl0qMaYGojIClVN97qtIb6nWERaAPOA/1LVtV62TwJ+BQxT1SJ3XVtV3SUinYEvgPNV9afqjpOenq4ZGRn1/waMT6Vlyve78liwIYuFP2axZlceqpDcPIJh3VIY0SOFc7umEB9t32BnTGMU8AThBvEgcERVZ1ZZfwHwFE5y8PrlBCLyF+ADVZ1b3TEsQQReTn4RizdmV9Qu8gqKCQ0R+ndwahfDu6dweps4RKx2YUxjEJAEISIpQLGqHhCRKOBT4DFV/cCjTD+czulRqrrJY30CTjIpEpFkYAkwVlXXV3dMSxCNS2mZsmrHARb+mMWCH7NYu+sgAE07PxAAABkNSURBVK3i3NpF95YMPS2ZuEirXRgTKIFKEGcAs4FQnNFSc1R1hojMADJUdb6I/BvoA+x2X/azqo4RkSHAC0CZ+9pZqvpKTce0BNG4ZR0qZNGPTu1i8aZsDhWWEBYiDOiYwIgeTu2ie6tYq10Y04AaRRNTQ7AEceooKS1j5c/ltYtsftjt1C7axEdWNEUN7ZpM8wibkd4Yf7IEYRq9PXmFLNqYxYIN2Xy1OYf8ohLCQ4Wz0hIZ4SaMri2bW+3CmHpmCcKcUo6WlLFi+34W/ugMpf1x7yEA2raIYkSPFIZ3a8mQrklEN7PahTEnyxKEOaXtOlBQkSy+3pzDkaOlNAsN4ezOie59Fyl0So6x2oUxJ8AShGkyikpKydi2nwUbnJFRP2UfBqBDYjQjuqcwvEdLBndOIjI8NMCRGnNqsARhmqwd+44cq138lENhcRkRYSEM7pLE8G4pjOjRko5JMYEO05hGyxKECQqFxaV8u3UfC9yEsTXHqV10To5hWHfnvouBnRKtdmGMB0sQJihtyzlcMYx26ZZcikrKiAoPZUiXJIb3aMnwbim0T4wOdJjGBJQlCBP0Co6WsnRLbkXC+HnfEQC6tmxe0RSVnpZARJjVLkxwsQRhjAdVZUvOYRb+mM3CH7NYtmUfR0vLiGkWypCuyRX3XaS2iAp0qMb4XXUJwgaSm6AjInRJaU6XlObcdE4nDheVsOSn3Iq+i8/W7wWge6tYhrv3XaSnJRAeat/Qa4KL1SCM8aCqbM7Kr0gWy7fto7hUiY0IY2jXZOdGve4taRUXGehQjakX1sRkzAnKLyrh6805FUNpd+cVAtCzTZxz30X3lvTv0IIwq12YU5QlCGPqgary495DLNjg9F1kbN9PaZkSFxnGuac5X706rHsKLWOtdmFOHUGdIIqLi9m5cyeFhYUBisrURmRkJO3atSM8/NT5boi8guJKtYusQ0UA9G4b53Z0t6Rv+xaE2levmkYsqBPE1q1biY2NJSkpyebqaaRUldzcXA4dOkSnTp0CHc4JUVXW7z7Iwh+zWbAhi5U/76dMoUV0OOedlsKgzkn0bhtHt1axdqOeaVSCehRTYWEhaWlplhwaMREhKSmJ7OzsQIdywkSEXqnx9EqN5/YRXTlw5Chfbspxv3o1i/mrMwEICxG6tmzO6alx9E6Np1dqHKenxhFr36pnGqEmnyAASw6ngKb2O2oR3YxLz0zl0jNTKStTduw/wrrMg6zLzGNd5kEWb8zhvZW7Ksp3TIqmV2qcm2ScnymxEQF8B8YESYIwJpBCQoSOSTF0TIrh4j5tKtZnHSysSBprdx3k+115fPT9nortLWMj6N22PGE4SaNdQlSTS6am8fJbghCRSGAxEOEeZ66qPlSlTATwV2AAkAtMVNVt7rb7gZuAUmCaqn7ir1j96cCBA7z55pvcdtttdX7txRdfzJtvvkmLFi18lnnwwQc577zzuOCCC04mTADS0tLIyMggOTn5pPdlatYyLpKWcZGM6NGyYl1eQTHrPWoa6zLzWPhjFmVuV2FcZNixWkZbJ2l0To6xYbbGL/xZgygCRqpqvoiEA1+JyMequtSjzE3AflXtKiJXAY8BE0XkdOAqoBeQCvxbRLqpaqkf4/WLAwcO8Oyzz3pNECUlJYSF+f4VfPTRRzXuf8aMGScVn2lc4qPCGdwlicFdkirWFRaXsmHPIdbucpLG+sw8/rp0O0dLygCIDA+hR+tjtQzrDDf1xW8JQp3hUfnu03D3UXXI1FjgYXd5LvC0OPXnscDbqloEbBWRzcBAYMnJxDT9/XWszzx4Mrs4zumpcTx0aS+f2++77z5++ukn+vbty4UXXsgll1zCAw88QEJCAhs2bGDjxo2MGzeOHTt2UFhYyB133MGUKVOAY5/o8/PzueiiizjnnHP45ptvaNu2Lf/85z+Jiopi8uTJjB49mssvv5y0tDRuuOEG3n//fYqLi/n73/9Ojx49yM7O5pprriEzM5PBgwfz2WefsWLFimprCk888QSvvvoqADfffDN33nknhw8f5sorr2Tnzp2UlpbywAMPMHHiRO677z7mz59PWFgYv/jFL5g5c2a9nuNgFxkeSt/2Lejb/lhNsqS0jJ+yD1ckjXWZecxflcnflv0MQGiIcJrbGd4rNZ7e1hluToBf+yBEJBRYAXQFnlHVZVWKtAV2AKhqiYjkAUnues+axk53nbdjTAGmAHTo0KFe468Pjz76KGvXrmXVqlUALFy4kJUrV7J27dqKIZ2vvvoqiYmJFBQUcNZZZzFhwgSSkpIq7WfTpk289dZbvPTSS1x55ZW8++67TJo06bjjJScns3LlSp599llmzpzJyy+/zPTp0xk5ciT3338///rXv3jllVeqjXnFihW89tprLFu2DFXl7LPPZtiwYWzZsoXU1FQ+/PBDAPLy8sjNzWXevHls2LABEeHAgQP1cdpMDcJCQ+jeOpburWOZMMBZZ53hpr75NUG4TUJ9RaQFME9Eeqvq2no+xovAi+DcB1Fd2eo+6TekgQMHVhrv/+STTzJv3jwAduzYwaZNm45LEJ06daJv374ADBgwgG3btnnd9/jx4yvKvPfeewB89dVXFfsfNWoUCQkJ1cb31VdfcdlllxETE1Oxzy+//JJRo0Zx9913c++99zJ69GjOPfdcSkpKiIyM5KabbmL06NGMHj26jmfD1JfadIavyzzI2l0Hj+sM75Ua59Ehbp3hxtEgo5hU9YCILABGAZ4JYhfQHtgpImFAPE5ndfn6cu3cdU1C+YUXnBrFv//9b5YsWUJ0dDTDhw/3etd3RMSxT3mhoaEUFBR43Xd5udDQUEpKSuo17m7durFy5Uo++ugjfv/733P++efz4IMP8u233/L5558zd+5cnn76ab744ot6Pa45OTV1hq/PPMjazDwWbcyu1Bleca+GdYYHLX+OYkoBit3kEAVciNMJ7Wk+cANO38LlwBeqqiIyH3hTRJ7A6aQ+DfjWX7H6U2xsLIcOHfK5PS8vj4SEBKKjo9mwYQNLly71WfZEDR06lDlz5nDvvffy6aefsn///mrLn3vuuUyePJn77rsPVWXevHm8/vrrZGZmkpiYyKRJk2jRogUvv/wy+fn5HDlyhIsvvpihQ4fSuXPneo/f1L/qOsPLh92uz8zj9aXbKfLRGd4rNY7ura0zvCnzZw2iDTDb7YcIAeao6gciMgPIUNX5wCvA624n9D6ckUuo6joRmQOsB0qA20/FEUwASUlJDB06lN69e3PRRRdxySWXVNo+atQonn/+eXr27En37t0ZNGhQvcfw0EMPcfXVV/P6668zePBgWrduTWxsrM/y/fv3Z/LkyQwcOBBwOqn79evHJ598wj333ENISAjh4eE899xzHDp0iLFjx1JYWIiq8sQTT9R7/KZhVNcZXp401mXmMX+1787w8jvD46wzvElo8nMx/fDDD/Ts2TNAETUORUVFhIaGEhYWxpIlS5g6dWpFp3ljYr+rU4OqsmNfAWsz8zzu1zhItjtZIVhn+KkkqOdiMvDzzz9z5ZVXUlZWRrNmzXjppZcCHZI5hYkIHZKi6ZAUfUKd4eX3alhneONnCSIInHbaaXz33XeBDsM0cbXpDF+XeZDFm3IodXvDrTO8cbMEYYzxm5o6w8ubpzw7wyPCQjitVXM6JTenU1I0ackxpCXH0CkphoSYZoF6K0HJEoQxpkHV1Bm+LvMgG/ceYtWO/Xy4JrNi6C04CcdJFk7i6JQcQ1qSk0Dio6xjvL5ZgjDGBJznneHj+x9bX1RSyo59BWzLOcy23MNsdX8u37aff67OxHOMTWJMM9LcxNHZrXWUJ4/mEXapOxF21owxjVZEWChdWzana8vmx20rLC7l531HnKSR4ySPrTmH+Xpz5elFAFJiI+iUFENacnRFc1V5AolqZvdx+GIJohFq3rw5+fn5ZGZmMm3aNObOnXtcmeHDhzNz5kzS072OTgNg1qxZTJkyhejoaKB204fXxsMPP0zz5s359a9/fVL7MeZkRIaH0q1VLN1aHX9Pz5GjJWzLOXKs1uHWPL7YkE1O/s5KZVvHRZKWHF2puapTcgwdEqOD/iZASxCNWGpqqtfkUFuzZs1i0qRJFQmiNtOHG9MURDdzRkednhp33LZDhcVsz/WoeeQ6Pz9Zt5d9h49WlBOB1PgoJ3EkR5OW5PZ5JMfQPiGaZmFNf6RVcCWIj++DPd/X7z5b94GLHvW5+b777qN9+/bcfvvtwLFP37feeitjx45l//79FBcX88gjjzB27NhKr922bRujR49m7dq1FBQUcOONN7J69Wp69OhRaS6mqVOnsnz5cgoKCrj88suZPn06Tz75JJmZmYwYMYLk5GQWLFhQ6QuBvE3nvW3bNp/TivuyatUqbr31Vo4cOUKXLl149dVXSUhI4Mknn+T5558nLCyM008/nbfffptFixZxxx13AM5Y+sWLF1d7R7cx/hAbGU7vtvH0bht/3La8I8UVCaO8v2NbzmHmr8rkYOGxuc1CQ4S2LaIqdZiXN121S4hqMsN0gytBBMDEiRO58847KxLEnDlz+OSTT4iMjGTevHnExcWRk5PDoEGDGDNmjM+bhp577jmio6P54YcfWLNmDf37H+vJ++Mf/0hiYiKlpaWcf/75rFmzhmnTpvHEE0+wYMGC4773wdd03gkJCbWeVrzc9ddfz1NPPcWwYcN48MEHmT59OrNmzeLRRx9l69atREREVEwBPnPmTJ555hmGDh1Kfn4+kZGRJ3t6jalX8dHh9I2uPMIKnLvH9x8prtRcVZ5AVm7fT37RseQRFiJ0SIyu6OPolHxsObVFFKEhp86NgcGVIKr5pO8v/fr1Iysri8zMTLKzs0lISKB9+/YUFxfz29/+lsWLFxMSEsKuXbvYu3cvrVu39rqfxYsXM23aNADOOOMMzjjjjIptc+bM4cUXX6SkpITdu3ezfv36Stur8jWd95gxY2o9rTg4Ew0eOHCAYcOGAXDDDTdwxRVXVMR47bXXMm7cOMaNGwc4kwbeddddXHvttYwfP5527drV8iwaE1giQmJMMxJjmjGgY+Xp8lWVnPyjlfo7yjvMl/yUS0HxsWnkmoWG0CEpulLiKO8wbx0XSUgjSx7BlSAC5IorrmDu3Lns2bOHiRMnAvC3v/2N7OxsVqxYQXh4OGlpaV6n+a7J1q1bmTlzJsuXLychIYHJkyef0H7K1XZa8Zp8+OGHLF68mPfff58//vGPfP/999x3331ccsklfPTRRwwdOpRPPvmEHj16nHCsxjQGIkJKbAQpsRGclZZYaZuqsvdgUaXmqvLlxZuyK742FpzZcjsmxrgd5s2dBOL2e6TERgRkShJLEA1g4sSJ3HLLLeTk5LBo0SLA+fTdsmVLwsPDWbBgAdu3b692H+eddx5vvvkmI0eOZO3ataxZswaAgwcPEhMTQ3x8PHv37uXjjz9m+PDhwLGpxqs2Mfmazruu4uPjSUhI4Msvv+Tcc8/l9ddfZ9iwYZSVlbFjxw5GjBjBOeecw9tvv01+fj65ubn06dOHPn36sHz5cjZs2GAJwjRpIkLr+Ehax0dWupscnG8A3H2w8FjScBPH5qx8vtiQRXHpsZs8YpqF0rGik7xyh3lSTDO/JQ9LEA2gV69eHDp0iLZt29KmjTO52bXXXsull15Knz59SE9Pr/FCOXXqVG688UZ69uxJz549GTDA+Z7JM888k379+tGjRw/at2/P0KFDK14zZcoURo0aRWpqKgsWLKhY72s67+qak3yZPXt2RSd1586dee211ygtLWXSpEnk5eWhqkybNo0WLVrwwAMPsGDBAkJCQujVqxcXXXRRnY9nTFMR4nZ0t20RxdCulT/ElZSWkXmg8LgO83WZefxr3Z6KuawAYiPC6N46lr/fOrjeE4VN920aDftdGVOz4tIydu4vqJQ4jpaU8egE3/2O1bHpvo0xpokIDw2hk3sz3wg/H6tpDNY1xhhT74IiQTSlZrSmyn5HxjQ+fksQItJeRBaIyHoRWScid3gpc4+IrHIfa0WkVEQS3W3bROR7d1vG8UeoncjISHJzc+0C1IipKrm5uXbjnDGNjD/7IEqAu1V1pYjEAitE5DNVXV9eQFX/BPwJQEQuBf5bVfd57GOEquacTBDt2rVj586dZGdnn8xujJ9FRkbajXPGNDJ+SxCquhvY7S4fEpEfgLbAeh8vuRp4q77jCA8Pp1OnTvW9W2OMafIapA9CRNKAfsAyH9ujgVHAux6rFfhURFaIyJRq9j1FRDJEJMNqCcYYU3/8niBEpDnOhf9OVT3oo9ilwNdVmpfOUdX+wEXA7SJynrcXquqLqpququkpKSn1GrsxxgQzvyYIEQnHSQ5/U9X3qil6FVWal1R1l/szC5gHDPRXnMYYY47ntzupxbnnezawT1XvrKZcPLAVaK+qh911MUCI23cRA3wGzFDVf9VwzGyg+kmNfEsGTqpD3E8srrqxuOrG4qqbphhXR1X12vziz1FMQ4HrgO9FZJW77rdABwBVfd5ddxnwaXlycLUC5rnzioQBb9aUHNx9nnAbk4hk+LrdPJAsrrqxuOrG4qqbYIvLn6OYvgJqnDlKVf8C/KXKui3AmX4JzBhjTK0ExZ3Uxhhj6s4SxDEvBjoAHyyuurG46sbiqpugiqtJTfdtjDGm/lgNwhhjjFeWIIwxxngVdAlCREaJyI8isllE7vOyPUJE3nG3L3OnCWkMcU0WkWyP2W9vboCYXhWRLBFZ62O7iMiTbsxrRKS/v2OqZVzDRSTP41w92EBx1WYG4wY/Z7WMq8HPmYhEisi3IrLajWu6lzIN/v9Yy7ga/P/R49ihIvKdiHzgZVv9ni9VDZoHEAr8BHQGmgGrgdOrlLkNeN5dvgp4p5HENRl4uoHP13lAf2Ctj+0XAx/jDGceBCxrJHENBz4IwN9XG6C/uxwLbPTye2zwc1bLuBr8nLnnoLm7HI4zV9ugKmUC8f9Ym7ga/P/R49h3AW96+33V9/kKthrEQGCzqm5R1aPA28DYKmXG4twBDjAXON+9KzzQcTU4VV0M7KumyFjgr+pYCrQQkTaNIK6AUNXdqrrSXT4ElM9g7KnBz1kt42pw7jnId5+Gu4+qo2Ya/P+xlnEFhIi0Ay4BXvZRpF7PV7AliLbADo/nOzn+H6WijKqWAHlAUiOIC2CC2ywxV0Ta+zmm2qht3IEw2G0i+FhEejX0wcX3DMYBPWfVxAUBOGduc8kqIAv4TFV9nq8G/H+sTVwQmP/HWcBvgDIf2+v1fAVbgjiVvQ+kqeoZOHNTza6hfDBbiTO/zJnAU8A/GvLgUrsZjBtcDXEF5Jypaqmq9gXaAQNFpHdDHLcmtYirwf8fRWQ0kKWqK/x9rHLBliB2AZ6Zvp27zmsZEQkD4oHcQMelqrmqWuQ+fRkY4OeYaqM257PBqerB8iYCVf0ICBeR5IY4ttQ8g3FAzllNcQXynLnHPAAswPleGE+B+H+sMa4A/T8OBcaIyDacZuiRIvJGlTL1er6CLUEsB04TkU4i0gynE2d+lTLzgRvc5cuBL9Tt8QlkXFXaqcfgtCMH2nzgendkziAgT51vEgwoEWld3u4qIgNx/s79flFxj/kK8IOqPuGjWIOfs9rEFYhzJiIpItLCXY4CLgQ2VCnW4P+PtYkrEP+Pqnq/qrZT1TSca8QXqjqpSrF6PV/+nM210VHVEhH5FfAJzsihV1V1nYjMADJUdT7OP9LrIrIZpyP0qkYS1zQRGYPzXd/7cEZR+JWIvIUzuiVZRHYCD+F02KHObLwf4YzK2QwcAW70d0y1jOtyYKqIlAAFwFUNkOShdjMYB+Kc1SauQJyzNsBsEQnFSUhzVPWDQP8/1jKuBv9/9MWf58um2jDGGONVsDUxGWOMqSVLEMYYY7yyBGGMMcYrSxDGGGO8sgRhjDHGK0sQxtRAREo9Zu1cJV5m2z2JfaeJj1lpjQm0oLoPwpgTVOBOu2BMULEahDEnSES2icjjIvK9+/0BXd31aSLyhTuR2+ci0sFd30pE5rkT4q0WkSHurkJF5CVxvnvgU/fuXURkmjjf4bBGRN4O0Ns0QcwShDE1i6rSxDTRY1ueqvYBnsaZaROcye5muxO5/Q140l3/JLDInRCvP7DOXX8a8Iyq9gIOABPc9fcB/dz93OqvN2eML3YntTE1EJF8VW3uZf02YKSqbnEnw9ujqkkikgO0UdVid/1uVU0WkWygncckb+XTb3+mqqe5z+8FwlX1ERH5F5CPM7PqPzy+o8CYBmE1CGNOjvpYrosij+VSjvUNXgI8g1PbWO7OzmlMg7EEYczJmejxc4m7/A3HJkm7FvjSXf4cmAoVX0gT72unIhICtFfVBcC9ONM2H1eLMcaf7BOJMTWL8pgFFeBfqlo+1DVBRNbg1AKudtf9F/CaiNwDZHNsxtY7gBdF5CacmsJUwNdU36HAG24SEeBJ97sJjGkw1gdhzAly+yDSVTUn0LEY4w/WxGSMMcYrq0EYY4zxymoQxhhjvLIEYYwxxitLEMYYY7yyBGGMMcYrSxDGGGO8+v8x8QG8JkpG4gAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i1_lxW_9l7wc"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}