main_icarl.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "main_icarl.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/danielegenta/ArtReader/blob/master/main_icarl.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1-Tkq4Z64NfD",
        "colab_type": "code",
        "outputId": "12120027-454c-4a9c-899a-63414e182031",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "\"\"\"\n",
        "  Following the iCaRL paper specifications.\n",
        "  ...documentation ...\n",
        "\"\"\""
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'\\n  Following the iCaRL paper specifications.\\n  ...documentation ...\\n'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tMSxfKS2gIKU",
        "colab_type": "code",
        "outputId": "cbaf4a32-9054-4c84-ea5b-b097c312781e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "\"\"\"!pip3 install 'torch==1.3.1'\n",
        "!pip3 install 'torchvision==0.5.0'\n",
        "!pip3 install 'Pillow-SIMD'\n",
        "!pip3 install 'tqdm'\"\"\"\n",
        "# !pip install --upgrade wandb"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"!pip3 install 'torch==1.3.1'\\n!pip3 install 'torchvision==0.5.0'\\n!pip3 install 'Pillow-SIMD'\\n!pip3 install 'tqdm'\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iiz6sjyFgQFs",
        "colab_type": "code",
        "outputId": "8b1eaa35-2b0a-4dde-b747-0fa62222f942",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "source": [
        "import os\n",
        "import logging\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "from torch.utils.data import Subset, DataLoader\n",
        "from torch.backends import cudnn\n",
        "\n",
        "import torchvision\n",
        "from torchvision import transforms\n",
        "\n",
        "from PIL import Image\n",
        "from tqdm import tqdm\n",
        "import random\n",
        "\n",
        "\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import time\n",
        "\n",
        "from sklearn.metrics import confusion_matrix\n",
        "import math"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
            "  import pandas.util.testing as tm\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BToWlSKc4km7",
        "colab_type": "code",
        "outputId": "1f69a63a-0fea-4faa-cc9e-1d1e702e07f2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "source": [
        "DATA_DIR = 'DATA' # here the dataset will be downloaded\n",
        "\n",
        "# Clone github repository with dataset handler\n",
        "!rm -r Cifar100/\n",
        "!rm -r $DATA_DIR\n",
        "!mkdir \"DATA\"\n",
        "if not os.path.isdir('./Cifar100'):\n",
        "  !git clone -b exemplars https://github.com/danielegenta/Progetto-MLDL.git\n",
        "  !mv 'Progetto-MLDL' 'Cifar100'\n",
        "  !rm -r Cifar100/Theoretical-Sources\n",
        "  !rm -rf Cifar100/ProjectMLDL.ipynb"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'Progetto-MLDL'...\n",
            "remote: Enumerating objects: 192, done.\u001b[K\n",
            "remote: Counting objects: 100% (192/192), done.\u001b[K\n",
            "remote: Compressing objects: 100% (128/128), done.\u001b[K\n",
            "remote: Total 1417 (delta 124), reused 127 (delta 64), pack-reused 1225\u001b[K\n",
            "Receiving objects: 100% (1417/1417), 6.83 MiB | 13.68 MiB/s, done.\n",
            "Resolving deltas: 100% (869/869), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3Raa-DyJgUwV",
        "colab_type": "text"
      },
      "source": [
        "dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tTxhdzcVgWmO",
        "colab_type": "code",
        "outputId": "1f4f4143-cc0e-4dfe-ae84-edf362aae0cd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 255
        }
      },
      "source": [
        "# Download dataset from the official source and save it into DATA/cifar-100-pyhton\n",
        "\n",
        "if not os.path.isdir('./{}'.format(\"$DATA_DIR/cifar-100-python\")):\n",
        "    !wget https://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz\n",
        "    !tar -xf 'cifar-100-python.tar.gz'  \n",
        "    !mkdir $DATA_DIR\n",
        "    !mv 'cifar-100-python' \"$DATA_DIR/cifar-100-python\"\n",
        "    !rm -rf 'cifar-100-python.tar.gz"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-05-29 23:10:07--  https://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz\n",
            "Resolving www.cs.toronto.edu (www.cs.toronto.edu)... 128.100.3.30\n",
            "Connecting to www.cs.toronto.edu (www.cs.toronto.edu)|128.100.3.30|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 169001437 (161M) [application/x-gzip]\n",
            "Saving to: ‘cifar-100-python.tar.gz.1’\n",
            "\n",
            "cifar-100-python.ta 100%[===================>] 161.17M  44.3MB/s    in 4.0s    \n",
            "\n",
            "2020-05-29 23:10:11 (40.3 MB/s) - ‘cifar-100-python.tar.gz.1’ saved [169001437/169001437]\n",
            "\n",
            "mkdir: cannot create directory ‘DATA’: File exists\n",
            "/bin/bash: -c: line 0: unexpected EOF while looking for matching `''\n",
            "/bin/bash: -c: line 1: syntax error: unexpected end of file\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zjIXkQbKgZH3",
        "colab_type": "code",
        "outputId": "a1671b8d-7e6c-4ac4-ed0b-b0ce20178a8e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "from Cifar100 import utils\n",
        "\n",
        "\n",
        "dictHyperparams = utils.getHyperparams()\n",
        "print(dictHyperparams)\n",
        "\n",
        "DEVICE = dictHyperparams[\"DEVICE\"] # 'cuda' or 'cpu'\n",
        "NUM_CLASSES = dictHyperparams[\"NUM_CLASSES\"] \n",
        "\n",
        "BATCH_SIZE = dictHyperparams[\"BATCH_SIZE\"]     # Higher batch sizes allows for larger learning rates. An empirical heuristic suggests that, when changing\n",
        "                     # the batch size, learning rate should change by the same factor to have comparable results\n",
        "\n",
        "LR = dictHyperparams[\"LR\"]          # The initial Learning Rate\n",
        "MOMENTUM = dictHyperparams[\"MOMENTUM\"]       # Hyperparameter for SGD, keep this at 0.9 when using SGD\n",
        "WEIGHT_DECAY = dictHyperparams[\"WEIGHT_DECAY\"] # Regularization, you can keep this at the default\n",
        "\n",
        "NUM_EPOCHS = dictHyperparams[\"NUM_EPOCHS\"]     # Total number of training epochs (iterations over dataset)\n",
        "GAMMA = dictHyperparams[\"GAMMA\"]         # Multiplicative factor for learning rate step-down\n",
        "\n",
        "LOG_FREQUENCY = dictHyperparams[\"LOG_FREQUENCY\"]\n",
        "MILESTONES = dictHyperparams[\"MILESTONES\"]\n",
        "RANDOM_SEED = dictHyperparams[\"SEED\"]\n",
        "\n",
        "# icarl params\n",
        "herding = False # if false random exemplars, if true nme (herding)\n",
        "classifier = \"FCC\" # NCM, FCC ..."
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'LR': 2, 'MOMENTUM': 0.9, 'WEIGHT_DECAY': 1e-05, 'NUM_EPOCHS': 70, 'MILESTONES': [49, 63], 'BATCH_SIZE': 128, 'DEVICE': 'cuda', 'GAMMA': 0.2, 'SEED': 42, 'LOG_FREQUENCY': 10, 'NUM_CLASSES': 100}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JnOcQlG_ga8h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_transform, eval_transform = utils.getTransformations()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WHXbDzgjgk_B",
        "colab_type": "code",
        "outputId": "5674b2e7-ef4c-4a0b-ab1b-950294b148ac",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "from Cifar100.Dataset.cifar100 import CIFAR100\n",
        "\n",
        "# Import dataset\n",
        "train_dataset = CIFAR100(DATA_DIR, split='train', transform=train_transform)\n",
        "test_dataset = CIFAR100(DATA_DIR, split='test', transform=eval_transform)\n",
        "\n",
        "# check if datasets have been correctly loaded\n",
        "print(len(train_dataset))\n",
        "print(len(test_dataset))"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "50000\n",
            "10000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m85q6ZMLgsC4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from Cifar100.reverse_index import ReverseIndex\n",
        "\n",
        "def build_test_splits(dataset, reverse_index):\n",
        "    splits = dict()\n",
        "    groups = list(reverse_index.getGroups())\n",
        "    for g in groups:\n",
        "        labels_of_groups = reverse_index.getLabelsOfGroup(g)\n",
        "        indices = list(dataset.df[dataset.df['labels'].isin(labels_of_groups)].index)\n",
        "        splits[g] = indices\n",
        "    return splits"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CgY-syfF3WRw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# performing the train/val split\n",
        "train_splits = train_dataset.split_in_train_val_groups(ratio=1, seed=RANDOM_SEED)\n",
        "outputs_labels_mapping = ReverseIndex(train_dataset, train_splits)\n",
        "\n",
        "# performing the test split (coherent with train/val)\n",
        "test_splits = build_test_splits(test_dataset, outputs_labels_mapping)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PsFyMkAyguQW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Dataloaders iterate over pytorch datasets and transparently provide useful functions (e.g. parallelization and shuffling)\n",
        "train_subsets = []\n",
        "val_subsets = []\n",
        "test_subsets = []\n",
        "\n",
        "for v in train_splits.values():\n",
        "    train_subs = Subset(train_dataset, v['train'])\n",
        "    #val_subs = Subset(train_dataset, v['val'])\n",
        "    train_subsets.append(train_subs)\n",
        "    #val_subsets.append(val_subs)\n",
        "\n",
        "for i in range(0,10): # for each group of classes\n",
        "    v=test_splits[i]\n",
        "    test_subs = Subset(test_dataset, v)\n",
        "    test_subsets.append(test_subs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "axANZYKXg6wn",
        "colab_type": "text"
      },
      "source": [
        "**Exemplars management**<br>\n",
        "From iCaRL. We have an exemplar set for each class that we have seen so far. The cardinality of each exemplar set is constant and it is equal, at any time, to m = K/t. Where K is a constraint equal to the amount of memory we're allocating for the exemplars and t is the number of classes that has been seen so far. Implementing iCaRL, whenever a group of (10) classes is trained, it is trained on the train data for those classes (as before) + the current exemplars sets.*italicized text*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cx0Woq8uhXyR",
        "colab_type": "code",
        "outputId": "22fed9a8-5a03-4c91-caac-295017e187c3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "from Cifar100.icarl_model import ICaRL\n",
        "\n",
        "# default params\n",
        "\n",
        "K = 2000\n",
        "n_classes = 0\n",
        "feature_size = 2048\n",
        "\n",
        "icarl = ICaRL(feature_size, n_classes, BATCH_SIZE, WEIGHT_DECAY, LR, GAMMA, NUM_EPOCHS, DEVICE,MILESTONES,MOMENTUM, K, herding, outputs_labels_mapping)\n",
        "icarl.cuda() "
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ICaRL(\n",
              "  (net): ResNet(\n",
              "    (conv1): Conv2d(3, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "    (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    (relu): ReLU(inplace=True)\n",
              "    (layer1): Sequential(\n",
              "      (0): BasicBlock(\n",
              "        (conv1): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (1): BasicBlock(\n",
              "        (conv1): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (2): BasicBlock(\n",
              "        (conv1): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (3): BasicBlock(\n",
              "        (conv1): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (4): BasicBlock(\n",
              "        (conv1): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (layer2): Sequential(\n",
              "      (0): BasicBlock(\n",
              "        (conv1): Conv2d(16, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (downsample): Sequential(\n",
              "          (0): Conv2d(16, 32, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "          (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        )\n",
              "      )\n",
              "      (1): BasicBlock(\n",
              "        (conv1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (2): BasicBlock(\n",
              "        (conv1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (3): BasicBlock(\n",
              "        (conv1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (4): BasicBlock(\n",
              "        (conv1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (layer3): Sequential(\n",
              "      (0): BasicBlock(\n",
              "        (conv1): Conv2d(32, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (downsample): Sequential(\n",
              "          (0): Conv2d(32, 64, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        )\n",
              "      )\n",
              "      (1): BasicBlock(\n",
              "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (2): BasicBlock(\n",
              "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (3): BasicBlock(\n",
              "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (4): BasicBlock(\n",
              "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (avgpool): AvgPool2d(kernel_size=8, stride=1, padding=0)\n",
              "    (fc): Linear(in_features=64, out_features=0, bias=True)\n",
              "  )\n",
              "  (feature_extractor): ResNet(\n",
              "    (conv1): Conv2d(3, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "    (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    (relu): ReLU(inplace=True)\n",
              "    (layer1): Sequential(\n",
              "      (0): BasicBlock(\n",
              "        (conv1): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (1): BasicBlock(\n",
              "        (conv1): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (2): BasicBlock(\n",
              "        (conv1): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (3): BasicBlock(\n",
              "        (conv1): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (4): BasicBlock(\n",
              "        (conv1): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (layer2): Sequential(\n",
              "      (0): BasicBlock(\n",
              "        (conv1): Conv2d(16, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (downsample): Sequential(\n",
              "          (0): Conv2d(16, 32, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "          (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        )\n",
              "      )\n",
              "      (1): BasicBlock(\n",
              "        (conv1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (2): BasicBlock(\n",
              "        (conv1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (3): BasicBlock(\n",
              "        (conv1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (4): BasicBlock(\n",
              "        (conv1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (layer3): Sequential(\n",
              "      (0): BasicBlock(\n",
              "        (conv1): Conv2d(32, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (downsample): Sequential(\n",
              "          (0): Conv2d(32, 64, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        )\n",
              "      )\n",
              "      (1): BasicBlock(\n",
              "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (2): BasicBlock(\n",
              "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (3): BasicBlock(\n",
              "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (4): BasicBlock(\n",
              "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (avgpool): AvgPool2d(kernel_size=8, stride=1, padding=0)\n",
              "    (fc): Sequential()\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ppBh08iGBARC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def computeAccuracy(method, net, loader, reverse_index, dataset, all_preds_cm, all_labels_cm):\n",
        "  total = 0.0\n",
        "  correct = 0.0\n",
        "  for indices, images, labels in loader:\n",
        "        images = images.to(DEVICE)\n",
        "        labels = labels.to(DEVICE)\n",
        "        labels = reverse_index.getNodes(labels)\n",
        "        \n",
        "        # add other classifiers\n",
        "        if classifier == 'NCM':\n",
        "          preds = net.classify(images)\n",
        "        elif classifier == 'FCC':\n",
        "          preds = net.FCC_classify(images)\n",
        "\n",
        "        correct += torch.sum(preds == labels.data).data.item()\n",
        "  accuracy = correct/len(dataset)\n",
        "  if method == 'test':\n",
        "    all_preds_cm.extend(preds.tolist())\n",
        "    all_labels_cm.extend(labels.data.tolist())\n",
        "  return accuracy, all_preds_cm, all_labels_cm"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wupANuY0g1pn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def incrementalTraining(icarl, train_subsets, val_subsets, test_subsets,eval_transform, reverse_index, K):\n",
        "    \n",
        "    all_accuracies = []\n",
        "    all_preds_cm = []\n",
        "    all_labels_cm = []\n",
        "    group_id=1\n",
        "    test_set = None\n",
        "    train_set_big = None # train subsets so far (needed for exemplars)\n",
        "\n",
        "    #for train_subset, val_subset, test_subset in zip(train_subsets, val_subsets, test_subsets):\n",
        "    for train_subset, test_subset in zip(train_subsets, test_subsets):\n",
        "        print(\"GROUP: \",group_id)\n",
        "        if test_set is None:\n",
        "          test_set = test_subset\n",
        "          train_set_big = train_subset\n",
        "        else:\n",
        "          test_set = utils.joinSubsets(test_dataset, [test_set, test_subset])\n",
        "          train_set_big = utils.joinSubsets(train_dataset, [train_set_big, train_subset])\n",
        "\n",
        "        train_dataloader = DataLoader(train_subset, batch_size=BATCH_SIZE,shuffle=True, num_workers=4)\n",
        "        #val_dataloader = DataLoader(val_subset, batch_size=BATCH_SIZE,shuffle=True, num_workers=4)\n",
        "        test_dataloader = DataLoader(test_set, batch_size=BATCH_SIZE,shuffle=True, num_workers=4)\n",
        "\n",
        "        ####### iCaRL implementation(following alg. 2,3,4,5 on icarl paper) ##################\n",
        "        \n",
        "        new_classes_examined = list(train_dataset.df.loc[train_subset.indices, 'labels'].value_counts().index)\n",
        "        \n",
        "        # 1 - update representation of the net \n",
        "        #  alg. 3 icarl\n",
        "        # (here the trainset will be augmented with the exemplars too)\n",
        "        # (here the classes are incremented too)\n",
        "        icarl.update_representation(train_subset, train_dataset, new_classes_examined)\n",
        "\n",
        "        # 2 - update m (number of images per class in the exemplar set corresponding to that class)\n",
        "        m = int(math.ceil(K/icarl.n_classes))\n",
        "\n",
        "        print(\"Reducing each exemplar set to size: {}\".format(m))\n",
        "\n",
        "        # 3 - reduce exemplar set for all the previously seen classes\n",
        "        # alg.5 icarl\n",
        "        icarl.reduce_exemplar_sets(m)\n",
        "\n",
        "        # retrieve the 10 classes in the current subset\n",
        "        # NB. Here there will be exemplars too! (if i do not want that, use new_classes_examined)\n",
        "        classes_current_subset = list(train_dataset.df.loc[train_subset.indices, 'labels'].value_counts().index)\n",
        "        \n",
        "        print(\"Constructing exemplar sets class...\")\n",
        "        \n",
        "        # 4 - construct the exemplar set for the new classes\n",
        "        for y in new_classes_examined: # for each class in the current subset\n",
        "          \n",
        "          \n",
        "          # extract all the imgs in the train subset that are linked to this class\n",
        "          images_current_class = train_subset.dataset.df.loc[train_dataset.df['labels'] == y, 'data'] #they're TENSORS NOT IMAGES (the conversion will be done later)         \n",
        "          imgs_idxs = images_current_class.index # the indexes of all the images in the current classe being considered 0...49k\n",
        "          class_train_subset = Subset(train_dataset, imgs_idxs)#subset of the train dataset where i have all the imgs of class y\n",
        "\n",
        "          # alg. 4 icarl\n",
        "          #print(type(class_train_subset))\n",
        "          icarl.construct_exemplar_set(class_train_subset,m,y) # why eval? ref: https://github.com/donlee90/icarl/blob/master/main.py\n",
        "\n",
        "        # update the num classes seen so far\n",
        "        icarl.n_known = icarl.n_classes #n_classes is incremented in 1: updateRepresentation\n",
        "\n",
        "        print(\"Performing classification...\")\n",
        "\n",
        "        # start classifier ...\n",
        "        icarl.computeMeans()\n",
        "        \n",
        "        #train accuracy\n",
        "        train_accuracy, _, _ = computeAccuracy('train',icarl, train_dataloader, reverse_index, train_subset,all_preds_cm, all_labels_cm)\n",
        "        print ('Train Accuracy (on current group): %.2f\\n' % (100.0 * train_accuracy))\n",
        "\n",
        "        # --- not used\n",
        "        #val_accuracy, _, _ = computeAccuracy('val',icarl, val_dataloader, reverse_index, val_subset)\n",
        "        #print ('Val Accuracy (on current group): %.2f\\n' % (100.0 * val_accuracy))\n",
        "\n",
        "        #test\n",
        "        test_accuracy, all_preds_cm, all_labels_cm = computeAccuracy('test',icarl, test_dataloader, reverse_index, test_set, all_preds_cm, all_labels_cm)\n",
        "        all_accuracies.append(test_accuracy)\n",
        "        print ('Test Accuracy (all groups seen so far): %.2f\\n' % (100.0 * test_accuracy))\n",
        "\n",
        "        print (\"the model knows %d classes:\\n \" % icarl.n_known)\n",
        "\n",
        "        group_id+=1\n",
        "        \n",
        "    return all_accuracies, np.array(all_preds_cm), np.array(all_labels_cm)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-bmxtCL8AvYD",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "6ea3a859-a9a0-4e8b-d357-b7a8ed6470dc"
      },
      "source": [
        "accuracies, all_preds_cm, all_labels_cm = incrementalTraining(icarl, train_subsets, val_subsets, test_subsets,eval_transform, outputs_labels_mapping, K)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "GROUP:  1\n",
            "NUM_EPOCHS:  0 / 70\n",
            "LOSS:  0.3345637023448944\n",
            "NUM_EPOCHS:  1 / 70\n",
            "LOSS:  0.32189568877220154\n",
            "NUM_EPOCHS:  2 / 70\n",
            "LOSS:  0.3147762417793274\n",
            "NUM_EPOCHS:  3 / 70\n",
            "LOSS:  0.30610963702201843\n",
            "NUM_EPOCHS:  4 / 70\n",
            "LOSS:  0.301776647567749\n",
            "NUM_EPOCHS:  5 / 70\n",
            "LOSS:  0.3008427321910858\n",
            "NUM_EPOCHS:  6 / 70\n",
            "LOSS:  0.2794293761253357\n",
            "NUM_EPOCHS:  7 / 70\n",
            "LOSS:  0.2689259648323059\n",
            "NUM_EPOCHS:  8 / 70\n",
            "LOSS:  0.25819525122642517\n",
            "NUM_EPOCHS:  9 / 70\n",
            "LOSS:  0.24111692607402802\n",
            "NUM_EPOCHS:  10 / 70\n",
            "LOSS:  0.24532583355903625\n",
            "NUM_EPOCHS:  11 / 70\n",
            "LOSS:  0.239095076918602\n",
            "NUM_EPOCHS:  12 / 70\n",
            "LOSS:  0.22961096465587616\n",
            "NUM_EPOCHS:  13 / 70\n",
            "LOSS:  0.22819189727306366\n",
            "NUM_EPOCHS:  14 / 70\n",
            "LOSS:  0.20226679742336273\n",
            "NUM_EPOCHS:  15 / 70\n",
            "LOSS:  0.20246298611164093\n",
            "NUM_EPOCHS:  16 / 70\n",
            "LOSS:  0.1945960968732834\n",
            "NUM_EPOCHS:  17 / 70\n",
            "LOSS:  0.21510639786720276\n",
            "NUM_EPOCHS:  18 / 70\n",
            "LOSS:  0.18178707361221313\n",
            "NUM_EPOCHS:  19 / 70\n",
            "LOSS:  0.19951121509075165\n",
            "NUM_EPOCHS:  20 / 70\n",
            "LOSS:  0.19929265975952148\n",
            "NUM_EPOCHS:  21 / 70\n",
            "LOSS:  0.2299317866563797\n",
            "NUM_EPOCHS:  22 / 70\n",
            "LOSS:  0.19997969269752502\n",
            "NUM_EPOCHS:  23 / 70\n",
            "LOSS:  0.17561782896518707\n",
            "NUM_EPOCHS:  24 / 70\n",
            "LOSS:  0.18129944801330566\n",
            "NUM_EPOCHS:  25 / 70\n",
            "LOSS:  0.17375534772872925\n",
            "NUM_EPOCHS:  26 / 70\n",
            "LOSS:  0.14977048337459564\n",
            "NUM_EPOCHS:  27 / 70\n",
            "LOSS:  0.1864868700504303\n",
            "NUM_EPOCHS:  28 / 70\n",
            "LOSS:  0.17948007583618164\n",
            "NUM_EPOCHS:  29 / 70\n",
            "LOSS:  0.16883495450019836\n",
            "NUM_EPOCHS:  30 / 70\n",
            "LOSS:  0.162698894739151\n",
            "NUM_EPOCHS:  31 / 70\n",
            "LOSS:  0.1826053112745285\n",
            "NUM_EPOCHS:  32 / 70\n",
            "LOSS:  0.14924904704093933\n",
            "NUM_EPOCHS:  33 / 70\n",
            "LOSS:  0.17108246684074402\n",
            "NUM_EPOCHS:  34 / 70\n",
            "LOSS:  0.17567189037799835\n",
            "NUM_EPOCHS:  35 / 70\n",
            "LOSS:  0.1407763957977295\n",
            "NUM_EPOCHS:  36 / 70\n",
            "LOSS:  0.15534833073616028\n",
            "NUM_EPOCHS:  37 / 70\n",
            "LOSS:  0.13823245465755463\n",
            "NUM_EPOCHS:  38 / 70\n",
            "LOSS:  0.15501892566680908\n",
            "NUM_EPOCHS:  39 / 70\n",
            "LOSS:  0.14755883812904358\n",
            "NUM_EPOCHS:  40 / 70\n",
            "LOSS:  0.13200142979621887\n",
            "NUM_EPOCHS:  41 / 70\n",
            "LOSS:  0.12516437470912933\n",
            "NUM_EPOCHS:  42 / 70\n",
            "LOSS:  0.13100068271160126\n",
            "NUM_EPOCHS:  43 / 70\n",
            "LOSS:  0.15097863972187042\n",
            "NUM_EPOCHS:  44 / 70\n",
            "LOSS:  0.13796812295913696\n",
            "NUM_EPOCHS:  45 / 70\n",
            "LOSS:  0.14244499802589417\n",
            "NUM_EPOCHS:  46 / 70\n",
            "LOSS:  0.13649536669254303\n",
            "NUM_EPOCHS:  47 / 70\n",
            "LOSS:  0.11574270576238632\n",
            "NUM_EPOCHS:  48 / 70\n",
            "LOSS:  0.11157029122114182\n",
            "NUM_EPOCHS:  49 / 70\n",
            "LOSS:  0.10135755687952042\n",
            "NUM_EPOCHS:  50 / 70\n",
            "LOSS:  0.10757491737604141\n",
            "NUM_EPOCHS:  51 / 70\n",
            "LOSS:  0.09848669171333313\n",
            "NUM_EPOCHS:  52 / 70\n",
            "LOSS:  0.10289257764816284\n",
            "NUM_EPOCHS:  53 / 70\n",
            "LOSS:  0.09267039597034454\n",
            "NUM_EPOCHS:  54 / 70\n",
            "LOSS:  0.084396593272686\n",
            "NUM_EPOCHS:  55 / 70\n",
            "LOSS:  0.10047551244497299\n",
            "NUM_EPOCHS:  56 / 70\n",
            "LOSS:  0.09714897722005844\n",
            "NUM_EPOCHS:  57 / 70\n",
            "LOSS:  0.08658257871866226\n",
            "NUM_EPOCHS:  58 / 70\n",
            "LOSS:  0.0799257755279541\n",
            "NUM_EPOCHS:  59 / 70\n",
            "LOSS:  0.07922106236219406\n",
            "NUM_EPOCHS:  60 / 70\n",
            "LOSS:  0.08184745907783508\n",
            "NUM_EPOCHS:  61 / 70\n",
            "LOSS:  0.08602651208639145\n",
            "NUM_EPOCHS:  62 / 70\n",
            "LOSS:  0.060437578707933426\n",
            "NUM_EPOCHS:  63 / 70\n",
            "LOSS:  0.09101887047290802\n",
            "NUM_EPOCHS:  64 / 70\n",
            "LOSS:  0.07045741379261017\n",
            "NUM_EPOCHS:  65 / 70\n",
            "LOSS:  0.060872264206409454\n",
            "NUM_EPOCHS:  66 / 70\n",
            "LOSS:  0.06965401023626328\n",
            "NUM_EPOCHS:  67 / 70\n",
            "LOSS:  0.05517173931002617\n",
            "NUM_EPOCHS:  68 / 70\n",
            "LOSS:  0.07246614992618561\n",
            "NUM_EPOCHS:  69 / 70\n",
            "LOSS:  0.04890773445367813\n",
            "Reducing each exemplar set to size: 200\n",
            "Constructing exemplar sets class...\n",
            "Performing classification...\n",
            "Train Accuracy (on current group): 18.00\n",
            "\n",
            "Test Accuracy (all groups seen so far): 75.20\n",
            "\n",
            "the model knows 10 classes:\n",
            " \n",
            "GROUP:  2\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-15-f0631819ecc0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0maccuracies\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mall_preds_cm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mall_labels_cm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mincrementalTraining\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0micarl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_subsets\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_subsets\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_subsets\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0meval_transform\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutputs_labels_mapping\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mK\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-14-79fc2517c890>\u001b[0m in \u001b[0;36mincrementalTraining\u001b[0;34m(icarl, train_subsets, val_subsets, test_subsets, eval_transform, reverse_index, K)\u001b[0m\n\u001b[1;32m     15\u001b[0m           \u001b[0mtrain_set_big\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_subset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m           \u001b[0mtest_set\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoinSubsets\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_dataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mtest_set\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_subset\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m           \u001b[0mtrain_set_big\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoinSubsets\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_dataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mtrain_set_big\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_subset\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/content/Cifar100/utils.py\u001b[0m in \u001b[0;36mjoinSubsets\u001b[0;34m(dataset, subsets)\u001b[0m\n\u001b[1;32m    136\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0ms\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msubsets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    137\u001b[0m         \u001b[0mindices\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindices\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 138\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mSubset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindices\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    139\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    140\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'Subset' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1q_B01Oa82wF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "if herding:\n",
        "  method = 'iCaRL_{}_herding'.format(classifier)\n",
        "else\n",
        "  method = 'iCaRL_{}_random'.format(classifier)\n",
        "\n",
        "\n",
        "print(\"metrics iCaRL for seed {}\".format(RANDOM_SEED))\n",
        "\n",
        "# accuracy \n",
        "data_plot_line=[]\n",
        "\n",
        "classes_per_group = 10\n",
        "for group_classes in range(0,10):\n",
        "    data_plot_line.append(((group_classes + 1)*classes_per_group, accuracies[group_classes]))\n",
        "\n",
        "# plot accuracy trend\n",
        "utils.plotAccuracyTrend(method, data_plot_line, RANDOM_SEED)\n",
        "\n",
        "# confusion matrix\n",
        "confusionMatrixData = confusion_matrix(all_labels_cm, all_preds_cm)\n",
        "utils.plotConfusionMatrix(method, confusionMatrixData, RANDOM_SEED)\n",
        "\n",
        "# write to JSON file\n",
        "utils.writeMetrics(method, RANDOM_SEED, accuracies, confusionMatrixData)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}