Train_DeepLogo2_by_detr.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "04sJNxj-tglb"
      },
      "source": [
        "## Setup DeepLogo2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Xqxzz0b-u2kN"
      },
      "outputs": [],
      "source": [
        "!pip install -q yacs\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ry_IQkoVteFD"
      },
      "outputs": [],
      "source": [
        "!git clone https://github.com/satojkovic/DeepLogo2.git\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1NzKfSgjugvL"
      },
      "outputs": [],
      "source": [
        "%%bash\n",
        "cd DeepLogo2\n",
        "wget http://image.ntua.gr/iva/datasets/flickr_logos/flickr_logos_27_dataset.tar.gz\n",
        "#cp /content/drive/MyDrive/DeepLogo2/train_data/flickr_logos_27_dataset.tar.gz .\n",
        "tar zxvf flickr_logos_27_dataset.tar.gz\n",
        "cd flickr_logos_27_dataset\n",
        "tar zxvf flickr_logos_27_dataset_images.tar.gz\n",
        "cd ..\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DiXUU3GUutp8"
      },
      "outputs": [],
      "source": [
        "%%bash\n",
        "cd DeepLogo2\n",
        "python preproc_annot.py\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xdLxSVq_wL9m"
      },
      "outputs": [],
      "source": [
        "%%bash\n",
        "cd DeepLogo2\n",
        "python flickr2coco.py --mode train --output_dir flickr_logos_27_dataset\n",
        "python flickr2coco.py --mode test --output_dir flickr_logos_27_dataset\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3H-20-VL57gy"
      },
      "source": [
        "## Visualize data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "o89VPK8T59nt"
      },
      "outputs": [],
      "source": [
        "from pycocotools.coco import COCO\n",
        "import numpy as np\n",
        "import skimage.io as io\n",
        "import matplotlib.pyplot as plt\n",
        "import os\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "P2Eg28_V6bgv"
      },
      "outputs": [],
      "source": [
        "coco = COCO('/content/DeepLogo2/flickr_logos_27_dataset/flickr_logos_27_train.json')\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZOQT32Fs638-"
      },
      "outputs": [],
      "source": [
        "# load and display image\n",
        "catIds = coco.getCatIds(catNms=['Intel'])\n",
        "imgIds = coco.getImgIds(catIds=catIds)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8bR9fMoVIing"
      },
      "outputs": [],
      "source": [
        "print(catIds)\n",
        "print(imgIds)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iXOqTVcU69iI"
      },
      "outputs": [],
      "source": [
        "img_id = imgIds[np.random.randint(0,len(imgIds))]\n",
        "print('Image id {}'.format(img_id))\n",
        "\n",
        "img = coco.loadImgs(img_id)[0]\n",
        "\n",
        "img_name = os.path.join('DeepLogo2', 'flickr_logos_27_dataset', 'flickr_logos_27_dataset_images', img['file_name'])\n",
        "print('Image name: {}'.format(img_name))\n",
        "\n",
        "I = io.imread(img_name)\n",
        "plt.figure()\n",
        "plt.imshow(I)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JZ8Qd27vFT1S"
      },
      "outputs": [],
      "source": [
        "annIds = coco.getAnnIds(imgIds=img['id'], catIds=catIds)\n",
        "anns = coco.loadAnns(annIds)\n",
        "\n",
        "plt.imshow(I)\n",
        "coco.showAnns(anns, draw_bbox=True)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6jFOUwcZw9tT"
      },
      "source": [
        "## Setup detr"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EawJT9X9xBku"
      },
      "outputs": [],
      "source": [
        "import torch\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JTrniKdKw9Mq"
      },
      "outputs": [],
      "source": [
        "%%bash\n",
        "cd DeepLogo2\n",
        "python delete_head_and_save.py\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XiF7cdd5xmo9"
      },
      "source": [
        "## Train!!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LixsQaaAOPEn"
      },
      "outputs": [],
      "source": [
        "%cd DeepLogo2\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XhxoZ3SIxmBi"
      },
      "outputs": [],
      "source": [
        "!python detr/main.py \\\n",
        "  --dataset_file \"flickr_logos_27\" \\\n",
        "  --coco_path \"flickr_logos_27_dataset\" \\\n",
        "  --output_dir \"outputs\" \\\n",
        "  --resume \"detr-r50_no-class-head.pth\" \\\n",
        "  --epochs 100\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "odQmIBfYXfAL"
      },
      "source": [
        "## Display results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lNoxt6PwTLgE"
      },
      "outputs": [],
      "source": [
        "from detr.util.plot_utils import plot_logs\n",
        "\n",
        "from pathlib import Path\n",
        "\n",
        "log_directory = [Path('outputs/')]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sb5269jUTTH3"
      },
      "outputs": [],
      "source": [
        "fields_of_interest = (\n",
        "    'loss',\n",
        "    'mAP',\n",
        "    )\n",
        "\n",
        "plot_logs(log_directory,\n",
        "          fields_of_interest)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6_KN6HlATeRi"
      },
      "outputs": [],
      "source": [
        "fields_of_interest = (\n",
        "    'class_error',\n",
        "    'cardinality_error_unscaled',\n",
        "    )\n",
        "\n",
        "plot_logs(log_directory,\n",
        "          fields_of_interest)   \n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MqNzCFMaSgab"
      },
      "outputs": [],
      "source": [
        "!ls outputs\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3jYBs1mKTpQG"
      },
      "outputs": [],
      "source": [
        "model = torch.hub.load('facebookresearch/detr',\n",
        "                       'detr_resnet50',\n",
        "                       pretrained=False,\n",
        "                       num_classes=27)\n",
        "\n",
        "#checkpoint = torch.load('outputs/checkpoint.pth',\n",
        "#                        map_location='cpu')\n",
        "checkpoint = torch.load('/content/drive/MyDrive/DeepLogo2/model_detr/checkpoint.pth', map_location='cpu')\n",
        "\n",
        "model.load_state_dict(checkpoint['model'],\n",
        "                      strict=False)\n",
        "\n",
        "model.eval();\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Uzxm4hyrPXTW"
      },
      "outputs": [],
      "source": [
        "COLORS = [[0.000, 0.447, 0.741], [0.850, 0.325, 0.098], [0.929, 0.694, 0.125],\n",
        "          [0.494, 0.184, 0.556], [0.466, 0.674, 0.188], [0.301, 0.745, 0.933]]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IVafnH6xPfnV"
      },
      "outputs": [],
      "source": [
        "import config\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Hav12YyuPmfa"
      },
      "outputs": [],
      "source": [
        "cfg = config.get_cfg_defaults()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gMJ5TNyPP330"
      },
      "outputs": [],
      "source": [
        "finetuned_classes = cfg.CLASS_NAMES\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nHksC3sTQB-p"
      },
      "outputs": [],
      "source": [
        "import torchvision.transforms as T\n",
        "\n",
        "# standard PyTorch mean-std input image normalization\n",
        "transform = T.Compose([\n",
        "    T.Resize(800),\n",
        "    T.ToTensor(),\n",
        "    T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
        "])\n",
        "\n",
        "# for output bounding box post-processing\n",
        "def box_cxcywh_to_xyxy(x):\n",
        "    x_c, y_c, w, h = x.unbind(1)\n",
        "    b = [(x_c - 0.5 * w), (y_c - 0.5 * h),\n",
        "         (x_c + 0.5 * w), (y_c + 0.5 * h)]\n",
        "    return torch.stack(b, dim=1)\n",
        "\n",
        "def rescale_bboxes(out_bbox, size):\n",
        "    img_w, img_h = size\n",
        "    b = box_cxcywh_to_xyxy(out_bbox)\n",
        "    b = b * torch.tensor([img_w, img_h, img_w, img_h], dtype=torch.float32)\n",
        "    return b\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7wgsy1SLXdvA"
      },
      "outputs": [],
      "source": [
        "def plot_finetuned_results(pil_img, prob=None, boxes=None, save_dir=None, image_name=None):\n",
        "    plt.figure(figsize=(16,10))\n",
        "    plt.imshow(pil_img)\n",
        "    ax = plt.gca()\n",
        "    colors = COLORS * 100\n",
        "    if prob is not None and boxes is not None:\n",
        "      for p, (xmin, ymin, xmax, ymax), c in zip(prob, boxes.tolist(), colors):\n",
        "          ax.add_patch(plt.Rectangle((xmin, ymin), xmax - xmin, ymax - ymin,\n",
        "                                    fill=False, color=c, linewidth=3))\n",
        "          cl = p.argmax()\n",
        "          text = f'{finetuned_classes[cl]}: {p[cl]:0.2f}'\n",
        "          ax.text(xmin, ymin, text, fontsize=15,\n",
        "                  bbox=dict(facecolor='yellow', alpha=0.5))\n",
        "    plt.axis('off')\n",
        "    if save_dir is not None and image_name is not None:\n",
        "      plt.savefig(os.path.join(save_dir, image_name))\n",
        "    plt.show()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BCkc3GCMYAh_"
      },
      "outputs": [],
      "source": [
        "def filter_bboxes_from_outputs(outputs,\n",
        "                               threshold=0.7):\n",
        "  \n",
        "  # keep only predictions with confidence above threshold\n",
        "  probas = outputs['pred_logits'].softmax(-1)[0, :, :-1]\n",
        "  keep = probas.max(-1).values > threshold\n",
        "\n",
        "  probas_to_keep = probas[keep]\n",
        "\n",
        "  # convert boxes from [0; 1] to image scales\n",
        "  bboxes_scaled = rescale_bboxes(outputs['pred_boxes'][0, keep], im.size)\n",
        "  \n",
        "  return probas_to_keep, bboxes_scaled\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "V4IH8USwXjT_"
      },
      "outputs": [],
      "source": [
        "def run_workflow(my_image, my_model, save_dir, image_name):\n",
        "  # mean-std normalize the input image (batch-size: 1)\n",
        "  img = transform(my_image).unsqueeze(0)\n",
        "\n",
        "  # propagate through the model\n",
        "  outputs = my_model(img)\n",
        "\n",
        "  for threshold in [0.8]: #[0.9, 0.7]:\n",
        "    \n",
        "    probas_to_keep, bboxes_scaled = filter_bboxes_from_outputs(outputs,\n",
        "                                                              threshold=threshold)\n",
        "\n",
        "    plot_finetuned_results(my_image,\n",
        "                           probas_to_keep, \n",
        "                           bboxes_scaled, save_dir, image_name)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ja8T3PszXlpl"
      },
      "outputs": [],
      "source": [
        "from PIL import Image\n",
        "\n",
        "#img_name = '/content/DeepLogo2/flickr_logos_27_dataset/flickr_logos_27_dataset_images/4763207899.jpg'\n",
        "img_name = '/content/DeepLogo2/flickr_logos_27_dataset/flickr_logos_27_dataset_images/1690276845.jpg'\n",
        "\n",
        "im = Image.open(img_name)\n",
        "\n",
        "run_workflow(im,\n",
        "            model, None, None)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wafpXN_8xV-7"
      },
      "source": [
        "## Inference with all test images"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RyHGL_wNSzeX"
      },
      "outputs": [],
      "source": [
        "import json\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4-sR_LbaxmH-"
      },
      "outputs": [],
      "source": [
        "with open('flickr_logos_27_dataset/flickr_logos_27_test.json', 'r') as f:\n",
        "  test_data = json.load(f)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "n2ztUw08xxno"
      },
      "outputs": [],
      "source": [
        "test_image_names = [d['file_name'] for d in test_data['images']]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "y_BjZU_5sDnP"
      },
      "outputs": [],
      "source": [
        "save_dir = '/content/drive/MyDrive/DeepLogo2/result_images_epoch100_model'\n",
        "if not os.path.exists(save_dir):\n",
        "  os.makedirs(save_dir)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qZ_Ts7Xjx6N2"
      },
      "outputs": [],
      "source": [
        "for test_image_name in test_image_names:\n",
        "  test_image_full_name = os.path.join(cfg.IMAGE_DIR, test_image_name)\n",
        "  im = Image.open(test_image_full_name)\n",
        "  run_workflow(im, model, save_dir, test_image_name)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "puFTzazH1H2W"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Train_DeepLogo2_by_detr",
      "private_outputs": true,
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}