vizwiz_caption_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from vizwiz_api.vizwiz import VizWiz\n",
    "from vizwiz_eval_cap.eval import VizWizEvalCap\n",
    "import matplotlib.pyplot as plt\n",
    "import skimage.io as io\n",
    "import numpy as np\n",
    "import pylab\n",
    "#pylab.rcParams['figure.figsize'] = (8.0, 10.0)\n",
    "\n",
    "import json\n",
    "from json import encoder\n",
    "encoder.FLOAT_REPR = lambda o: format(o, '.3f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found Stanford CoreNLP.\r\n"
     ]
    }
   ],
   "source": [
    "# download Stanford models\n",
    "!bash get_stanford_models.sh # for Windows with Windows Subsystem for Linux (WSL) installed\n",
    "#!./get_stanford_models.sh   # for Linux / MacOS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# file containing groundtruth\n",
    "annFile = './annotations/val.json'\n",
    "resFile = './results/fake_caption_val.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading annotations into memory...\n",
      "Done (t=0.10s)\n",
      "creating index...\n",
      "index created! imgs = 7750, anns = 33145\n"
     ]
    }
   ],
   "source": [
    "# initialize VizWiz API for groundtruth \n",
    "vizwiz = VizWiz(annFile, ignore_rejected=True, ignore_precanned=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading and preparing results...\n",
      "DONE (t=0.02s)\n",
      "creating index...\n",
      "index created! imgs = 7750, anns = 7750\n"
     ]
    }
   ],
   "source": [
    "# load generated results\n",
    "vizwizRes = vizwiz.loadRes(resFile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create VizWizEval object by taking groundtruth and generated results\n",
    "vizwizEval = VizWizEvalCap(vizwiz, vizwizRes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tokenization...\n",
      "setting up scorers...\n",
      "computing Bleu score...\n",
      "{'testlen': 86410, 'reflen': 83404, 'guess': [86410, 78868, 71335, 63802], 'correct': [38063, 17675, 12489, 10415]}\n",
      "ratio: 1.036041436861529\n",
      "Bleu_1: 0.440\n",
      "Bleu_2: 0.314\n",
      "Bleu_3: 0.259\n",
      "Bleu_4: 0.230\n",
      "computing Rouge score...\n",
      "ROUGE_L: 0.348\n",
      "computing CIDEr score...\n",
      "CIDEr: 0.488\n",
      "computing SPICE score...\n",
      "SPICE: 0.094\n",
      "computing METEOR score...\n",
      "METEOR: 0.160\n"
     ]
    }
   ],
   "source": [
    "# evaluate results\n",
    "# SPICE will take a few minutes the first time, but speeds up due to caching\n",
    "vizwizEval.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bleu_1: 0.440\n",
      "Bleu_2: 0.314\n",
      "Bleu_3: 0.259\n",
      "Bleu_4: 0.230\n",
      "ROUGE_L: 0.348\n",
      "CIDEr: 0.488\n",
      "SPICE: 0.094\n",
      "METEOR: 0.160\n"
     ]
    }
   ],
   "source": [
    "# print output evaluation scores\n",
    "for metric, score in vizwizEval.eval.items():\n",
    "    print('%s: %.3f'%(metric, score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save evaluation results to ./results folder\n",
    "eval_imgs_file = './results/VizWiz_Fake_Captions_eval_imgs.json'\n",
    "eval_file = './results/VizWiz_Fake_Captions_eval.json'\n",
    "\n",
    "json.dump(vizwizEval.evalImgs, open(eval_imgs_file, 'w'), indent=2)\n",
    "json.dump(vizwizEval.eval,     open(eval_file, 'w'), indent=2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "name": ""
 },
 "nbformat": 4,
 "nbformat_minor": 1
}