detect_outfits_eval2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "bc019ca2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-06-08T06:50:08.330774Z",
     "start_time": "2022-06-08T06:50:08.326361Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import json\n",
    "\n",
    "from pycocotools.coco import COCO\n",
    "from pycocotools.cocoeval import COCOeval\n",
    "                                \n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "id": "4a661d1e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-06-08T06:50:08.337748Z",
     "start_time": "2022-06-08T06:50:08.334463Z"
    }
   },
   "outputs": [],
   "source": [
    "p_dir = '/home/zyp/w/d2l/detect_outfits'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "5abc8e82",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-06-08T06:50:08.341714Z",
     "start_time": "2022-06-08T06:50:08.339176Z"
    }
   },
   "outputs": [],
   "source": [
    "epochs = 60\n",
    "conf_thres_val = 0.05\n",
    "\n",
    "exp_dir = p_dir + f'/log/bln_epochs{epochs}/{conf_thres_val}-exp'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "id": "ff8a3ab3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-06-08T06:50:08.364586Z",
     "start_time": "2022-06-08T06:50:08.343515Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1593 591\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "22"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def verify_pre(gt_file, dt_file):\n",
    "    gt = json.load(open(gt_file, 'r'))\n",
    "    dt = json.load(open(dt_file, 'r'))\n",
    "    gt_imgs = pd.DataFrame(gt['images'])['id'].to_numpy()\n",
    "    #print(len(gt_imgs), gt_imgs[:10])\n",
    "    dt_imgs = pd.DataFrame(dt)['image_id'].to_numpy()\n",
    "    print(len(dt_imgs), len(set(dt_imgs)))\n",
    "    \n",
    "    no_detect = []\n",
    "    for img_id in gt_imgs:\n",
    "        if not img_id in dt_imgs:\n",
    "            no_detect.append(img_id)\n",
    "    \n",
    "    return no_detect\n",
    "    \n",
    "gt_file = f'{p_dir}/data/origin/valid_local.json'\n",
    "dt_file = f'{exp_dir}/answer.json'\n",
    "no_detect = verify_pre(gt_file, dt_file)\n",
    "len(no_detect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "0773872b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-06-08T06:50:08.391666Z",
     "start_time": "2022-06-08T06:50:08.366337Z"
    }
   },
   "outputs": [],
   "source": [
    "origin_train_file = p_dir + '/data/origin/train.json'\n",
    "data = json.load(open(origin_train_file, 'r'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "id": "94b6918c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-06-08T06:50:08.397226Z",
     "start_time": "2022-06-08T06:50:08.392677Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{87: 'belt', 1034: 'sunglasses', 131: 'boot', 318: 'cowboy_hat', 588: 'jacket'}"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cats = pd.DataFrame(data[\"categories\"])[['id', 'name']].set_index('id')\n",
    "cats_dict = cats.squeeze().to_dict()\n",
    "cats_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "id": "870b122b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-06-08T06:50:09.163484Z",
     "start_time": "2022-06-08T06:50:09.151232Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>category_id</th>\n",
       "      <th>bbox</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2857187227633385663</td>\n",
       "      <td>1034</td>\n",
       "      <td>[446.00026355, 73.00002816, 17.9999869, 9.9999...</td>\n",
       "      <td>0.082424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2857187227633385663</td>\n",
       "      <td>1034</td>\n",
       "      <td>[153.99986895, 93.99997440000001, 20.9999961, ...</td>\n",
       "      <td>0.166514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2857187227633385663</td>\n",
       "      <td>1034</td>\n",
       "      <td>[661.00033095, 2.0479999999878373e-05, 20.9999...</td>\n",
       "      <td>0.370207</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2857187227633385663</td>\n",
       "      <td>1034</td>\n",
       "      <td>[314.0000295, 72.9999872, 100.99999100000001, ...</td>\n",
       "      <td>0.873189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1855409563895400077</td>\n",
       "      <td>1034</td>\n",
       "      <td>[250.999819, 388.9997312, 278.99974199999997, ...</td>\n",
       "      <td>0.683113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1588</th>\n",
       "      <td>6689251422544595225</td>\n",
       "      <td>318</td>\n",
       "      <td>[294.000092, 112.99993599999999, 418.000215999...</td>\n",
       "      <td>0.206803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1589</th>\n",
       "      <td>6689251422544595225</td>\n",
       "      <td>318</td>\n",
       "      <td>[25.000004800000003, 327.0000128, 26.9999664, ...</td>\n",
       "      <td>0.257942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1590</th>\n",
       "      <td>6689251422544595225</td>\n",
       "      <td>318</td>\n",
       "      <td>[153.99975, 358.00053760000003, 72.999972, 35....</td>\n",
       "      <td>0.725709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1591</th>\n",
       "      <td>6689251422544595225</td>\n",
       "      <td>1034</td>\n",
       "      <td>[343.99994000000004, 299.0000128, 258.00005600...</td>\n",
       "      <td>0.844421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1592</th>\n",
       "      <td>7832136250470041636</td>\n",
       "      <td>588</td>\n",
       "      <td>[499.00031999999993, 290.99980800000003, 120.9...</td>\n",
       "      <td>0.812556</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1593 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 image_id  category_id  \\\n",
       "0     2857187227633385663         1034   \n",
       "1     2857187227633385663         1034   \n",
       "2     2857187227633385663         1034   \n",
       "3     2857187227633385663         1034   \n",
       "4     1855409563895400077         1034   \n",
       "...                   ...          ...   \n",
       "1588  6689251422544595225          318   \n",
       "1589  6689251422544595225          318   \n",
       "1590  6689251422544595225          318   \n",
       "1591  6689251422544595225         1034   \n",
       "1592  7832136250470041636          588   \n",
       "\n",
       "                                                   bbox     score  \n",
       "0     [446.00026355, 73.00002816, 17.9999869, 9.9999...  0.082424  \n",
       "1     [153.99986895, 93.99997440000001, 20.9999961, ...  0.166514  \n",
       "2     [661.00033095, 2.0479999999878373e-05, 20.9999...  0.370207  \n",
       "3     [314.0000295, 72.9999872, 100.99999100000001, ...  0.873189  \n",
       "4     [250.999819, 388.9997312, 278.99974199999997, ...  0.683113  \n",
       "...                                                 ...       ...  \n",
       "1588  [294.000092, 112.99993599999999, 418.000215999...  0.206803  \n",
       "1589  [25.000004800000003, 327.0000128, 26.9999664, ...  0.257942  \n",
       "1590  [153.99975, 358.00053760000003, 72.999972, 35....  0.725709  \n",
       "1591  [343.99994000000004, 299.0000128, 258.00005600...  0.844421  \n",
       "1592  [499.00031999999993, 290.99980800000003, 120.9...  0.812556  \n",
       "\n",
       "[1593 rows x 4 columns]"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt = pd.DataFrame(json.load(open(dt_file, 'r')))\n",
    "dt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "id": "687c4532",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-06-08T06:50:08.400287Z",
     "start_time": "2022-06-08T06:50:08.398321Z"
    }
   },
   "outputs": [],
   "source": [
    "cat_ids = [87, 1034, 131, 318, 588]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "id": "afca9911",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-06-08T06:50:09.149023Z",
     "start_time": "2022-06-08T06:50:08.401492Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/zyp/w/d2l/detect_outfits/data/origin/valid_local.json\n",
      "/home/zyp/w/d2l/detect_outfits/log/bln_epochs60/0.05-exp/answer.json\n",
      "loading annotations into memory...\n",
      "Done (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "Loading and preparing results...\n",
      "DONE (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *bbox*\n",
      "DONE (t=0.21s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=0.05s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.279\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.500\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.274\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.146\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.282\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.327\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.246\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.360\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.360\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.238\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.361\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.401\n",
      "\n",
      "\n",
      "\n",
      "87:belt\n",
      "/home/zyp/w/d2l/detect_outfits/data/origin/valid_local.json\n",
      "/home/zyp/w/d2l/detect_outfits/log/bln_epochs60/0.05-exp/answer.json\n",
      "loading annotations into memory...\n",
      "Done (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "Loading and preparing results...\n",
      "DONE (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *bbox*\n",
      "DONE (t=0.01s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=0.00s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.000\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.000\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.000\n",
      "\n",
      "\n",
      "\n",
      "1034:sunglasses\n",
      "/home/zyp/w/d2l/detect_outfits/data/origin/valid_local.json\n",
      "/home/zyp/w/d2l/detect_outfits/log/bln_epochs60/0.05-exp/answer.json\n",
      "loading annotations into memory...\n",
      "Done (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "Loading and preparing results...\n",
      "DONE (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *bbox*\n",
      "DONE (t=0.10s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=0.02s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.436\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.760\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.444\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.146\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.437\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.613\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.353\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.503\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.505\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.238\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.529\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.671\n",
      "\n",
      "\n",
      "\n",
      "131:boot\n",
      "/home/zyp/w/d2l/detect_outfits/data/origin/valid_local.json\n",
      "/home/zyp/w/d2l/detect_outfits/log/bln_epochs60/0.05-exp/answer.json\n",
      "loading annotations into memory...\n",
      "Done (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "Loading and preparing results...\n",
      "DONE (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *bbox*\n",
      "DONE (t=0.13s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=0.01s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.188\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.345\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.185\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.094\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.221\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.171\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.299\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.299\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.224\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.313\n",
      "\n",
      "\n",
      "\n",
      "318:cowboy_hat\n",
      "/home/zyp/w/d2l/detect_outfits/data/origin/valid_local.json\n",
      "/home/zyp/w/d2l/detect_outfits/log/bln_epochs60/0.05-exp/answer.json\n",
      "loading annotations into memory...\n",
      "Done (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "Loading and preparing results...\n",
      "DONE (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *bbox*\n",
      "DONE (t=0.03s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=0.01s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.373\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.706\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.325\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.390\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.381\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.403\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.474\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.474\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.450\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.476\n",
      "\n",
      "\n",
      "\n",
      "588:jacket\n",
      "/home/zyp/w/d2l/detect_outfits/data/origin/valid_local.json\n",
      "/home/zyp/w/d2l/detect_outfits/log/bln_epochs60/0.05-exp/answer.json\n",
      "loading annotations into memory...\n",
      "Done (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "Loading and preparing results...\n",
      "DONE (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *bbox*\n",
      "DONE (t=0.08s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=0.01s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.398\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.689\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.418\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.207\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.421\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.304\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.522\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.522\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.243\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.547\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def eval(gt_file, dt_file, cat_ids):\n",
    "    print(gt_file)\n",
    "    print(dt_file)\n",
    "    if len(json.load(open(dt_file, 'r'))) == 0:\n",
    "        print(\"EMPTY! \", dt_file)\n",
    "        return\n",
    "    cocoGt = COCO(gt_file)\n",
    "    cocoDt = cocoGt.loadRes(dt_file)\n",
    "\n",
    "    # running evaluation\n",
    "    cocoEval = COCOeval(cocoGt,cocoDt,iouType='bbox')\n",
    "    cocoEval.params.catIds  = cat_ids\n",
    "    cocoEval.evaluate()\n",
    "    cocoEval.accumulate()\n",
    "\n",
    "    cocoEval.summarize()\n",
    "\n",
    "gt_file = f'{p_dir}/data/origin/valid_local.json'\n",
    "dt_file = f'{exp_dir}/answer.json'\n",
    "eval(gt_file, dt_file, cat_ids)\n",
    "print('\\n\\n')\n",
    "\n",
    "for cid in cat_ids:\n",
    "    print(f'{cid}:{cats_dict[cid]}') \n",
    "    eval(gt_file, dt_file, [cid])\n",
    "    print('\\n\\n')\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bfd3b48c",
   "metadata": {},
   "source": [
    "---\n",
    "# 图片分类观察"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "id": "bbe858f5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-06-08T06:50:09.178161Z",
     "start_time": "2022-06-08T06:50:09.164681Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{87: {361189392801794812, 15742841918304422417, 18027551165445256055},\n",
       " 1034: {1051329269795162,\n",
       "  12322045450093703,\n",
       "  30111963444120511,\n",
       "  109127104396560597,\n",
       "  234303366212454352,\n",
       "  237594034503790855,\n",
       "  309151613107896442,\n",
       "  380548125617260848,\n",
       "  582210798814430064,\n",
       "  611995930297011282,\n",
       "  612732370816370444,\n",
       "  640205953571587690,\n",
       "  663160437824981644,\n",
       "  718461807596553802,\n",
       "  722253231118222830,\n",
       "  746054828961565687,\n",
       "  832439245772227152,\n",
       "  928841596103688816,\n",
       "  1137984495341109671,\n",
       "  1240669949590208752,\n",
       "  1285321158120244552,\n",
       "  1303068949300710487,\n",
       "  1350285758125050859,\n",
       "  1474788841078032089,\n",
       "  1505780244926694019,\n",
       "  1518973667491580861,\n",
       "  1571967087446787270,\n",
       "  1626138972757263702,\n",
       "  1665878299524628872,\n",
       "  1767322329468272888,\n",
       "  1781078496580242076,\n",
       "  1855409563895400077,\n",
       "  1916992662359411230,\n",
       "  1966200073333507223,\n",
       "  1981991348494928073,\n",
       "  1982338909631482811,\n",
       "  1985141223068338433,\n",
       "  2112915898585481552,\n",
       "  2218177035611309287,\n",
       "  2256277289317272015,\n",
       "  2283305493816510452,\n",
       "  2289452935013239649,\n",
       "  2400798047493948052,\n",
       "  2426196205656115681,\n",
       "  2445901120283379629,\n",
       "  2581269785293458045,\n",
       "  2684110452832321624,\n",
       "  2704366883452039217,\n",
       "  2713395367162119613,\n",
       "  2760756768683920086,\n",
       "  2796566292397130477,\n",
       "  2857187227633385663,\n",
       "  2872540895448466839,\n",
       "  2885298456529340440,\n",
       "  2910159385377671215,\n",
       "  2938158876283371688,\n",
       "  2938467728867527235,\n",
       "  3046521862611994474,\n",
       "  3084042411437294856,\n",
       "  3198469235171263689,\n",
       "  3288605339635040652,\n",
       "  3332496558875214824,\n",
       "  3419140327262519402,\n",
       "  3434919251017401286,\n",
       "  3500696335668251660,\n",
       "  3528085279492407352,\n",
       "  3586658173135686439,\n",
       "  3635500371787680060,\n",
       "  3658430888054161689,\n",
       "  3703447301976360090,\n",
       "  3709474992202154059,\n",
       "  3904324108930485212,\n",
       "  4042760930597603647,\n",
       "  4066742418324196442,\n",
       "  4071470054909346456,\n",
       "  4118040019090737652,\n",
       "  4137154025788344035,\n",
       "  4185552434613822036,\n",
       "  4252563396947411211,\n",
       "  4336931505913923657,\n",
       "  4341560691582534065,\n",
       "  4468883762907842441,\n",
       "  4484402209189101657,\n",
       "  4521792232513835249,\n",
       "  4525629445820091207,\n",
       "  4556131432338227232,\n",
       "  4598640595634687219,\n",
       "  4617225678360052077,\n",
       "  4646668364494080012,\n",
       "  4706635784441627645,\n",
       "  4707301555880953646,\n",
       "  4707417484469274647,\n",
       "  4728795898516931657,\n",
       "  4753125727256546064,\n",
       "  4765815440999186877,\n",
       "  4774169644061262431,\n",
       "  4989218614788228456,\n",
       "  5133467881759082952,\n",
       "  5182930338823898016,\n",
       "  5232673097732491269,\n",
       "  5233097859204484047,\n",
       "  5284827416798726070,\n",
       "  5314111996025372865,\n",
       "  5343924297653291302,\n",
       "  5352230123301374462,\n",
       "  5387943228095302049,\n",
       "  5403454467915645641,\n",
       "  5482560941588709616,\n",
       "  5484868941851526036,\n",
       "  5501148532789873627,\n",
       "  5504272848151087682,\n",
       "  5531320181795422806,\n",
       "  5638645289377138403,\n",
       "  5710729011934659611,\n",
       "  6039190026149595679,\n",
       "  6048010658051885262,\n",
       "  6076106482609620440,\n",
       "  6154717543661580814,\n",
       "  6191694076112667647,\n",
       "  6232649279544952902,\n",
       "  6239456230510724812,\n",
       "  6255401246120313658,\n",
       "  6413184395330159632,\n",
       "  6413592483885253285,\n",
       "  6511218266802815244,\n",
       "  6555588283181664102,\n",
       "  6564535052440011674,\n",
       "  6592867390010985239,\n",
       "  6595339565575893623,\n",
       "  6599194943775050455,\n",
       "  6611076319330108849,\n",
       "  6682992666199294817,\n",
       "  6689251422544595225,\n",
       "  6756743911051594072,\n",
       "  6791123471590654860,\n",
       "  6806020996490758454,\n",
       "  6827964664801230867,\n",
       "  6935039005795028026,\n",
       "  6942020156444576004,\n",
       "  7014889830545424462,\n",
       "  7021396960595280830,\n",
       "  7086139811995585268,\n",
       "  7121543204932519260,\n",
       "  7124248788343262427,\n",
       "  7184660539322074462,\n",
       "  7216799955255296887,\n",
       "  7297397893907842857,\n",
       "  7393489923608938826,\n",
       "  7402761920314639204,\n",
       "  7427056413137931223,\n",
       "  7465552093003386019,\n",
       "  7485452361317156152,\n",
       "  7492167310613646465,\n",
       "  7660713350982829646,\n",
       "  7675961763337309239,\n",
       "  7809350185552448040,\n",
       "  7846154815584864465,\n",
       "  7880204563697011614,\n",
       "  7931242543287599634,\n",
       "  8109085488976168805,\n",
       "  8118231320731454477,\n",
       "  8150078808617946072,\n",
       "  8172724458527426863,\n",
       "  8179971067011269237,\n",
       "  8493561186008204078,\n",
       "  8545619211903371635,\n",
       "  8551301523214108057,\n",
       "  8670427681095125402,\n",
       "  8748776321518667224,\n",
       "  8798667085752633277,\n",
       "  8826836625468580412,\n",
       "  8977744313475596849,\n",
       "  8990877779853150471,\n",
       "  9010791804619522552,\n",
       "  9012856302230505664,\n",
       "  9138764246206941242,\n",
       "  9138836055418700078,\n",
       "  9152134274479032471,\n",
       "  9217718990977404077,\n",
       "  9241013329614669254,\n",
       "  9277693975496690057,\n",
       "  9279907409635903283,\n",
       "  9396376153387106412,\n",
       "  9396614638197410063,\n",
       "  9411240452772268871,\n",
       "  9539337138986144032,\n",
       "  9625715035648034023,\n",
       "  9786843797522809247,\n",
       "  9819315933966895668,\n",
       "  9854935185815667655,\n",
       "  9915783434409509236,\n",
       "  10014058281592555206,\n",
       "  10023775293813034034,\n",
       "  10168562931551439205,\n",
       "  10173014402019264442,\n",
       "  10205101731515467274,\n",
       "  10254041778968167654,\n",
       "  10271701854510778821,\n",
       "  10293711823090188849,\n",
       "  10315042340401612870,\n",
       "  10351647340494130436,\n",
       "  10419109102424392419,\n",
       "  10504029654568536434,\n",
       "  10505098703874556445,\n",
       "  10565105629840051202,\n",
       "  10568624625521003204,\n",
       "  10575191555793437649,\n",
       "  10600938004858734872,\n",
       "  10614047211542546102,\n",
       "  10664221153528940035,\n",
       "  10676808211641468055,\n",
       "  10704193860542236406,\n",
       "  10733378649585582886,\n",
       "  10749858488493245674,\n",
       "  10902880096530710890,\n",
       "  10943543725097157629,\n",
       "  11035467640764891276,\n",
       "  11077918592262048416,\n",
       "  11182428789780188056,\n",
       "  11351191007868357656,\n",
       "  11376899357295316011,\n",
       "  11385933995178722454,\n",
       "  11579212355051094015,\n",
       "  11580404330314139269,\n",
       "  11594635318640086077,\n",
       "  11598110560695027249,\n",
       "  11956511587667479636,\n",
       "  12062742969098505213,\n",
       "  12081834027749772034,\n",
       "  12092915629815564841,\n",
       "  12100205952956596816,\n",
       "  12295531903464866067,\n",
       "  12420913531771682833,\n",
       "  12451118512885597282,\n",
       "  12470867380652671668,\n",
       "  12584610462733239674,\n",
       "  12645353053713991639,\n",
       "  12695916980795640490,\n",
       "  12802242825334752470,\n",
       "  12812169714933237637,\n",
       "  12831491046559254455,\n",
       "  12850219179867046434,\n",
       "  12888595742608720871,\n",
       "  12891610637324160028,\n",
       "  12951714297518719221,\n",
       "  12955155802574751655,\n",
       "  13101904300012602847,\n",
       "  13107310198409527613,\n",
       "  13144118987709436074,\n",
       "  13175064252498347637,\n",
       "  13225553196092656039,\n",
       "  13258886509146703274,\n",
       "  13270254508350803052,\n",
       "  13311708006529730023,\n",
       "  13321477824604013629,\n",
       "  13354865280159450006,\n",
       "  13366497630173136063,\n",
       "  13416499948728060813,\n",
       "  13435039019669356837,\n",
       "  13453196076894520861,\n",
       "  13464099786839065229,\n",
       "  13516896190717232817,\n",
       "  13569724381052732066,\n",
       "  13665100130565491233,\n",
       "  13768029605765024048,\n",
       "  13802689711343468802,\n",
       "  13808254318587348405,\n",
       "  13863584798223862874,\n",
       "  13912797337393727452,\n",
       "  13921495967385127225,\n",
       "  14173448140542148829,\n",
       "  14257843606941704813,\n",
       "  14280561738673029352,\n",
       "  14343327526824544045,\n",
       "  14426452349536633242,\n",
       "  14567982851571714016,\n",
       "  14642072306031002844,\n",
       "  14684903171259925952,\n",
       "  14710645168656810415,\n",
       "  14725521494309276846,\n",
       "  14750711626181577102,\n",
       "  14820190336415793215,\n",
       "  14826760581675483637,\n",
       "  14845075736061804883,\n",
       "  14846156373384824041,\n",
       "  14968094710882076471,\n",
       "  15008198619510311612,\n",
       "  15058192756080905248,\n",
       "  15130390819252404352,\n",
       "  15170576319669405666,\n",
       "  15252176678376507279,\n",
       "  15405118453255438876,\n",
       "  15407111316054247237,\n",
       "  15430317768905853266,\n",
       "  15576746580641808833,\n",
       "  15697918461187198072,\n",
       "  15742841918304422417,\n",
       "  15836214784034702860,\n",
       "  15846599051419814425,\n",
       "  15935851674294913662,\n",
       "  16016777576946012823,\n",
       "  16177706748348041635,\n",
       "  16449417795648979272,\n",
       "  16454855895458090054,\n",
       "  16515318826902619240,\n",
       "  16548411463351209204,\n",
       "  16710409792424542803,\n",
       "  16740056570912375242,\n",
       "  16798687852312333259,\n",
       "  16943991443158304460,\n",
       "  16990361038906458833,\n",
       "  17319819593864382441,\n",
       "  17361671222225503242,\n",
       "  17445073639585319219,\n",
       "  17560703488640451235,\n",
       "  17571277802569334848,\n",
       "  17579692191741740446,\n",
       "  17688660768333040845,\n",
       "  17747472273926948471,\n",
       "  17862046114836926869,\n",
       "  17865746493702144871,\n",
       "  17878807805699515256,\n",
       "  17883221508049275619,\n",
       "  18032421503599653637,\n",
       "  18064019585154484490,\n",
       "  18124577360176941226,\n",
       "  18126937105667689670,\n",
       "  18143923431386155253,\n",
       "  18182049503330239661,\n",
       "  18282778423379788089,\n",
       "  18367514140051211642,\n",
       "  18436001647700513616,\n",
       "  18441757516110037647},\n",
       " 131: {129079177713156329,\n",
       "  163629646070489479,\n",
       "  197403166931571069,\n",
       "  225571192671666446,\n",
       "  361189392801794812,\n",
       "  415005614690845618,\n",
       "  422927580100059208,\n",
       "  431011366082134725,\n",
       "  722253231118222830,\n",
       "  746054828961565687,\n",
       "  880781782471303749,\n",
       "  1029636628289487028,\n",
       "  1517331449844510932,\n",
       "  1981464537726005917,\n",
       "  2173952321382822637,\n",
       "  2581269785293458045,\n",
       "  3160673984440732882,\n",
       "  4252563396947411211,\n",
       "  4499650502620955212,\n",
       "  4548611005517259282,\n",
       "  4728795898516931657,\n",
       "  4753125727256546064,\n",
       "  4816926677610171608,\n",
       "  4882447797394842490,\n",
       "  5182930338823898016,\n",
       "  5349446069412498484,\n",
       "  5387943228095302049,\n",
       "  5694969506726596815,\n",
       "  5834924333130746019,\n",
       "  6361342986696216822,\n",
       "  7442965608465793685,\n",
       "  7578714033824158460,\n",
       "  7608504560824355207,\n",
       "  8097849524737987287,\n",
       "  8327782685404956837,\n",
       "  8513848987445068404,\n",
       "  9426694868157669777,\n",
       "  9539337138986144032,\n",
       "  9579757113935106805,\n",
       "  10126222363807482456,\n",
       "  10186826553705595228,\n",
       "  10272147831383770013,\n",
       "  10419109102424392419,\n",
       "  10519719465361952873,\n",
       "  10741055394883719605,\n",
       "  12389486278301481112,\n",
       "  13002446459736394438,\n",
       "  13101904300012602847,\n",
       "  13453196076894520861,\n",
       "  13802689711343468802,\n",
       "  13851856976678816436,\n",
       "  14594012944480612952,\n",
       "  14660484272217293223,\n",
       "  14858082324158222432,\n",
       "  15407111316054247237,\n",
       "  15588681276876621630,\n",
       "  16016777576946012823,\n",
       "  16738112267599208485,\n",
       "  17286679165084156072,\n",
       "  17862046114836926869,\n",
       "  17879202626420945636,\n",
       "  18126937105667689670},\n",
       " 318: {1236924810535851,\n",
       "  14469898273883403,\n",
       "  30111963444120511,\n",
       "  91221584055767439,\n",
       "  131204811506606646,\n",
       "  159718802377749007,\n",
       "  163629646070489479,\n",
       "  225196398715822739,\n",
       "  718461807596553802,\n",
       "  1395964540776002045,\n",
       "  1474788841078032089,\n",
       "  1767322329468272888,\n",
       "  1871541381945586428,\n",
       "  2145161729616106010,\n",
       "  2218177035611309287,\n",
       "  2435954503637012821,\n",
       "  2684110452832321624,\n",
       "  2697998183209050321,\n",
       "  2704366883452039217,\n",
       "  2720312598376974056,\n",
       "  2873374505891553669,\n",
       "  2987169401986318415,\n",
       "  3155407330712081183,\n",
       "  3288605339635040652,\n",
       "  3500696335668251660,\n",
       "  3578670370181263676,\n",
       "  3998620645639041227,\n",
       "  4002964439692820881,\n",
       "  4185552434613822036,\n",
       "  4271009591755023257,\n",
       "  4454441222796054859,\n",
       "  4521792232513835249,\n",
       "  4753125727256546064,\n",
       "  5061181095693354036,\n",
       "  5834924333130746019,\n",
       "  5945732250395042006,\n",
       "  6154717543661580814,\n",
       "  6578021406860915270,\n",
       "  6595339565575893623,\n",
       "  6689251422544595225,\n",
       "  7402761920314639204,\n",
       "  7429611928544097252,\n",
       "  7778229653669699649,\n",
       "  7931242543287599634,\n",
       "  8380319250355237659,\n",
       "  8513848987445068404,\n",
       "  8551301523214108057,\n",
       "  8826836625468580412,\n",
       "  8877684231367606830,\n",
       "  8938627875605656027,\n",
       "  9241013329614669254,\n",
       "  9279907409635903283,\n",
       "  9328036797108142804,\n",
       "  9411240452772268871,\n",
       "  9480328123803140852,\n",
       "  9850643803053461672,\n",
       "  9889055444737132466,\n",
       "  9985640253043810406,\n",
       "  10023775293813034034,\n",
       "  10125691149470515673,\n",
       "  10396596135015287645,\n",
       "  10504029654568536434,\n",
       "  10565105629840051202,\n",
       "  10575191555793437649,\n",
       "  10751723499421295224,\n",
       "  10863729095631655226,\n",
       "  10902880096530710890,\n",
       "  11351191007868357656,\n",
       "  11849532236477522152,\n",
       "  12081834027749772034,\n",
       "  12557558520353260803,\n",
       "  12888595742608720871,\n",
       "  13051922260802667252,\n",
       "  13107310198409527613,\n",
       "  13144118987709436074,\n",
       "  13175064252498347637,\n",
       "  13768029605765024048,\n",
       "  13992877745271081688,\n",
       "  14076540767549301626,\n",
       "  14143426525230629654,\n",
       "  14257843606941704813,\n",
       "  14280561738673029352,\n",
       "  14292777138545234433,\n",
       "  14594012944480612952,\n",
       "  15354350893781992427,\n",
       "  15735844816242225624,\n",
       "  15846599051419814425,\n",
       "  15954567595197366460,\n",
       "  16738112267599208485,\n",
       "  16899436376352989682,\n",
       "  17037940196354470820,\n",
       "  17571277802569334848,\n",
       "  17862046114836926869,\n",
       "  17878807805699515256,\n",
       "  18027551165445256055,\n",
       "  18032421503599653637,\n",
       "  18064019585154484490,\n",
       "  18367514140051211642},\n",
       " 588: {7882653480062099,\n",
       "  12322045450093703,\n",
       "  14469898273883403,\n",
       "  36003892120453674,\n",
       "  44257180350186682,\n",
       "  109127104396560597,\n",
       "  129079177713156329,\n",
       "  152471063117694353,\n",
       "  194540194948320024,\n",
       "  225571192671666446,\n",
       "  233682374288966483,\n",
       "  406121626945719239,\n",
       "  499658991158014770,\n",
       "  612732370816370444,\n",
       "  640205953571587690,\n",
       "  663160437824981644,\n",
       "  722253231118222830,\n",
       "  749981824292359277,\n",
       "  880781782471303749,\n",
       "  963016938462577225,\n",
       "  1350285758125050859,\n",
       "  1474788841078032089,\n",
       "  1504343790265566862,\n",
       "  1505780244926694019,\n",
       "  1553720624601040028,\n",
       "  1626138972757263702,\n",
       "  1753145250752465552,\n",
       "  1810971313380360435,\n",
       "  1880461203843875359,\n",
       "  1916992662359411230,\n",
       "  1981991348494928073,\n",
       "  1994965430299658431,\n",
       "  2218177035611309287,\n",
       "  2221187077996055642,\n",
       "  2231299513403442034,\n",
       "  2256277289317272015,\n",
       "  2269423983223091662,\n",
       "  2283305493816510452,\n",
       "  2373972791203793636,\n",
       "  2442486155143431682,\n",
       "  2445901120283379629,\n",
       "  2545399075610626470,\n",
       "  2671220527567996420,\n",
       "  2684110452832321624,\n",
       "  2704366883452039217,\n",
       "  2713395367162119613,\n",
       "  2720312598376974056,\n",
       "  2760756768683920086,\n",
       "  2763477049389286405,\n",
       "  2774242970372343952,\n",
       "  2782413500153954875,\n",
       "  2834318722035616406,\n",
       "  2948087986851841679,\n",
       "  2963284519494420043,\n",
       "  2987169401986318415,\n",
       "  3096533552427827802,\n",
       "  3129725882096124412,\n",
       "  3160673984440732882,\n",
       "  3185882869663466602,\n",
       "  3369968592491480421,\n",
       "  3419140327262519402,\n",
       "  3430217035398383644,\n",
       "  3568578828985237633,\n",
       "  3658430888054161689,\n",
       "  3729085702800901256,\n",
       "  3845507853590071683,\n",
       "  3889848957158525243,\n",
       "  3890401139036818344,\n",
       "  3892917444373973289,\n",
       "  3998620645639041227,\n",
       "  4015822448835529272,\n",
       "  4042760930597603647,\n",
       "  4066742418324196442,\n",
       "  4213162884423123255,\n",
       "  4468883762907842441,\n",
       "  4484402209189101657,\n",
       "  4525629445820091207,\n",
       "  4625755888220800869,\n",
       "  4651304649399504020,\n",
       "  4664101340805038085,\n",
       "  4728795898516931657,\n",
       "  4753125727256546064,\n",
       "  4816926677610171608,\n",
       "  4818870340515924424,\n",
       "  4844463938115896471,\n",
       "  5021749895197892839,\n",
       "  5061181095693354036,\n",
       "  5182930338823898016,\n",
       "  5314111996025372865,\n",
       "  5343924297653291302,\n",
       "  5349446069412498484,\n",
       "  5501148532789873627,\n",
       "  5540536366021282805,\n",
       "  5638645289377138403,\n",
       "  5762830396441102412,\n",
       "  5961784329751170080,\n",
       "  6046233555117049282,\n",
       "  6048010658051885262,\n",
       "  6088698192374205613,\n",
       "  6191694076112667647,\n",
       "  6227052611673916866,\n",
       "  6271165056271581288,\n",
       "  6334138831767505290,\n",
       "  6358834569215443302,\n",
       "  6433198507259888465,\n",
       "  6451265900487838063,\n",
       "  6544725785763643588,\n",
       "  6578021406860915270,\n",
       "  6599194943775050455,\n",
       "  6791123471590654860,\n",
       "  6935039005795028026,\n",
       "  7014889830545424462,\n",
       "  7021396960595280830,\n",
       "  7086139811995585268,\n",
       "  7276767453136389232,\n",
       "  7284150763761470003,\n",
       "  7429334482308760067,\n",
       "  7473992051537972077,\n",
       "  7485452361317156152,\n",
       "  7608504560824355207,\n",
       "  7631204979618252479,\n",
       "  7646089654749102884,\n",
       "  7660713350982829646,\n",
       "  7751559312693004876,\n",
       "  7809350185552448040,\n",
       "  7832136250470041636,\n",
       "  7880204563697011614,\n",
       "  8002994150041880858,\n",
       "  8150078808617946072,\n",
       "  8179971067011269237,\n",
       "  8327782685404956837,\n",
       "  8411060938190292433,\n",
       "  8595059693287801620,\n",
       "  8663668411557972848,\n",
       "  8946860741468914862,\n",
       "  9046773008794662425,\n",
       "  9060712207924332481,\n",
       "  9303049203019320031,\n",
       "  9396376153387106412,\n",
       "  9427043956226885244,\n",
       "  9443660659611205659,\n",
       "  9483805788083552844,\n",
       "  9491379842992996352,\n",
       "  9559476362514825680,\n",
       "  9647568288402180803,\n",
       "  9745641618502911656,\n",
       "  9786843797522809247,\n",
       "  9819315933966895668,\n",
       "  9850643803053461672,\n",
       "  9854935185815667655,\n",
       "  9875918191732590472,\n",
       "  9889055444737132466,\n",
       "  10023775293813034034,\n",
       "  10102069283307914829,\n",
       "  10126222363807482456,\n",
       "  10173014402019264442,\n",
       "  10293091206462637668,\n",
       "  10298335377470227275,\n",
       "  10396596135015287645,\n",
       "  10504029654568536434,\n",
       "  10519719465361952873,\n",
       "  10658615151618132152,\n",
       "  10676808211641468055,\n",
       "  10703971434732716081,\n",
       "  10863729095631655226,\n",
       "  10943543725097157629,\n",
       "  11035467640764891276,\n",
       "  11056367724053299679,\n",
       "  11087216166170755671,\n",
       "  11161668664974543218,\n",
       "  11351191007868357656,\n",
       "  11365268465171270013,\n",
       "  11579212355051094015,\n",
       "  11672322185624715645,\n",
       "  11753039704187963669,\n",
       "  11849532236477522152,\n",
       "  11927765013648836858,\n",
       "  12005219712062982002,\n",
       "  12082873435589969452,\n",
       "  12250414504260077686,\n",
       "  12352448932365108422,\n",
       "  12420913531771682833,\n",
       "  12557558520353260803,\n",
       "  12625736473427181652,\n",
       "  12645353053713991639,\n",
       "  12687169766246023629,\n",
       "  12801279001559796415,\n",
       "  12812169714933237637,\n",
       "  12920729038905183286,\n",
       "  13002446459736394438,\n",
       "  13051922260802667252,\n",
       "  13141489865741927452,\n",
       "  13175064252498347637,\n",
       "  13205326900291928416,\n",
       "  13266760700493563626,\n",
       "  13270254508350803052,\n",
       "  13309908261647256437,\n",
       "  13311708006529730023,\n",
       "  13321477824604013629,\n",
       "  13415416439841063802,\n",
       "  13431939200385714014,\n",
       "  13432068203811277665,\n",
       "  13453196076894520861,\n",
       "  13527861041464935658,\n",
       "  13536661239795122438,\n",
       "  13550434398052648023,\n",
       "  13768029605765024048,\n",
       "  13798499693784565263,\n",
       "  13808254318587348405,\n",
       "  13851856976678816436,\n",
       "  13863584798223862874,\n",
       "  14019134061893814457,\n",
       "  14055518650453035257,\n",
       "  14076540767549301626,\n",
       "  14079653347337274443,\n",
       "  14107250227799556434,\n",
       "  14114366336811724442,\n",
       "  14173448140542148829,\n",
       "  14271913302580235272,\n",
       "  14280561738673029352,\n",
       "  14283232099584803620,\n",
       "  14343327526824544045,\n",
       "  14382564896264947628,\n",
       "  14594012944480612952,\n",
       "  14660484272217293223,\n",
       "  14684903171259925952,\n",
       "  14826760581675483637,\n",
       "  14952223541450604873,\n",
       "  15029412485693178440,\n",
       "  15058192756080905248,\n",
       "  15170576319669405666,\n",
       "  15208594918626558408,\n",
       "  15346435050801760037,\n",
       "  15430317768905853266,\n",
       "  15539470226144002466,\n",
       "  15576746580641808833,\n",
       "  15588681276876621630,\n",
       "  15620990830163561664,\n",
       "  15637851809725154047,\n",
       "  15689360184757684459,\n",
       "  15737766506096133249,\n",
       "  15756126907672035675,\n",
       "  15766349096669512041,\n",
       "  15783393575846445204,\n",
       "  15800467872957958463,\n",
       "  15821460922009879220,\n",
       "  15946399028261587608,\n",
       "  16260112100939366463,\n",
       "  16454855895458090054,\n",
       "  16477766384044474817,\n",
       "  16510491732513098040,\n",
       "  16519394007465886408,\n",
       "  16636246187966138834,\n",
       "  16740056570912375242,\n",
       "  16798494727509410405,\n",
       "  16826234764308876414,\n",
       "  16880812242345886885,\n",
       "  16943991443158304460,\n",
       "  16947891437095028478,\n",
       "  16950217592651516426,\n",
       "  16964891005553508856,\n",
       "  16990361038906458833,\n",
       "  17064226764686953232,\n",
       "  17259787909477338087,\n",
       "  17286679165084156072,\n",
       "  17319819593864382441,\n",
       "  17361671222225503242,\n",
       "  17391206429943825642,\n",
       "  17530579275155878049,\n",
       "  17571277802569334848,\n",
       "  17597506408669054449,\n",
       "  17688660768333040845,\n",
       "  17744692971153557227,\n",
       "  17852017246389077276,\n",
       "  17862046114836926869,\n",
       "  17883221508049275619,\n",
       "  17910308277066534016,\n",
       "  17988861403555431660,\n",
       "  18061070582157806086,\n",
       "  18115547698591533245,\n",
       "  18124577360176941226,\n",
       "  18172545432611457220,\n",
       "  18310794816329745612,\n",
       "  18351124882918574448,\n",
       "  18441757516110037647}}"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_imgs = {}\n",
    "for c in cat_ids:\n",
    "    cat_imgs[c] = set()\n",
    "for d in dt.to_numpy():\n",
    "    img_id = d[0]\n",
    "    cat_id = d[1]\n",
    "    cat_imgs[cat_id].add(img_id)\n",
    "\n",
    "cat_imgs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "id": "623fe437",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-06-08T06:50:09.367342Z",
     "start_time": "2022-06-08T06:50:09.179650Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/zyp/w/d2l/detect_outfits/log/bln_epochs60/0.05-exp/belt\n",
      "/home/zyp/w/d2l/detect_outfits/log/bln_epochs60/0.05-exp/sunglasses\n",
      "/home/zyp/w/d2l/detect_outfits/log/bln_epochs60/0.05-exp/boot\n",
      "/home/zyp/w/d2l/detect_outfits/log/bln_epochs60/0.05-exp/cowboy_hat\n",
      "/home/zyp/w/d2l/detect_outfits/log/bln_epochs60/0.05-exp/jacket\n"
     ]
    }
   ],
   "source": [
    "for cat_id in cat_imgs:\n",
    "    cat_name = cats_dict[cat_id]\n",
    "    os.makedirs(f'{exp_dir}/{cat_name}', exist_ok=True)\n",
    "    print(f'{exp_dir}/{cat_name}')\n",
    "    \n",
    "    img_ids = cat_imgs[cat_id]\n",
    "    for img_id in img_ids:\n",
    "        img = imgs.loc[img_id]\n",
    "        img_file = img.file_name\n",
    "        sh.copy(f'{exp_dir}/{img_file}', f'{exp_dir}/{cat_name}/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "id": "8608a735",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-06-08T06:58:59.268949Z",
     "start_time": "2022-06-08T06:58:59.265123Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "361189392801794812 050333d6ce970afc.jpg True\n",
      "18027551165445256055 fa2eba455a8b3377.jpg True\n",
      "15742841918304422417 da79cf50c7aa0a11.jpg True\n"
     ]
    }
   ],
   "source": [
    "for img_id in cat_imgs[87]:\n",
    "    img = imgs.loc[img_id]\n",
    "    in_valid = img_id in valid_imgs.index\n",
    "    print(img_id, img.file_name, in_valid)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0765c956",
   "metadata": {},
   "source": [
    "---\n",
    "# Eval by YOLOv5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "id": "f2234e49",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-06-08T07:44:01.189389Z",
     "start_time": "2022-06-08T07:43:04.945479Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mval: \u001b[0mdata=/home/zyp/w/d2l/detect_outfits/data/yolo2/data.yaml, weights=['/home/zyp/w/d2l/detect_outfits/log/bln_epochs40/weights/last.pt'], batch_size=32, imgsz=640, conf_thres=0.05, iou_thres=0.5, task=val, device=, workers=8, single_cls=False, augment=False, verbose=False, save_txt=False, save_hybrid=False, save_conf=False, save_json=False, project=../../github/yolov5/runs/val, name=/home/zyp/w/d2l/detect_outfits/log/yolo_evl/, exist_ok=False, half=False, dnn=False\n",
      "WARNING: confidence threshold 0.05 > 0.001 produces invalid results ⚠️\n",
      "YOLOv5 🚀 v6.1-237-g50b886d Python-3.8.12 torch-1.11.0+cu102 CPU\n",
      "\n",
      "Fusing layers... \n",
      "Model summary: 213 layers, 7023610 parameters, 0 gradients, 15.8 GFLOPs\n",
      "\u001b[34m\u001b[1mval: \u001b[0mScanning '/home/zyp/w/d2l/detect_outfits/data/yolo2/valid/labels.cache' ima\u001b[0m\n",
      "               Class     Images     Labels          P          R     mAP@.5 mAP@\n",
      "                 all        613       1114      0.821      0.517      0.565      0.343\n",
      "                belt        613          6          1          0          0          0\n",
      "          sunglasses        613        501      0.858      0.723      0.821      0.506\n",
      "                boot        613        102      0.768      0.373      0.489      0.303\n",
      "          cowboy_hat        613         88      0.709      0.807       0.76      0.436\n",
      "              jacket        613        417      0.768      0.683      0.755      0.471\n",
      "Speed: 1.0ms pre-process, 81.4ms inference, 0.3ms NMS per image at shape (32, 3, 640, 640)\n",
      "Results saved to \u001b[1m/home/zyp/w/d2l/detect_outfits/log/yolo_evl3\u001b[0m\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n",
      "Exception ignored in: <function StorageWeakRef.__del__ at 0x7f96bd456040>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 36, in __del__\n",
      "  File \"/home/zyp/miniconda3/lib/python3.8/site-packages/torch/storage.py\", line 520, in _free_weak_ref\n",
      "AttributeError: 'NoneType' object has no attribute '_free_weak_ref'\n"
     ]
    }
   ],
   "source": [
    "!python ~/w/github/yolov5/val.py --weights {p_dir}/log/bln_epochs40/weights/last.pt \\\n",
    "                            --data {p_dir}/data/yolo2/data.yaml --name {p_dir}/log/yolo_evl/ --iou-thres 0.5 \\\n",
    "                            --conf-thres 0.05 --img 640 "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}