MLclassifiers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "missing-letter",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import torch\n",
    "from torchvision import datasets, models, transforms\n",
    "from tqdm import tqdm\n",
    "from PIL import Image\n",
    "from torch.autograd import Variable\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.svm import SVC, LinearSVC\n",
    "from sklearn.model_selection import cross_val_score, GridSearchCV\n",
    "from sklearn.metrics import confusion_matrix, accuracy_score, precision_score, recall_score, f1_score, classification_report, roc_curve, auc\n",
    "from efficientnet_pytorch import EfficientNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "disciplinary-drink",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = 'Input_dataset'\n",
    "class_names = ['Diptera', 'Tephritidae']\n",
    "# train_path = data_dir +'/train/'\n",
    "# val_path = data_dir +'/val/'\n",
    "# test_path = data_dir +'/test/'\n",
    "# device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "developmental-vampire",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in C:\\Users\\dukes/.cache\\torch\\hub\\pytorch_vision_v0.9.0\n"
     ]
    }
   ],
   "source": [
    "# model = torch.hub.load('pytorch/vision:v0.9.0', 'resnet50', pretrained=True)\n",
    "# layer = model._modules.get('avgpool')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "introductory-motel",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = torch.hub.load('pytorch/vision:v0.9.0', 'resnet152', pretrained=True)\n",
    "layer = model._modules.get('avgpool')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "premium-company",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method Module.eval of ResNet(\n",
       "  (conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
       "  (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (relu): ReLU(inplace=True)\n",
       "  (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
       "  (layer1): Sequential(\n",
       "    (0): Bottleneck(\n",
       "      (conv1): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): Bottleneck(\n",
       "      (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "    )\n",
       "    (2): Bottleneck(\n",
       "      (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "    )\n",
       "  )\n",
       "  (layer2): Sequential(\n",
       "    (0): Bottleneck(\n",
       "      (conv1): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "        (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): Bottleneck(\n",
       "      (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "    )\n",
       "    (2): Bottleneck(\n",
       "      (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "    )\n",
       "    (3): Bottleneck(\n",
       "      (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "    )\n",
       "  )\n",
       "  (layer3): Sequential(\n",
       "    (0): Bottleneck(\n",
       "      (conv1): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(512, 1024, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "        (1): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "    )\n",
       "    (2): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "    )\n",
       "    (3): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "    )\n",
       "    (4): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "    )\n",
       "    (5): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "    )\n",
       "  )\n",
       "  (layer4): Sequential(\n",
       "    (0): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(1024, 2048, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "        (1): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): Bottleneck(\n",
       "      (conv1): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "    )\n",
       "    (2): Bottleneck(\n",
       "      (conv1): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace=True)\n",
       "    )\n",
       "  )\n",
       "  (avgpool): AdaptiveAvgPool2d(output_size=(1, 1))\n",
       "  (fc): Linear(in_features=2048, out_features=1000, bias=True)\n",
       ")>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "educational-facial",
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = transforms.Resize((224, 224))\n",
    "normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
    "                                 std=[0.229, 0.224, 0.225])\n",
    "to_tensor = transforms.ToTensor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "governing-appeal",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████| 45103/45103 [2:25:38<00:00,  5.16it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████| 5287/5287 [19:14<00:00,  4.58it/s]\n"
     ]
    }
   ],
   "source": [
    "# contains the hog and color features from images\n",
    "features_list = []\n",
    "# labels\n",
    "Y = []\n",
    "# number of classes\n",
    "l = len(class_names)\n",
    "\n",
    "# Function to Extract features from the images\n",
    "def create_feature_matrix(folderName):\n",
    "    sourceFiles=os.listdir(data_dir +'/'+ folderName)\n",
    "    for i in tqdm(sourceFiles):\n",
    "        # load image\n",
    "        file_path = os.path.join(data_dir +'/'+ folderName +'/'+i)\n",
    "        img = Image.open(file_path).convert(\"RGB\")\n",
    "        t_img = Variable(normalize(to_tensor(scaler(img))).unsqueeze(0))\n",
    "#         print(np.array(img).shape)\n",
    "#         print(t_img.shape)\n",
    "#         print(t_img)\n",
    "\n",
    "        # layer_output_size = 2048 for ResNet50/ ResNeSt50\n",
    "        # layer_output_size = 1280 for EfficientNet-B0\n",
    "        features = torch.zeros(1, 2048, 1, 1)\n",
    "        def copy_data(m,i,o):\n",
    "            features.copy_(o.data)\n",
    "        h = layer.register_forward_hook(copy_data)\n",
    "        model(t_img)\n",
    "        h.remove()\n",
    "        features = features.numpy()[:, :, 0, 0]\n",
    "        features_list.append(features)\n",
    "        \n",
    "        for j in range(l):\n",
    "            if (folderName==class_names[j]):\n",
    "                Y.append(j)\n",
    "              \n",
    "                \n",
    "# Organize file names and class labels in X and Y variables\n",
    "for j in range(l):\n",
    "    create_feature_matrix(class_names[j])  \n",
    "    \n",
    "# convert list of arrays into a matrix     \n",
    "X_features = np.array(features_list)\n",
    "Y = np.array(Y) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "unavailable-compound",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50390, 2048)\n",
      "(50390,)\n"
     ]
    }
   ],
   "source": [
    "X = X_features.reshape((X_features.shape[0],X_features.shape[2]))\n",
    "print(X.shape)\n",
    "print(Y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "secure-technician",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(45351, 2048)\n",
      "(45351,)\n",
      "(5039, 2048)\n",
      "(5039,)\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y,\n",
    "                                            stratify=Y, \n",
    "                                            test_size= 1/10)\n",
    "print(X_train.shape)\n",
    "print(Y_train.shape)\n",
    "print(X_test.shape)\n",
    "print(Y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "published-explorer",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC()"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# svm_model = SVC(kernel='rbf')\n",
    "# svm_model.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "stuck-afternoon",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(class_weight={1: 8.5})"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wclf = SVC(kernel='rbf', class_weight={1: 8.5})\n",
    "wclf.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "choice-cassette",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Y_pred = svm_model.predict(X_test)\n",
    "Y_predw = wclf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "numerous-malta",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[4510    0]\n",
      " [ 529    0]]\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Diptera       0.90      1.00      0.94      4510\n",
      " Tephritidae       0.00      0.00      0.00       529\n",
      "\n",
      "    accuracy                           0.90      5039\n",
      "   macro avg       0.45      0.50      0.47      5039\n",
      "weighted avg       0.80      0.90      0.85      5039\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\dukes\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\dukes\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\dukes\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set score for SVM: 0.895085\n",
      "Testing  set score for SVM: 0.895019\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<bound method ClassifierMixin.score of SVC()>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# print(confusion_matrix(Y_test, Y_pred))\n",
    "# print(\"\\n\")\n",
    "# print(classification_report(Y_test, Y_pred, target_names= class_names))\n",
    "\n",
    "# print(\"Training set score for SVM: %f\" % svm_model.score(X_train , Y_train))\n",
    "# print(\"Testing  set score for SVM: %f\" % svm_model.score(X_test  , Y_test))\n",
    "\n",
    "# svm_model.score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "premier-creature",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[3462 1048]\n",
      " [ 143  386]]\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Diptera       0.96      0.77      0.85      4510\n",
      " Tephritidae       0.27      0.73      0.39       529\n",
      "\n",
      "    accuracy                           0.76      5039\n",
      "   macro avg       0.61      0.75      0.62      5039\n",
      "weighted avg       0.89      0.76      0.80      5039\n",
      "\n",
      "Training set score for SVM: 0.800820\n",
      "Testing  set score for SVM: 0.763644\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<bound method ClassifierMixin.score of SVC(class_weight={1: 8.5})>"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(confusion_matrix(Y_test, Y_predw))\n",
    "print(\"\\n\")\n",
    "print(classification_report(Y_test, Y_predw, target_names= class_names))\n",
    "\n",
    "print(\"Training set score for SVM: %f\" % wclf.score(X_train , Y_train))\n",
    "print(\"Testing  set score for SVM: %f\" % wclf.score(X_test  , Y_test))\n",
    "\n",
    "wclf.score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "faced-essence",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best C: 1.0 \n",
      "\n",
      "Best Kernel: rbf \n",
      "\n",
      "Best Gamma: scale \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Best C:',wclf.C,\"\\n\") \n",
    "print('Best Kernel:',wclf.kernel,\"\\n\")\n",
    "print('Best Gamma:',wclf.gamma,\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "designing-march",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, estimator=SVC(),\n",
       "             param_grid=[{'C': [1, 10, 100],\n",
       "                          'gamma': [0.5, 0.01, 0.001, 0.0001],\n",
       "                          'kernel': ['rbf']}])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params_grid = [{'kernel': ['rbf'], 'gamma': [0.5, 0.01, 1e-3, 1e-4],\n",
    "                     'C': [1, 10, 100]}]\n",
    "# A low C makes the decision surface smooth, while a high C aims at classifying \n",
    "# all training examples correctly.\n",
    "\n",
    "# Performing CV to tune parameters for best SVM fit \n",
    "svm_model = GridSearchCV(SVC(), params_grid, cv=5)\n",
    "svm_model.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "curious-sharp",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score for training data: 0.9057472799056174 \n",
      "\n",
      "Best C: 10 \n",
      "\n",
      "Best Kernel: rbf \n",
      "\n",
      "Best Gamma: 0.5 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# View the accuracy score\n",
    "print('Best score for training data:', svm_model.best_score_,\"\\n\") \n",
    "\n",
    "# View the best parameters for the model found using grid search\n",
    "print('Best C:',svm_model.best_estimator_.C,\"\\n\") \n",
    "print('Best Kernel:',svm_model.best_estimator_.kernel,\"\\n\")\n",
    "print('Best Gamma:',svm_model.best_estimator_.gamma,\"\\n\")\n",
    "\n",
    "final_model = svm_model.best_estimator_\n",
    "Y_pred = final_model.predict(X_test)\n",
    "# Y_pred_label = list(encoder.inverse_transform(Y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "unexpected-interval",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[11170   106]\n",
      " [ 1089   233]]\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Diptera       0.91      0.99      0.95     11276\n",
      " Tephritidae       0.69      0.18      0.28      1322\n",
      "\n",
      "    accuracy                           0.91     12598\n",
      "   macro avg       0.80      0.58      0.61     12598\n",
      "weighted avg       0.89      0.91      0.88     12598\n",
      "\n",
      "Training set score for SVM: 0.999974\n",
      "Testing  set score for SVM: 0.905144\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<bound method BaseSearchCV.score of GridSearchCV(cv=5, estimator=SVC(),\n",
       "             param_grid=[{'C': [1, 10, 100],\n",
       "                          'gamma': [0.5, 0.01, 0.001, 0.0001],\n",
       "                          'kernel': ['rbf']}])>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Making the Confusion Matrix\n",
    "#print(pd.crosstab(Y_test_label, Y_pred_label, rownames=['Actual Activity'], colnames=['Predicted Activity']))\n",
    "print(confusion_matrix(Y_test, Y_pred))\n",
    "print(\"\\n\")\n",
    "print(classification_report(Y_test, Y_pred, target_names= class_names))\n",
    "\n",
    "print(\"Training set score for SVM: %f\" % final_model.score(X_train , Y_train))\n",
    "print(\"Testing  set score for SVM: %f\" % final_model.score(X_test  , Y_test))\n",
    "\n",
    "svm_model.score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "descending-thompson",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8954157570946616\n",
      "[[4502    8]\n",
      " [ 519   10]]\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Diptera       0.90      1.00      0.94      4510\n",
      " Tephritidae       0.56      0.02      0.04       529\n",
      "\n",
      "    accuracy                           0.90      5039\n",
      "   macro avg       0.73      0.51      0.49      5039\n",
      "weighted avg       0.86      0.90      0.85      5039\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# training a KNN classifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn = KNeighborsClassifier(n_neighbors = 4).fit(X_train, Y_train)\n",
    "  \n",
    "# accuracy on X_test\n",
    "accuracy = knn.score(X_test, Y_test)\n",
    "print(accuracy)\n",
    "  \n",
    "# creating a confusion matrix\n",
    "knn_predictions = knn.predict(X_test) \n",
    "cm = confusion_matrix(Y_test, knn_predictions)\n",
    "print(cm)\n",
    "print(\"\\n\")\n",
    "print(classification_report(Y_test, knn_predictions, target_names= class_names))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "informational-problem",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7328835086326652\n",
      "[[3328 1182]\n",
      " [ 164  365]]\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Diptera       0.95      0.74      0.83      4510\n",
      " Tephritidae       0.24      0.69      0.35       529\n",
      "\n",
      "    accuracy                           0.73      5039\n",
      "   macro avg       0.59      0.71      0.59      5039\n",
      "weighted avg       0.88      0.73      0.78      5039\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# training a Naive Bayes classifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "gnb = GaussianNB().fit(X_train, Y_train)\n",
    "gnb_predictions = gnb.predict(X_test)\n",
    "  \n",
    "# accuracy on X_test\n",
    "accuracy = gnb.score(X_test, Y_test)\n",
    "print(accuracy)\n",
    "  \n",
    "# creating a confusion matrix\n",
    "cm = confusion_matrix(Y_test, gnb_predictions)\n",
    "print(cm)\n",
    "print(\"\\n\")\n",
    "print(classification_report(Y_test, gnb_predictions, target_names= class_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "aerial-wings",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6793014487001389\n",
      "[[3041 1469]\n",
      " [ 147  382]]\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Diptera       0.95      0.67      0.79      4510\n",
      " Tephritidae       0.21      0.72      0.32       529\n",
      "\n",
      "    accuracy                           0.68      5039\n",
      "   macro avg       0.58      0.70      0.56      5039\n",
      "weighted avg       0.88      0.68      0.74      5039\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# training a Naive Bayes classifier\n",
    "from sklearn.naive_bayes import ComplementNB\n",
    "cmb = ComplementNB().fit(X_train, Y_train)\n",
    "cmb_predictions = cmb.predict(X_test)\n",
    "  \n",
    "# accuracy on X_test\n",
    "accuracy = cmb.score(X_test, Y_test)\n",
    "print(accuracy)\n",
    "  \n",
    "# creating a confusion matrix\n",
    "cm = confusion_matrix(Y_test, cmb_predictions)\n",
    "print(cm)\n",
    "print(\"\\n\")\n",
    "print(classification_report(Y_test, cmb_predictions, target_names= class_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "signal-hamburg",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8950188529470133\n",
      "[[4510    0]\n",
      " [ 529    0]]\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Diptera       0.90      1.00      0.94      4510\n",
      " Tephritidae       0.00      0.00      0.00       529\n",
      "\n",
      "    accuracy                           0.90      5039\n",
      "   macro avg       0.45      0.50      0.47      5039\n",
      "weighted avg       0.80      0.90      0.85      5039\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\dukes\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\dukes\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\dukes\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "# training a Naive Bayes classifier\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "cmb = MultinomialNB().fit(X_train, Y_train)\n",
    "cmb_predictions = cmb.predict(X_test)\n",
    "  \n",
    "# accuracy on X_test\n",
    "accuracy = cmb.score(X_test, Y_test)\n",
    "print(accuracy)\n",
    "  \n",
    "# creating a confusion matrix\n",
    "cm = confusion_matrix(Y_test, cmb_predictions)\n",
    "print(cm)\n",
    "print(\"\\n\")\n",
    "print(classification_report(Y_test, cmb_predictions, target_names= class_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "measured-attitude",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}