ObjectDetectionTransferLearning.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/suryan-s/AWS-v0.1/blob/main/ObjectDetectionTransferLearning.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QUANWN3rpfC9"
      },
      "source": [
        "# 0. Setup Paths"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "interval = setInterval(function() { \n",
        "    console.log(\"working\")\n",
        "    var selector = \"#top-toolbar > colab-connect-button\"\n",
        "    document.querySelector(\"#top-toolbar > colab-connect-button\").shadowRoot.querySelector(\"#connect\").click()\n",
        "    setTimeout(function() {\n",
        "            document.querySelector(\"#top-toolbar > colab-connect-button\").shadowRoot.querySelector(\"#connect\").click()\n",
        "    }, 1000)\n",
        "}, 60*1000)"
      ],
      "metadata": {
        "id": "YzDnZIGL7O7h"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!cp /content/drive/MyDrive/Project/images /content/Tensorflow/workspace"
      ],
      "metadata": {
        "id": "OVKLxcU-ggIs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "\n",
        "os.environ['KAGGLE_USERNAME'] = \"suryanleo\" # username from the json file\n",
        "os.environ['KAGGLE_KEY'] = \"a8a0350c0ed88e63fa51d3f303d8fdfd\" # key from the json file\n",
        "\n",
        "!kaggle datasets download -d andrewmvd/face-mask-detection"
      ],
      "metadata": {
        "id": "bEkEtec8p1N7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "146BB11JpfDA"
      },
      "outputs": [],
      "source": [
        "import os"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "42hJEdo_pfDB"
      },
      "outputs": [],
      "source": [
        "CUSTOM_MODEL_NAME = 'faster_rcnn_resnet101_v1' \n",
        "PRETRAINED_MODEL_NAME = 'faster_rcnn_resnet101_v1_640x640_coco17_tpu-8'\n",
        "PRETRAINED_MODEL_URL = 'http://download.tensorflow.org/models/object_detection/tf2/20200711/faster_rcnn_resnet101_v1_640x640_coco17_tpu-8.tar.gz'\n",
        "TF_RECORD_SCRIPT_NAME = 'generate_tfrecord.py'\n",
        "LABEL_MAP_NAME = 'label_map.pbtxt'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hbPhYVy_pfDB"
      },
      "outputs": [],
      "source": [
        "paths = {\n",
        "    'WORKSPACE_PATH': os.path.join('Tensorflow', 'workspace'),\n",
        "    'SCRIPTS_PATH': os.path.join('Tensorflow','scripts'),\n",
        "    'APIMODEL_PATH': os.path.join('Tensorflow','models'),\n",
        "    'ANNOTATION_PATH': os.path.join('Tensorflow', 'workspace','annotations'),\n",
        "    'IMAGE_PATH': os.path.join('Tensorflow', 'workspace','images'),\n",
        "    'MODEL_PATH': os.path.join('Tensorflow', 'workspace','models'),\n",
        "    'PRETRAINED_MODEL_PATH': os.path.join('Tensorflow', 'workspace','pre-trained-models'),\n",
        "    'CHECKPOINT_PATH': os.path.join('Tensorflow', 'workspace','models',CUSTOM_MODEL_NAME), \n",
        "    'OUTPUT_PATH': os.path.join('Tensorflow', 'workspace','models',CUSTOM_MODEL_NAME, 'export'), \n",
        "    'TFJS_PATH':os.path.join('Tensorflow', 'workspace','models',CUSTOM_MODEL_NAME, 'tfjsexport'), \n",
        "    'TFLITE_PATH':os.path.join('Tensorflow', 'workspace','models',CUSTOM_MODEL_NAME, 'tfliteexport'), \n",
        "    'PROTOC_PATH':os.path.join('Tensorflow','protoc')\n",
        " }"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LwhWZMI0pfDC"
      },
      "outputs": [],
      "source": [
        "files = {\n",
        "    'PIPELINE_CONFIG':os.path.join('Tensorflow', 'workspace','models', CUSTOM_MODEL_NAME, 'pipeline.config'),\n",
        "    'TF_RECORD_SCRIPT': os.path.join(paths['SCRIPTS_PATH'], TF_RECORD_SCRIPT_NAME), \n",
        "    'LABELMAP': os.path.join(paths['ANNOTATION_PATH'], LABEL_MAP_NAME)\n",
        "}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HR-TfDGrpfDC"
      },
      "outputs": [],
      "source": [
        "for path in paths.values():\n",
        "    if not os.path.exists(path):\n",
        "        if os.name == 'posix':\n",
        "            !mkdir -p {path}\n",
        "        if os.name == 'nt':\n",
        "            !mkdir {path}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OLU-rs_ipfDE"
      },
      "source": [
        "# 1. Download TF Models Pretrained Models from Tensorflow Model Zoo and Install TFOD"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "K-Cmz2edpfDE",
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "\n",
        "!pip install wget\n",
        "import wget"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iA1DIq5OpfDE"
      },
      "outputs": [],
      "source": [
        "if not os.path.exists(os.path.join(paths['APIMODEL_PATH'], 'research', 'object_detection')):\n",
        "    !git clone https://github.com/tensorflow/models {paths['APIMODEL_PATH']}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rJjMHbnDs3Tv"
      },
      "outputs": [],
      "source": [
        "# Install Tensorflow Object Detection \n",
        "if os.name=='posix':  \n",
        "    !apt-get install protobuf-compiler\n",
        "    !cd Tensorflow/models/research && protoc object_detection/protos/*.proto --python_out=. && cp object_detection/packages/tf2/setup.py . && python -m pip install . \n",
        "    \n",
        "if os.name=='nt':\n",
        "    url=\"https://github.com/protocolbuffers/protobuf/releases/download/v3.15.6/protoc-3.15.6-win64.zip\"\n",
        "    wget.download(url)\n",
        "    !move protoc-3.15.6-win64.zip {paths['PROTOC_PATH']}\n",
        "    !cd {paths['PROTOC_PATH']} && tar -xf protoc-3.15.6-win64.zip\n",
        "    os.environ['PATH'] += os.pathsep + os.path.abspath(os.path.join(paths['PROTOC_PATH'], 'bin'))   \n",
        "    !cd Tensorflow/models/research && protoc object_detection/protos/*.proto --python_out=. && copy object_detection\\\\packages\\\\tf2\\\\setup.py setup.py && python setup.py build && python setup.py install\n",
        "    !cd Tensorflow/models/research/slim && pip install -e . "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "scrolled": true,
        "id": "pKvUFDXvp0Rh"
      },
      "outputs": [],
      "source": [
        "!pip list"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "scrolled": true,
        "id": "RKLSfOW-p0Rh"
      },
      "outputs": [],
      "source": [
        "VERIFICATION_SCRIPT = os.path.join(paths['APIMODEL_PATH'], 'research', 'object_detection', 'builders', 'model_builder_tf2_test.py')\n",
        "# Verify Installation\n",
        "!python {VERIFICATION_SCRIPT}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "x3DIJ-rXp0Ri"
      },
      "outputs": [],
      "source": [
        "import object_detection"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "csofht2npfDE"
      },
      "outputs": [],
      "source": [
        "if os.name =='posix':\n",
        "    !wget {PRETRAINED_MODEL_URL}\n",
        "    !mv {PRETRAINED_MODEL_NAME+'.tar.gz'} {paths['PRETRAINED_MODEL_PATH']}\n",
        "    !cd {paths['PRETRAINED_MODEL_PATH']} && tar -zxvf {PRETRAINED_MODEL_NAME+'.tar.gz'}\n",
        "if os.name == 'nt':\n",
        "    wget.download(PRETRAINED_MODEL_URL)\n",
        "    !move {PRETRAINED_MODEL_NAME+'.tar.gz'} {paths['PRETRAINED_MODEL_PATH']}\n",
        "    !cd {paths['PRETRAINED_MODEL_PATH']} && tar -zxvf {PRETRAINED_MODEL_NAME+'.tar.gz'}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M5KJTnkfpfDC"
      },
      "source": [
        "# 2. Create Label Map"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "p1BVDWo7pfDC"
      },
      "outputs": [],
      "source": [
        "labels = [{'name':'with_mask', 'id':1}, {'name':'without_mask', 'id':2}, {'name':'mask_weared_incorrect', 'id':3}]\n",
        "\n",
        "with open(files['LABELMAP'], 'w') as f:\n",
        "    for label in labels:\n",
        "        f.write('item { \\n')\n",
        "        f.write('\\tname:\\'{}\\'\\n'.format(label['name']))\n",
        "        f.write('\\tid:{}\\n'.format(label['id']))\n",
        "        f.write('}\\n')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C88zyVELpfDC"
      },
      "source": [
        "# 3. Create TF records"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kvf5WccwrFGq"
      },
      "outputs": [],
      "source": [
        "# OPTIONAL IF RUNNING ON COLAB\n",
        "ARCHIVE_FILES = os.path.join(paths['IMAGE_PATH'], 'archive.tar.gz')\n",
        "if os.path.exists(ARCHIVE_FILES):\n",
        "  !tar -zxvf {ARCHIVE_FILES}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LHZtIWuHp0Rj"
      },
      "outputs": [],
      "source": [
        "wget.download('https://tensorflow-object-detection-api-tutorial.readthedocs.io/en/latest/_downloads/da4babe668a8afb093cc7776d7e630f3/generate_tfrecord.py')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UPFToGZqpfDD"
      },
      "outputs": [],
      "source": [
        "!python {files['TF_RECORD_SCRIPT']} -x {os.path.join(paths['IMAGE_PATH'], 'train')} -l {files['LABELMAP']} -o {os.path.join(paths['ANNOTATION_PATH'], 'train.record')} \n",
        "!python {files['TF_RECORD_SCRIPT']} -x {os.path.join(paths['IMAGE_PATH'], 'test')} -l {files['LABELMAP']} -o {os.path.join(paths['ANNOTATION_PATH'], 'test.record')} "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qT4QU7pLpfDE"
      },
      "source": [
        "# 4. Copy Model Config to Training Folder"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cOjuTFbwpfDF"
      },
      "outputs": [],
      "source": [
        "if os.name =='posix':\n",
        "    !cp {os.path.join(paths['PRETRAINED_MODEL_PATH'], PRETRAINED_MODEL_NAME, 'pipeline.config')} {os.path.join(paths['CHECKPOINT_PATH'])}\n",
        "if os.name == 'nt':\n",
        "    !copy {os.path.join(paths['PRETRAINED_MODEL_PATH'], PRETRAINED_MODEL_NAME, 'pipeline.config')} {os.path.join(paths['CHECKPOINT_PATH'])}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ga8gpNslpfDF"
      },
      "source": [
        "# 5. Update Config For Transfer Learning"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Z9hRrO_ppfDF"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf\n",
        "from object_detection.utils import config_util\n",
        "from object_detection.protos import pipeline_pb2\n",
        "from google.protobuf import text_format"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "c2A0mn4ipfDF"
      },
      "outputs": [],
      "source": [
        "config = config_util.get_configs_from_pipeline_file(files['PIPELINE_CONFIG'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uQA13-afpfDF"
      },
      "outputs": [],
      "source": [
        "config"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9vK5lotDpfDF"
      },
      "outputs": [],
      "source": [
        "pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()\n",
        "with tf.io.gfile.GFile(files['PIPELINE_CONFIG'], \"r\") as f:                                                                                                                                                                                                                     \n",
        "    proto_str = f.read()                                                                                                                                                                                                                                          \n",
        "    text_format.Merge(proto_str, pipeline_config)  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2oCJG49pp0Rk"
      },
      "outputs": [],
      "source": [
        "print(os.path.join(paths['PRETRAINED_MODEL_PATH'], PRETRAINED_MODEL_NAME, 'checkpoint', 'ckpt-0'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rP43Ph0JpfDG"
      },
      "outputs": [],
      "source": [
        "pipeline_config.model.faster_rcnn.num_classes = len(labels)\n",
        "pipeline_config.train_config.batch_size = 4\n",
        "pipeline_config.train_config.fine_tune_checkpoint = os.path.join(paths['PRETRAINED_MODEL_PATH'], PRETRAINED_MODEL_NAME, 'checkpoint', 'ckpt-0')\n",
        "pipeline_config.train_config.fine_tune_checkpoint_type = \"detection\"\n",
        "pipeline_config.train_input_reader.label_map_path= files['LABELMAP']\n",
        "pipeline_config.train_input_reader.tf_record_input_reader.input_path[:] = [os.path.join(paths['ANNOTATION_PATH'], 'train.record')]\n",
        "pipeline_config.eval_input_reader[0].label_map_path = files['LABELMAP']\n",
        "pipeline_config.eval_input_reader[0].tf_record_input_reader.input_path[:] = [os.path.join(paths['ANNOTATION_PATH'], 'test.record')]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oJvfgwWqpfDG"
      },
      "outputs": [],
      "source": [
        "config_text = text_format.MessageToString(pipeline_config)                                                                                                                                                                                                        \n",
        "with tf.io.gfile.GFile(files['PIPELINE_CONFIG'], \"wb\") as f:                                                                                                                                                                                                                     \n",
        "    f.write(config_text)   "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zr3ON7xMpfDG"
      },
      "source": [
        "# 6. Train the model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "B-Y2UQmQpfDG"
      },
      "outputs": [],
      "source": [
        "TRAINING_SCRIPT = os.path.join(paths['APIMODEL_PATH'], 'research', 'object_detection', 'model_main_tf2.py')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jMP2XDfQpfDH"
      },
      "outputs": [],
      "source": [
        "command = \"python {} --model_dir={} --pipeline_config_path={} --num_train_steps=15000\".format(TRAINING_SCRIPT, paths['CHECKPOINT_PATH'],files['PIPELINE_CONFIG'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "A4OXXi-ApfDH"
      },
      "outputs": [],
      "source": [
        "print(command)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!{command}"
      ],
      "metadata": {
        "id": "R8SuPwvUofiw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!{command}"
      ],
      "metadata": {
        "id": "wm4MbxWSNoiB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!apt install --allow-change-held-packages libcudnn8=8.1.0.77-1+cuda11.2"
      ],
      "metadata": {
        "id": "mFHcf25HN8H_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4_YRZu7npfDH"
      },
      "source": [
        "# 7. Evaluate the Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "80L7-fdPpfDH"
      },
      "outputs": [],
      "source": [
        "command = \"python {} --model_dir={} --pipeline_config_path={} --checkpoint_dir={}\".format(TRAINING_SCRIPT, paths['CHECKPOINT_PATH'],files['PIPELINE_CONFIG'], paths['CHECKPOINT_PATH'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lYsgEPx9pfDH"
      },
      "outputs": [],
      "source": [
        "print(command)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lqTV2jGBpfDH"
      },
      "outputs": [],
      "source": [
        "!{command}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "orvRk02UpfDI"
      },
      "source": [
        "# 8. Load Train Model From Checkpoint"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8TYk4_oIpfDI"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import tensorflow as tf\n",
        "from object_detection.utils import label_map_util\n",
        "from object_detection.utils import visualization_utils as viz_utils\n",
        "from object_detection.builders import model_builder\n",
        "from object_detection.utils import config_util"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vS2umPyCp0Rm"
      },
      "outputs": [],
      "source": [
        "# Prevent GPU complete consumption\n",
        "gpus = tf.config.list_physical_devices('GPU')\n",
        "if gpus:\n",
        "    try: \n",
        "        tf.config.experimental.set_virtual_device_configuration(\n",
        "            gpus[0], [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=5120)])\n",
        "    except RunTimeError as e:\n",
        "        print(e)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tDnQg-cYpfDI"
      },
      "outputs": [],
      "source": [
        "# Load pipeline config and build a detection model\n",
        "configs = config_util.get_configs_from_pipeline_file(files['PIPELINE_CONFIG'])\n",
        "detection_model = model_builder.build(model_config=configs['model'], is_training=False)\n",
        "\n",
        "# Restore checkpoint\n",
        "ckpt = tf.compat.v2.train.Checkpoint(model=detection_model)\n",
        "ckpt.restore(os.path.join(paths['CHECKPOINT_PATH'], 'ckpt-11')).expect_partial()\n",
        "\n",
        "@tf.function\n",
        "def detect_fn(image):\n",
        "    image, shapes = detection_model.preprocess(image)\n",
        "    prediction_dict = detection_model.predict(image, shapes)\n",
        "    detections = detection_model.postprocess(prediction_dict, shapes)\n",
        "    return detections"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0EmsmbBZpfDI"
      },
      "source": [
        "# 9. Detect from an Image"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Y_MKiuZ4pfDI"
      },
      "outputs": [],
      "source": [
        "import cv2 \n",
        "import numpy as np\n",
        "from matplotlib import pyplot as plt\n",
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cBDbIhNapfDI"
      },
      "outputs": [],
      "source": [
        "category_index = label_map_util.create_category_index_from_labelmap(files['LABELMAP'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Tpzn1SMry1yK"
      },
      "outputs": [],
      "source": [
        "IMAGE_PATH = os.path.join(paths['IMAGE_PATH'], 'testimg2.jpg')\n",
        "img = cv2.imread(IMAGE_PATH)\n",
        "image_np = np.array(img)\n",
        "\n",
        "input_tensor = tf.convert_to_tensor(np.expand_dims(image_np, 0), dtype=tf.float32)\n",
        "detections = detect_fn(input_tensor)\n",
        "\n",
        "num_detections = int(detections.pop('num_detections'))\n",
        "detections = {key: value[0, :num_detections].numpy()\n",
        "              for key, value in detections.items()}\n",
        "detections['num_detections'] = num_detections\n",
        "\n",
        "# detection_classes should be ints.\n",
        "detections['detection_classes'] = detections['detection_classes'].astype(np.int64)\n",
        "\n",
        "label_id_offset = 1\n",
        "image_np_with_detections = image_np.copy()\n",
        "\n",
        "viz_utils.visualize_boxes_and_labels_on_image_array(\n",
        "            image_np_with_detections,\n",
        "            detections['detection_boxes'],\n",
        "            detections['detection_classes']+label_id_offset,\n",
        "            detections['detection_scores'],\n",
        "            category_index,\n",
        "            use_normalized_coordinates=True,\n",
        "            max_boxes_to_draw=5,\n",
        "            min_score_thresh=.8,\n",
        "            agnostic_mode=False)\n",
        "\n",
        "plt.imshow(cv2.cvtColor(image_np_with_detections, cv2.COLOR_BGR2RGB))\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ErSx-ONop0Rn"
      },
      "outputs": [],
      "source": [
        "detections.keys()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IsNAaYAo0WVL"
      },
      "source": [
        "# 10. Real Time Detections from your Webcam"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "o_grs6OGpfDJ"
      },
      "outputs": [],
      "source": [
        "cap = cv2.VideoCapture(0)\n",
        "width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n",
        "height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n",
        "\n",
        "while cap.isOpened(): \n",
        "    ret, frame = cap.read()\n",
        "    image_np = np.array(frame)\n",
        "    \n",
        "    input_tensor = tf.convert_to_tensor(np.expand_dims(image_np, 0), dtype=tf.float32)\n",
        "    detections = detect_fn(input_tensor)\n",
        "    \n",
        "    num_detections = int(detections.pop('num_detections'))\n",
        "    detections = {key: value[0, :num_detections].numpy()\n",
        "                  for key, value in detections.items()}\n",
        "    detections['num_detections'] = num_detections\n",
        "\n",
        "    # detection_classes should be ints.\n",
        "    detections['detection_classes'] = detections['detection_classes'].astype(np.int64)\n",
        "\n",
        "    label_id_offset = 1\n",
        "    image_np_with_detections = image_np.copy()\n",
        "\n",
        "    viz_utils.visualize_boxes_and_labels_on_image_array(\n",
        "                image_np_with_detections,\n",
        "                detections['detection_boxes'],\n",
        "                detections['detection_classes']+label_id_offset,\n",
        "                detections['detection_scores'],\n",
        "                category_index,\n",
        "                use_normalized_coordinates=True,\n",
        "                max_boxes_to_draw=5,\n",
        "                min_score_thresh=.8,\n",
        "                agnostic_mode=False)\n",
        "\n",
        "    cv2.imshow('object detection',  cv2.resize(image_np_with_detections, (800, 600)))\n",
        "    \n",
        "    if cv2.waitKey(10) & 0xFF == ord('q'):\n",
        "        cap.release()\n",
        "        cv2.destroyAllWindows()\n",
        "        break"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5CcT_bnUp0Rn"
      },
      "source": [
        "# Sample test run live"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "frNx-hCdp0Rn"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import cv2 \n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from matplotlib import pyplot as plt\n",
        "from google.protobuf import text_format\n",
        "from object_detection.utils import config_util\n",
        "from object_detection.protos import pipeline_pb2\n",
        "from object_detection.utils import label_map_util\n",
        "from object_detection.builders import model_builder\n",
        "from object_detection.utils import visualization_utils as viz_utils\n",
        "\n",
        "WORKSPACE_PATH = 'Tensorflow/workspace'\n",
        "SCRIPTS_PATH = 'Tensorflow/scripts'\n",
        "APIMODEL_PATH = 'Tensorflow/models'\n",
        "ANNOTATION_PATH = WORKSPACE_PATH+'/annotations'\n",
        "IMAGE_PATH = WORKSPACE_PATH+'/images'\n",
        "MODEL_PATH = WORKSPACE_PATH+'/models'\n",
        "PRETRAINED_MODEL_PATH = WORKSPACE_PATH+'/pre-trained-models'\n",
        "CONFIG_PATH = MODEL_PATH+'/my_ssd_mobnet/pipeline.config'\n",
        "CHECKPOINT_PATH = MODEL_PATH+'/my_ssd_mobnet/'\n",
        "CUSTOM_MODEL_NAME = 'my_ssd_mobnet' \n",
        "\n",
        "# Load pipeline config and build a detection model\n",
        "CONFIG_PATH = MODEL_PATH+'/'+CUSTOM_MODEL_NAME+'/pipeline.config'\n",
        "configs = config_util.get_configs_from_pipeline_file(CONFIG_PATH)\n",
        "detection_model = model_builder.build(model_config=configs['model'], is_training=False)\n",
        "\n",
        "try:\n",
        "    # Restore checkpoint\n",
        "    ckpt = tf.compat.v2.train.Checkpoint(model=detection_model)\n",
        "    ckpt.restore(os.path.join(CHECKPOINT_PATH, 'ckpt-11')).expect_partial()\n",
        "except tf.errors.NotFoundError:\n",
        "    print(\"No checkpoint found\")\n",
        "except Exception as e:\n",
        "    print(e)\n",
        "\n",
        "category_index = label_map_util.create_category_index_from_labelmap(ANNOTATION_PATH+'/label_map.pbtxt')\n",
        "\n",
        "@tf.function\n",
        "def detect_fn(image):\n",
        "    image, shapes = detection_model.preprocess(image)\n",
        "    prediction_dict = detection_model.predict(image, shapes)\n",
        "    detections = detection_model.postprocess(prediction_dict, shapes)\n",
        "    return detections\n",
        "\n",
        "# Setup capture\n",
        "cap = cv2.VideoCapture(0)\n",
        "width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n",
        "height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n",
        "while True: \n",
        "    ret, frame = cap.read()\n",
        "    image_np = np.array(frame)\n",
        "    \n",
        "    input_tensor = tf.convert_to_tensor(np.expand_dims(image_np, 0), dtype=tf.float32)\n",
        "    detections = detect_fn(input_tensor)\n",
        "    \n",
        "    num_detections = int(detections.pop('num_detections'))\n",
        "    detections = {key: value[0, :num_detections].numpy()\n",
        "                  for key, value in detections.items()}\n",
        "    detections['num_detections'] = num_detections\n",
        "\n",
        "    # detection_classes should be ints.\n",
        "    detections['detection_classes'] = detections['detection_classes'].astype(np.int64)\n",
        "\n",
        "    label_id_offset = 1\n",
        "    image_np_with_detections = image_np.copy()\n",
        "\n",
        "    viz_utils.visualize_boxes_and_labels_on_image_array(\n",
        "                image_np_with_detections,\n",
        "                detections['detection_boxes'],\n",
        "                detections['detection_classes']+label_id_offset,\n",
        "                detections['detection_scores'],\n",
        "                category_index,\n",
        "                use_normalized_coordinates=True,\n",
        "                max_boxes_to_draw=2,\n",
        "                min_score_thresh=.7,\n",
        "                agnostic_mode=False)\n",
        "\n",
        "    cv2.imshow('object detection',  cv2.resize(image_np_with_detections, (800, 600)))\n",
        "    \n",
        "    if cv2.waitKey(1) & 0xFF == ord('q'):\n",
        "        cap.release()\n",
        "        cv2.destroyAllWindows()\n",
        "        break"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rzlM4jt0pfDJ"
      },
      "source": [
        "# 10. Freezing the Graph"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "n4olHB2npfDJ"
      },
      "outputs": [],
      "source": [
        "FREEZE_SCRIPT = os.path.join(paths['APIMODEL_PATH'], 'research', 'object_detection', 'exporter_main_v2.py ')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0AjO93QDpfDJ"
      },
      "outputs": [],
      "source": [
        "command = \"python {} --input_type=image_tensor --pipeline_config_path={} --trained_checkpoint_dir={} --output_directory={}\".format(FREEZE_SCRIPT ,files['PIPELINE_CONFIG'], paths['CHECKPOINT_PATH'], paths['OUTPUT_PATH'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "F6Lsp3tCpfDJ"
      },
      "outputs": [],
      "source": [
        "print(command)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1Sw1ULgHpfDJ"
      },
      "outputs": [],
      "source": [
        "!{command}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wTPmdqaXpfDK"
      },
      "source": [
        "# 11. Conversion to TFJS"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gZ6UzY_fpfDK",
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "!pip install tensorflowjs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0oxbVynHpfDK"
      },
      "outputs": [],
      "source": [
        "command = \"tensorflowjs_converter --input_format=tf_saved_model --output_node_names='detection_boxes,detection_classes,detection_features,detection_multiclass_scores,detection_scores,num_detections,raw_detection_boxes,raw_detection_scores' --output_format=tfjs_graph_model --signature_name=serving_default {} {}\".format(os.path.join(paths['OUTPUT_PATH'], 'saved_model'), paths['TFJS_PATH'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DB2AGNmJpfDK"
      },
      "outputs": [],
      "source": [
        "print(command)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "K7rfT4-hpfDK"
      },
      "outputs": [],
      "source": [
        "!{command}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "o8_hm-itpfDK"
      },
      "outputs": [],
      "source": [
        "# Test Code: https://github.com/nicknochnack/RealTimeSignLanguageDetectionwithTFJS"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VtUw73FHpfDK"
      },
      "source": [
        "# 12. Conversion to TFLite"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XviMtewLpfDK"
      },
      "outputs": [],
      "source": [
        "TFLITE_SCRIPT = os.path.join(paths['APIMODEL_PATH'], 'research', 'object_detection', 'export_tflite_graph_tf2.py ')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "us86cjC4pfDL"
      },
      "outputs": [],
      "source": [
        "command = \"python {} --pipeline_config_path={} --trained_checkpoint_dir={} --output_directory={}\".format(TFLITE_SCRIPT ,files['PIPELINE_CONFIG'], paths['CHECKPOINT_PATH'], paths['TFLITE_PATH'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "n1r5YO3rpfDL"
      },
      "outputs": [],
      "source": [
        "print(command)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "I-xWpHN8pfDL"
      },
      "outputs": [],
      "source": [
        "!{command}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iJfYMbN6pfDL"
      },
      "outputs": [],
      "source": [
        "FROZEN_TFLITE_PATH = os.path.join(paths['TFLITE_PATH'], 'saved_model')\n",
        "TFLITE_MODEL = os.path.join(paths['TFLITE_PATH'], 'saved_model', 'detect.tflite')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7DGgkFBdp0Rp"
      },
      "outputs": [],
      "source": [
        "command = \"tflite_convert \\\n",
        "--saved_model_dir={} \\\n",
        "--output_file={} \\\n",
        "--input_shapes=1,300,300,3 \\\n",
        "--input_arrays=normalized_input_image_tensor \\\n",
        "--output_arrays='TFLite_Detection_PostProcess','TFLite_Detection_PostProcess:1','TFLite_Detection_PostProcess:2','TFLite_Detection_PostProcess:3' \\\n",
        "--inference_type=FLOAT \\\n",
        "--allow_custom_ops\".format(FROZEN_TFLITE_PATH, TFLITE_MODEL, )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "E8GwUeoFpfDL"
      },
      "outputs": [],
      "source": [
        "print(command)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Nbd7gqHMpfDL"
      },
      "outputs": [],
      "source": [
        "!{command}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5NQqZRdA21Uc"
      },
      "source": [
        "# 13. Zip and Export Models "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tTVTGCQp2ZJJ"
      },
      "outputs": [],
      "source": [
        "!tar -czf models.tar.gz {paths['CHECKPOINT_PATH']}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "whShhB0x3PYJ"
      },
      "outputs": [],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!unzip face-mask-detection.zip"
      ],
      "metadata": {
        "id": "rNwiQNuPr4hW"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "ObjectDetectionTransferLearning.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "4_YRZu7npfDH",
        "orvRk02UpfDI",
        "0EmsmbBZpfDI",
        "IsNAaYAo0WVL",
        "5CcT_bnUp0Rn",
        "rzlM4jt0pfDJ",
        "wTPmdqaXpfDK",
        "VtUw73FHpfDK"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3.10.5 ('.venv': venv)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.5"
    },
    "vscode": {
      "interpreter": {
        "hash": "66e94f7b7489250cb157aed0c9bbcac446a8abe6882d1b25d8e0e210eea02991"
      }
    },
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}