Using LSTM and MediaPipe.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Importing Librabries\n",
    "\n",
    "\n",
    "import cv2 \n",
    "import numpy as np\n",
    "import os \n",
    "import matplotlib.pyplot as plt\n",
    "import time \n",
    "import mediapipe as mp\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "mp_holistic=  mp.solutions.holistic\n",
    "mp_drawing = mp.solutions.drawing_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mediapipe_detection(image, model):\n",
    "    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n",
    "    image.flags.writeable =False \n",
    "    results = model.process(image)\n",
    "    image.flags.writeable =True\n",
    "    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n",
    "    return image, results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "##rendering Landmarks \n",
    "def draw_landmarks(image, results):\n",
    "    mp_drawing.draw_landmarks(image, results.face_landmarks, mp_holistic.FACEMESH_TESSELATION)\n",
    "    mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS)\n",
    "    mp_drawing.draw_landmarks(image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS)\n",
    "    mp_drawing.draw_landmarks(image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_landmarks2(image,result):\n",
    "    mp_drawing.draw_landmarks(image, results.face_landmarks, mp_holistic.FACEMESH_TESSELATION,\n",
    "                              mp_drawing.DrawingSpec(color=(80,110,10),thickness=1,circle_radius =1 ),\n",
    "                              mp_drawing.DrawingSpec(color=(80,110,10),thickness=1,circle_radius =1 )\n",
    "                              \n",
    "                             \n",
    "                             )\n",
    "    mp_drawing.draw_landmarks(image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS,\n",
    "                              mp_drawing.DrawingSpec(color=(80,22,10),thickness=2,circle_radius =1 ),\n",
    "                              mp_drawing.DrawingSpec(color=(80,44,10),thickness=2,circle_radius =1 )\n",
    "                              )\n",
    "    mp_drawing.draw_landmarks(image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS,\n",
    "                              mp_drawing.DrawingSpec(color=(80,22,10),thickness=2,circle_radius =1 ),\n",
    "                              mp_drawing.DrawingSpec(color=(80,44,10),thickness=2,circle_radius =1 )\n",
    "                              )\n",
    "    mp_drawing.draw_landmarks(image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS,\n",
    "                              mp_drawing.DrawingSpec(color=(80,117,10),thickness=2,circle_radius =1 ),\n",
    "                              mp_drawing.DrawingSpec(color=(80,66,10),thickness=2,circle_radius =1 )\n",
    "                              )\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'results' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-6-e2dd042c616c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     15\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mpose\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mface\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlh\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrh\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 17\u001b[1;33m \u001b[0mtest\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mextract_keypoints\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresults\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     18\u001b[0m \u001b[0mtest\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'results' is not defined"
     ]
    }
   ],
   "source": [
    "\n",
    "##Extracting Landmarks \n",
    "\"\"\"pose =[]\n",
    "for res in results.pose_landmarks.landmark:\n",
    "    test = np.array([res.x,res.y,res.z,res.visibility])\n",
    "    pose.append(test)\"\"\"\n",
    "\n",
    "def extract_keypoints(results):\n",
    "    pose = np.array([[res.x, res.y, res.z, res.visibility] for res in results.pose_landmarks.landmark]).flatten() if results.pose_landmarks else np.zeros(33*4)\n",
    "    \n",
    "    face = np.array([[res.x, res.y, res.z] for res in results.face_landmarks.landmark]).flatten() if results.face_landmarks else np.zeros(468*3)\n",
    "    \n",
    "    lh = np.array([[res.x, res.y, res.z] for res in results.left_hand_landmarks.landmark]).flatten() if results.left_hand_landmarks else np.zeros(21*3)\n",
    "    \n",
    "    rh = np.array([[res.x, res.y, res.z] for res in results.right_hand_landmarks.landmark]).flatten() if results.right_hand_landmarks else np.zeros(21*3)\n",
    "    return np.concatenate([pose, face, lh, rh])\n",
    "\n",
    "test = extract_keypoints(results)\n",
    "test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "##Tracking Face and Hands\n",
    "i = 0\n",
    "cap = cv2.VideoCapture(0)\n",
    "with mp_holistic.Holistic(min_detection_confidence=0.5,min_tracking_confidence=0.5) as holistic:\n",
    "    while cap.isOpened():\n",
    "        ret, frame = cap.read()\n",
    "        #Using Mediapipe\n",
    "\n",
    "        image, results = mediapipe_detection(frame, holistic)   \n",
    "        \n",
    "        #Drawing Ladmarks\n",
    "        draw_landmarks2(image,results)\n",
    "        \n",
    "        \n",
    "        cv2.imshow(\"frame\" , image)\n",
    "       # cv2.imwrite(\"image.jpg\", image)\n",
    "        if cv2.waitKey(1)==ord('q'):\n",
    "            break\n",
    "    cap.release()\n",
    "    cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = os.path.join(\"mpn_data/\")\n",
    "actions= np.array(['Thankyou', \"Hello\",\"To_save\",\"Help\",\"Good\",\"Hope\",\"You\"])#\"A\",\"B\"])#\"C\",\"G\"])\n",
    "#actions= np.array([\"D\"])\n",
    "\n",
    "\n",
    "no_sequences = 60 #30 Videos for each actions\n",
    "sequence_length = 30 ##30 Frames\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Making Folders \n",
    "\n",
    "for action in actions:\n",
    "    for s in range(no_sequences):\n",
    "        try:\n",
    "            os.makedirs(os.path.join(data_path, action, str(s)))\n",
    "        except:\n",
    "            pass\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "##collecting images\n",
    "\n",
    "cap = cv2.VideoCapture(0)\n",
    "with mp_holistic.Holistic(min_detection_confidence=0.5,min_tracking_confidence=0.5) as holistic:\n",
    "    for action in actions:\n",
    "        for s in range(no_sequences):\n",
    "            for i in range(sequence_length):\n",
    "                ret, frame = cap.read()\n",
    "                #Using Mediapipe\n",
    "\n",
    "                image, results = mediapipe_detection(frame, holistic)   \n",
    "\n",
    "                #Drawing Ladmarks\n",
    "                draw_landmarks2(image,results)\n",
    "\n",
    "                if i==0:\n",
    "                    cv2.putText(image, \"Starting\", (120,200), cv2.FONT_HERSHEY_SIMPLEX,1,(0,255,0),4,cv2.LINE_AA)\n",
    "                    cv2.putText(image, \"Collecting Fraes for  {} Video Number {}\".format(action,s ), (15,12), cv2.FONT_HERSHEY_SIMPLEX,0.5,(0,0,255),1,cv2.LINE_AA)\n",
    "                    cv2.waitKey(1000)\n",
    "                else:\n",
    "                    cv2.putText(image, \"Collecting Frames for  {} Video Number {}\".format(action,s), (15,12), cv2.FONT_HERSHEY_SIMPLEX,0.5,(0,0,255),1,cv2.LINE_AA)\n",
    "\n",
    "                keypoints = extract_keypoints(results)\n",
    "                npy_path = os.path.join(data_path, action,str(s),str(i))\n",
    "                np.save(npy_path,keypoints)\n",
    "                cv2.imshow(\"frame\" , image)\n",
    "\n",
    "                # cv2.imwrite(\"image.jpg\", image)\n",
    "                if cv2.waitKey(1)==ord('q'):\n",
    "                    break\n",
    "    cap.release()\n",
    "    cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Thankyou': 0,\n",
       " 'Hello': 1,\n",
       " 'To_save': 2,\n",
       " 'Help': 3,\n",
       " 'Good': 4,\n",
       " 'Hope': 5,\n",
       " 'You': 6}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "\n",
    "label_map = {label:num for num,label in enumerate(actions)}\n",
    "\n",
    "label_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(420, 30, 1662)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sequences, labels = [],[]\n",
    "for action in actions:\n",
    "    for seq in range(no_sequences):\n",
    "        window=[]\n",
    "        for frame_num in range(sequence_length):\n",
    "            res = np.load(os.path.join(data_path,action , str(seq),\"{}.npy\".format(frame_num)))\n",
    "            window.append(res)\n",
    "        sequences.append(window)\n",
    "        labels.append(label_map[action])\n",
    "        \n",
    "np.array(sequences).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(420, 30, 1662)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = to_categorical(labels).astype(int)\n",
    "X = np.array(sequences)\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "##SPlitting Data\n",
    "\n",
    "X_train , X_test, y_train, y_test = train_test_split(X,y,test_size = 0.05)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import LSTM, Dense\n",
    "from tensorflow.keras.callbacks import TensorBoard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_dir = os.path.join(\"logs\")\n",
    "tb_callback = TensorBoard(log_dir=log_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 399 samples\n",
      "Epoch 1/150\n",
      "399/399 [==============================] - 4s 11ms/sample - loss: 2.4374 - categorical_accuracy: 0.1454\n",
      "Epoch 2/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.9664 - categorical_accuracy: 0.1479\n",
      "Epoch 3/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.9496 - categorical_accuracy: 0.1679\n",
      "Epoch 4/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.9201 - categorical_accuracy: 0.1980\n",
      "Epoch 5/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.8425 - categorical_accuracy: 0.2256\n",
      "Epoch 6/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.6808 - categorical_accuracy: 0.2732\n",
      "Epoch 7/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.6735 - categorical_accuracy: 0.24310s - loss: 1.6782 - categorical_accuracy: 0.24\n",
      "Epoch 8/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.4355 - categorical_accuracy: 0.3609\n",
      "Epoch 9/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.2748 - categorical_accuracy: 0.3609\n",
      "Epoch 10/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.1216 - categorical_accuracy: 0.5038\n",
      "Epoch 11/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.2443 - categorical_accuracy: 0.46120s - loss: 1.2948 - categorical_accu\n",
      "Epoch 12/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.9775 - categorical_accuracy: 0.5965\n",
      "Epoch 13/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.0424 - categorical_accuracy: 0.5539\n",
      "Epoch 14/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.0319 - categorical_accuracy: 0.5589\n",
      "Epoch 15/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.9706 - categorical_accuracy: 0.60150s - loss: 1.0284 - categorical_accura\n",
      "Epoch 16/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.7757 - categorical_accuracy: 0.6692\n",
      "Epoch 17/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.6999 - categorical_accuracy: 0.7018\n",
      "Epoch 18/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.7266 - categorical_accuracy: 0.7093\n",
      "Epoch 19/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.6456 - categorical_accuracy: 0.7368\n",
      "Epoch 20/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.5962 - categorical_accuracy: 0.7343\n",
      "Epoch 21/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.9246 - categorical_accuracy: 0.6591\n",
      "Epoch 22/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.1241 - categorical_accuracy: 0.5414\n",
      "Epoch 23/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.8724 - categorical_accuracy: 0.6942\n",
      "Epoch 24/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.6397 - categorical_accuracy: 0.7544\n",
      "Epoch 25/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.7948 - categorical_accuracy: 0.6842\n",
      "Epoch 26/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.5500 - categorical_accuracy: 0.8020\n",
      "Epoch 27/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.4530 - categorical_accuracy: 0.8120\n",
      "Epoch 28/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.5029 - categorical_accuracy: 0.7920\n",
      "Epoch 29/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.6259 - categorical_accuracy: 0.7619\n",
      "Epoch 30/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.4842 - categorical_accuracy: 0.8321\n",
      "Epoch 31/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.4331 - categorical_accuracy: 0.8346\n",
      "Epoch 32/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.3778 - categorical_accuracy: 0.8521\n",
      "Epoch 33/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.3750 - categorical_accuracy: 0.8622\n",
      "Epoch 34/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.4351 - categorical_accuracy: 0.8296\n",
      "Epoch 35/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.5414 - categorical_accuracy: 0.7744\n",
      "Epoch 36/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 3.3972 - categorical_accuracy: 0.3083\n",
      "Epoch 37/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 2.3324 - categorical_accuracy: 0.1253\n",
      "Epoch 38/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.9576 - categorical_accuracy: 0.1404\n",
      "Epoch 39/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.9440 - categorical_accuracy: 0.1554\n",
      "Epoch 40/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.9207 - categorical_accuracy: 0.1629\n",
      "Epoch 41/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.8903 - categorical_accuracy: 0.2080\n",
      "Epoch 42/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.7723 - categorical_accuracy: 0.3008\n",
      "Epoch 43/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.6696 - categorical_accuracy: 0.3108\n",
      "Epoch 44/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.8156 - categorical_accuracy: 0.2356\n",
      "Epoch 45/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.6456 - categorical_accuracy: 0.3058\n",
      "Epoch 46/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.6385 - categorical_accuracy: 0.3058\n",
      "Epoch 47/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.5732 - categorical_accuracy: 0.3383\n",
      "Epoch 48/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.9084 - categorical_accuracy: 0.2030\n",
      "Epoch 49/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.7158 - categorical_accuracy: 0.3033\n",
      "Epoch 50/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.4212 - categorical_accuracy: 0.3835\n",
      "Epoch 51/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.2267 - categorical_accuracy: 0.4386\n",
      "Epoch 52/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.2722 - categorical_accuracy: 0.4787\n",
      "Epoch 53/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.1136 - categorical_accuracy: 0.5388\n",
      "Epoch 54/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.0465 - categorical_accuracy: 0.5313\n",
      "Epoch 55/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.3072 - categorical_accuracy: 0.4035\n",
      "Epoch 56/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.1479 - categorical_accuracy: 0.4361\n",
      "Epoch 57/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.0727 - categorical_accuracy: 0.5163\n",
      "Epoch 58/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.3224 - categorical_accuracy: 0.4386\n",
      "Epoch 59/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.0193 - categorical_accuracy: 0.55891s - loss: 1.0787 - categorica\n",
      "Epoch 60/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.9199 - categorical_accuracy: 0.6441\n",
      "Epoch 61/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.9028 - categorical_accuracy: 0.6040\n",
      "Epoch 62/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.9920 - categorical_accuracy: 0.5439\n",
      "Epoch 63/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.9374 - categorical_accuracy: 0.5539\n",
      "Epoch 64/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.8342 - categorical_accuracy: 0.6491\n",
      "Epoch 65/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.7654 - categorical_accuracy: 0.6642\n",
      "Epoch 66/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.6912 - categorical_accuracy: 0.6842\n",
      "Epoch 67/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.8258 - categorical_accuracy: 0.6291\n",
      "Epoch 68/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.8656 - categorical_accuracy: 0.6341\n",
      "Epoch 69/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.7509 - categorical_accuracy: 0.6566\n",
      "Epoch 70/150\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.6436 - categorical_accuracy: 0.7218\n",
      "Epoch 71/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.5357 - categorical_accuracy: 0.7669\n",
      "Epoch 72/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.8344 - categorical_accuracy: 0.6441\n",
      "Epoch 73/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.6861 - categorical_accuracy: 0.6892\n",
      "Epoch 74/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.5887 - categorical_accuracy: 0.7644\n",
      "Epoch 75/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.4986 - categorical_accuracy: 0.7719\n",
      "Epoch 76/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.4838 - categorical_accuracy: 0.7970\n",
      "Epoch 77/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.6249 - categorical_accuracy: 0.7343\n",
      "Epoch 78/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.5038 - categorical_accuracy: 0.7870\n",
      "Epoch 79/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.4294 - categorical_accuracy: 0.8371\n",
      "Epoch 80/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.4349 - categorical_accuracy: 0.8070\n",
      "Epoch 81/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.3505 - categorical_accuracy: 0.8722\n",
      "Epoch 82/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.3301 - categorical_accuracy: 0.8697\n",
      "Epoch 83/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.3797 - categorical_accuracy: 0.8496\n",
      "Epoch 84/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.3575 - categorical_accuracy: 0.8722\n",
      "Epoch 85/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.2567 - categorical_accuracy: 0.9073\n",
      "Epoch 86/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.3385 - categorical_accuracy: 0.8922\n",
      "Epoch 87/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 1.3807 - categorical_accuracy: 0.6190\n",
      "Epoch 88/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.7868 - categorical_accuracy: 0.6617\n",
      "Epoch 89/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.5542 - categorical_accuracy: 0.8346\n",
      "Epoch 90/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.4021 - categorical_accuracy: 0.8722\n",
      "Epoch 91/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.3272 - categorical_accuracy: 0.8797\n",
      "Epoch 92/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.2759 - categorical_accuracy: 0.9148\n",
      "Epoch 93/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.3114 - categorical_accuracy: 0.8797\n",
      "Epoch 94/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.2758 - categorical_accuracy: 0.9048\n",
      "Epoch 95/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.3837 - categorical_accuracy: 0.8421\n",
      "Epoch 96/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.3689 - categorical_accuracy: 0.8647\n",
      "Epoch 97/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.5449 - categorical_accuracy: 0.7794\n",
      "Epoch 98/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.3182 - categorical_accuracy: 0.8822\n",
      "Epoch 99/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.2472 - categorical_accuracy: 0.9073\n",
      "Epoch 100/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.1843 - categorical_accuracy: 0.9449\n",
      "Epoch 101/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.1711 - categorical_accuracy: 0.9449\n",
      "Epoch 102/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.1651 - categorical_accuracy: 0.9474\n",
      "Epoch 103/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.9808 - categorical_accuracy: 0.6817\n",
      "Epoch 104/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.7120 - categorical_accuracy: 0.7419\n",
      "Epoch 105/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.4950 - categorical_accuracy: 0.8120\n",
      "Epoch 106/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.3547 - categorical_accuracy: 0.8897\n",
      "Epoch 107/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.2566 - categorical_accuracy: 0.9273\n",
      "Epoch 108/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.1926 - categorical_accuracy: 0.9348\n",
      "Epoch 109/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.1615 - categorical_accuracy: 0.95241s - loss: 0.1487 - categorical_ac\n",
      "Epoch 110/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.1516 - categorical_accuracy: 0.9524\n",
      "Epoch 111/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.1246 - categorical_accuracy: 0.9599\n",
      "Epoch 112/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.1056 - categorical_accuracy: 0.9674\n",
      "Epoch 113/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.1050 - categorical_accuracy: 0.9674\n",
      "Epoch 114/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.1403 - categorical_accuracy: 0.9474\n",
      "Epoch 115/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.1288 - categorical_accuracy: 0.9574\n",
      "Epoch 116/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.1138 - categorical_accuracy: 0.9649\n",
      "Epoch 117/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.0686 - categorical_accuracy: 0.9774\n",
      "Epoch 118/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.0811 - categorical_accuracy: 0.9749\n",
      "Epoch 119/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.1242 - categorical_accuracy: 0.9574\n",
      "Epoch 120/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.0992 - categorical_accuracy: 0.9724\n",
      "Epoch 121/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.0689 - categorical_accuracy: 0.9774\n",
      "Epoch 122/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.0557 - categorical_accuracy: 0.98751s - loss: 0.0502 - categorica\n",
      "Epoch 123/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.3348 - categorical_accuracy: 0.9023\n",
      "Epoch 124/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.5499 - categorical_accuracy: 0.8195\n",
      "Epoch 125/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.2791 - categorical_accuracy: 0.9148\n",
      "Epoch 126/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.2323 - categorical_accuracy: 0.9449\n",
      "Epoch 127/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.2454 - categorical_accuracy: 0.9098\n",
      "Epoch 128/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.2077 - categorical_accuracy: 0.9273\n",
      "Epoch 129/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.1184 - categorical_accuracy: 0.9574\n",
      "Epoch 130/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.0907 - categorical_accuracy: 0.9774\n",
      "Epoch 131/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.0686 - categorical_accuracy: 0.9875\n",
      "Epoch 132/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.0524 - categorical_accuracy: 0.9875\n",
      "Epoch 133/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.0799 - categorical_accuracy: 0.9724\n",
      "Epoch 134/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.1085 - categorical_accuracy: 0.9624\n",
      "Epoch 135/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.0829 - categorical_accuracy: 0.9674\n",
      "Epoch 136/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.1039 - categorical_accuracy: 0.9699\n",
      "Epoch 137/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.0614 - categorical_accuracy: 0.9799\n",
      "Epoch 138/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.0612 - categorical_accuracy: 0.9825\n",
      "Epoch 139/150\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.0424 - categorical_accuracy: 0.9850\n",
      "Epoch 140/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.0320 - categorical_accuracy: 0.9950\n",
      "Epoch 141/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.0266 - categorical_accuracy: 0.9925\n",
      "Epoch 142/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.0298 - categorical_accuracy: 0.9850\n",
      "Epoch 143/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.0530 - categorical_accuracy: 0.97741s - loss: 0.0403 - categorical_ac\n",
      "Epoch 144/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.1164 - categorical_accuracy: 0.9499\n",
      "Epoch 145/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.1830 - categorical_accuracy: 0.9474\n",
      "Epoch 146/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.1031 - categorical_accuracy: 0.9599\n",
      "Epoch 147/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.1833 - categorical_accuracy: 0.9323\n",
      "Epoch 148/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.1137 - categorical_accuracy: 0.9549\n",
      "Epoch 149/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.0588 - categorical_accuracy: 0.9774\n",
      "Epoch 150/150\n",
      "399/399 [==============================] - 2s 6ms/sample - loss: 0.0417 - categorical_accuracy: 0.9900\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x2a66a303388>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(LSTM(64, return_sequences=True, activation='relu', input_shape=(30,1662)))\n",
    "model.add(LSTM(128, return_sequences=True, activation='relu'))\n",
    "model.add(LSTM(64, return_sequences=False, activation='relu'))\n",
    "model.add(Dense(64, activation='relu'))\n",
    "model.add(Dense(32, activation='relu'))\n",
    "model.add(Dense(actions.shape[0], activation='softmax'))\n",
    "\n",
    "model.compile(optimizer= \"adam\", loss = \"categorical_crossentropy\",metrics=['categorical_accuracy'])\n",
    "model.fit(X_train, y_train, epochs = 150,callbacks= [tb_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[2.08563972e-04 5.92446577e-06 1.67406013e-03 5.45379237e-07\n",
      "  1.69252267e-03 6.28904527e-05 9.90751266e-01 2.21399910e-06\n",
      "  1.32138899e-03 2.68479506e-03 1.29575748e-03 2.99972337e-04]\n",
      " [2.72457284e-30 0.00000000e+00 1.74872153e-20 4.18375974e-36\n",
      "  9.69439548e-25 4.19537983e-28 3.21127190e-24 0.00000000e+00\n",
      "  4.79875267e-29 6.92112645e-10 4.08500707e-12 1.00000000e+00]\n",
      " [1.28851781e-04 2.03366473e-01 3.25196562e-03 3.46894958e-05\n",
      "  8.64347618e-04 4.32914276e-05 7.59682208e-02 2.91156888e-01\n",
      "  3.27217460e-01 9.01215374e-02 4.59866365e-03 3.24762193e-03]\n",
      " [1.38595671e-04 2.00853303e-01 1.24173649e-02 8.63568202e-05\n",
      "  1.56901814e-02 9.10296294e-05 3.50042023e-02 4.81850617e-02\n",
      "  2.71753758e-01 3.44852805e-01 6.00810014e-02 1.08463401e-02]\n",
      " [2.39869922e-01 1.37397992e-05 2.28760272e-01 2.78921336e-01\n",
      "  9.44785625e-02 1.02557480e-01 1.71606299e-02 9.03174150e-05\n",
      "  6.19769562e-05 5.79750980e-04 2.39098649e-02 1.35961166e-02]\n",
      " [4.62669939e-01 6.18245255e-12 2.12482666e-03 2.92237801e-05\n",
      "  5.89692263e-07 5.30180573e-01 4.45150724e-03 1.52820291e-11\n",
      "  1.98543777e-08 8.20340858e-07 2.12487517e-04 3.29969131e-04]\n",
      " [1.33388123e-04 4.47139087e-07 3.11441399e-04 1.38330165e-07\n",
      "  1.45802469e-05 6.11799478e-05 9.98729050e-01 4.01601454e-07\n",
      "  2.45446456e-04 1.16906864e-04 3.16331658e-04 7.07660001e-05]\n",
      " [3.89946690e-05 1.22493155e-01 3.94552859e-04 1.97232112e-06\n",
      "  7.60718603e-06 2.93945618e-06 1.09535875e-02 5.87397635e-01\n",
      "  2.31867850e-01 4.18253243e-02 1.51480269e-03 3.50150722e-03]\n",
      " [1.39427111e-05 1.02907248e-01 5.79746411e-05 2.04889446e-07\n",
      "  2.76332258e-07 2.56343242e-07 2.05419213e-03 7.37846017e-01\n",
      "  1.32102042e-01 2.15368457e-02 7.72428582e-04 2.70860037e-03]\n",
      " [6.41056104e-04 1.12584178e-10 3.85009997e-07 6.40491780e-04\n",
      "  1.60674723e-13 8.74146053e-08 1.01148507e-05 8.46730352e-10\n",
      "  2.16133686e-12 2.51117086e-07 3.25552683e-04 9.98382092e-01]\n",
      " [1.82422906e-01 1.77204608e-15 5.41943009e-04 4.08329743e-06\n",
      "  6.39730624e-09 8.16919804e-01 7.80551636e-05 7.59851573e-15\n",
      "  5.10012900e-11 2.09220179e-08 7.98903784e-06 2.50817629e-05]\n",
      " [1.42828430e-04 2.33721644e-01 6.28904672e-03 5.87721661e-05\n",
      "  3.35445581e-03 6.84017868e-05 7.79677778e-02 1.57529235e-01\n",
      "  3.50629240e-01 1.54868066e-01 1.09166671e-02 4.45386814e-03]\n",
      " [3.49156799e-05 1.42473206e-01 5.37743501e-04 2.59784156e-06\n",
      "  1.68210245e-05 4.13834232e-06 1.44772083e-02 5.18319726e-01\n",
      "  2.72497237e-01 4.70360480e-02 1.70805724e-03 2.89227045e-03]\n",
      " [4.00986464e-04 3.58537154e-14 1.04750274e-02 9.55283284e-01\n",
      "  2.69240011e-02 1.00808851e-04 3.04348816e-07 1.99281563e-12\n",
      "  3.80696966e-13 1.80276506e-11 6.81451475e-03 1.17346281e-06]\n",
      " [1.34374795e-05 6.97239651e-04 4.96301893e-03 2.01920229e-05\n",
      "  4.13009264e-02 4.08779306e-06 4.07779240e-04 2.35029274e-05\n",
      "  3.05064546e-04 3.95461731e-02 8.90175104e-01 2.25434601e-02]\n",
      " [4.32388347e-10 8.70957213e-12 2.56039319e-15 9.70068413e-14\n",
      "  2.13301406e-30 2.51600360e-20 3.75682151e-13 1.77346676e-13\n",
      "  4.01836215e-16 2.47891019e-09 9.39516485e-06 9.99990582e-01]\n",
      " [4.96856344e-04 3.77691500e-02 7.18672118e-06 2.18325283e-07\n",
      "  1.20944893e-10 2.64808886e-08 1.87411148e-03 2.23862618e-01\n",
      "  7.16912281e-03 1.28184542e-01 2.26874906e-03 5.98367453e-01]\n",
      " [5.61238267e-04 5.11820472e-05 2.22646985e-02 7.48197817e-06\n",
      "  2.03325748e-01 1.90944702e-04 7.24987745e-01 3.15158104e-05\n",
      "  5.74445166e-03 3.79367620e-02 3.36978305e-03 1.52842677e-03]\n",
      " [3.11612844e-01 3.41722017e-15 6.47482932e-01 5.80464897e-04\n",
      "  6.38169004e-05 4.01761644e-02 3.56000442e-06 5.35851847e-14\n",
      "  3.17970498e-12 2.77988355e-10 7.87625540e-05 1.48414665e-06]\n",
      " [2.76507599e-05 3.37626040e-01 1.24201644e-03 5.33540515e-06\n",
      "  3.07897863e-04 9.74506293e-06 1.37699004e-02 1.59214139e-01\n",
      "  3.51747155e-01 1.27935246e-01 5.93218161e-03 2.18260358e-03]\n",
      " [5.52916902e-07 8.06193283e-19 1.02896149e-04 9.99688268e-01\n",
      "  4.77398899e-05 1.97937675e-06 7.27765417e-12 4.05927636e-17\n",
      "  6.01934265e-19 1.27240728e-15 1.58584982e-04 8.83265461e-09]\n",
      " [5.55994511e-01 1.76169797e-05 1.38268739e-01 5.70571683e-02\n",
      "  1.66094154e-02 1.56017229e-01 5.23193777e-02 1.23605001e-04\n",
      "  1.35256007e-04 1.36505789e-03 5.21477265e-03 1.68772247e-02]\n",
      " [5.85730309e-09 1.04126846e-10 1.51455541e-12 1.61894838e-11\n",
      "  1.05904214e-26 4.26689340e-17 8.28930761e-12 3.59012754e-12\n",
      "  1.60081112e-14 1.47809462e-08 4.06868930e-05 9.99959350e-01]\n",
      " [1.34391841e-04 1.57011747e-02 4.82162163e-02 2.17839930e-04\n",
      "  2.19151825e-01 1.11229281e-04 1.91605929e-02 2.18615308e-03\n",
      "  1.82998292e-02 2.19175652e-01 4.24679607e-01 3.29655185e-02]\n",
      " [1.23024233e-01 7.69388613e-16 3.89330438e-04 3.36734229e-06\n",
      "  2.68960565e-09 8.76510084e-01 4.89463673e-05 3.57713110e-15\n",
      "  2.60976293e-11 1.19687975e-08 4.84253042e-06 1.93206233e-05]\n",
      " [1.26003614e-03 5.88743128e-02 3.00729356e-04 6.94860637e-06\n",
      "  1.61481054e-07 2.87790022e-06 1.29998196e-02 5.92904270e-01\n",
      "  5.24486080e-02 1.08503498e-01 3.95302242e-03 1.68745726e-01]\n",
      " [3.32844049e-01 3.44561992e-14 5.40788949e-01 5.97955426e-04\n",
      "  2.05275355e-05 1.25554934e-01 7.37155597e-06 2.04817920e-13\n",
      "  2.65154496e-11 2.06985384e-09 1.77475013e-04 8.82695713e-06]\n",
      " [1.46326478e-04 1.31865681e-06 8.03419971e-04 2.05288060e-07\n",
      "  1.20527991e-04 7.49458050e-05 9.96146679e-01 7.11452913e-07\n",
      "  7.86567340e-04 6.66422304e-04 1.04711344e-03 2.05739343e-04]\n",
      " [9.26082488e-03 1.81149318e-08 2.66190935e-02 6.76838636e-01\n",
      "  2.60318935e-01 1.68072432e-03 3.85292369e-04 3.79595576e-07\n",
      "  7.57399192e-08 9.73969122e-07 2.46040858e-02 2.91007484e-04]\n",
      " [9.42283943e-02 2.41563917e-04 2.68256068e-01 2.20553368e-01\n",
      "  1.46781951e-01 6.05177097e-02 1.07849181e-01 8.44898401e-04\n",
      "  5.14309679e-04 8.34820140e-03 3.90901528e-02 5.27742095e-02]\n",
      " [1.18418075e-05 3.48944157e-01 4.46116290e-04 1.64489165e-06\n",
      "  4.58190952e-05 3.27013436e-06 7.65089272e-03 2.41913632e-01\n",
      "  3.23011518e-01 7.40658194e-02 2.66759121e-03 1.23766845e-03]\n",
      " [2.02934951e-01 5.27698858e-15 3.58099341e-02 4.42596996e-04\n",
      "  4.29867328e-07 7.60776460e-01 7.14877842e-06 4.36371283e-14\n",
      "  8.67923008e-12 1.50837920e-09 1.78086975e-05 1.07048263e-05]\n",
      " [3.11727927e-05 1.17476240e-01 2.29329671e-04 1.06192715e-06\n",
      "  2.93554967e-06 1.54619590e-06 7.38925114e-03 6.35688543e-01\n",
      "  2.01231763e-01 3.36212479e-02 1.13963638e-03 3.18735279e-03]\n",
      " [6.94747068e-05 8.25402532e-16 4.12860699e-03 9.95386302e-01\n",
      "  4.68231628e-05 2.82972935e-04 1.44886797e-10 5.55904547e-15\n",
      "  5.26869952e-16 1.15436117e-13 8.58086278e-05 9.67060956e-08]\n",
      " [2.38464046e-02 2.72137782e-04 1.00499559e-02 8.00465292e-04\n",
      "  6.31271442e-03 4.88512637e-03 9.39503431e-01 5.37641463e-04\n",
      "  2.83576106e-03 5.08163963e-03 2.18990352e-03 3.68487160e-03]\n",
      " [1.69269151e-05 2.77588397e-01 5.65615424e-04 2.38755229e-06\n",
      "  4.94906017e-05 4.05231413e-06 1.15499711e-02 3.18180174e-01\n",
      "  3.24619383e-01 6.34859949e-02 2.44479091e-03 1.49282115e-03]]\n",
      "Model: \"sequential_22\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "lstm_69 (LSTM)               (None, 30, 64)            442112    \n",
      "_________________________________________________________________\n",
      "lstm_70 (LSTM)               (None, 30, 128)           98816     \n",
      "_________________________________________________________________\n",
      "lstm_71 (LSTM)               (None, 64)                49408     \n",
      "_________________________________________________________________\n",
      "dense_76 (Dense)             (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "dense_77 (Dense)             (None, 32)                2080      \n",
      "_________________________________________________________________\n",
      "dense_78 (Dense)             (None, 12)                396       \n",
      "=================================================================\n",
      "Total params: 596,972\n",
      "Trainable params: 596,972\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "\n",
    "res = model.predict(X_test)\n",
    "\n",
    "print(res)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "G\n",
      "G\n"
     ]
    }
   ],
   "source": [
    "print(actions[np.argmax(res[1])])\n",
    "\n",
    "print(actions[np.argmax(y_test[1])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"common_words_1.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[17,  2],\n",
       "        [ 1,  1]],\n",
       "\n",
       "       [[16,  0],\n",
       "        [ 1,  4]],\n",
       "\n",
       "       [[17,  1],\n",
       "        [ 0,  3]],\n",
       "\n",
       "       [[18,  0],\n",
       "        [ 0,  3]],\n",
       "\n",
       "       [[18,  0],\n",
       "        [ 0,  3]],\n",
       "\n",
       "       [[17,  0],\n",
       "        [ 2,  2]],\n",
       "\n",
       "       [[19,  1],\n",
       "        [ 0,  1]]], dtype=int64)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##MODEL EVALUATION\n",
    "\n",
    "from sklearn.metrics import multilabel_confusion_matrix,accuracy_score\n",
    "yhat = model.predict(X_test)\n",
    "\n",
    "ytrue= np.argmax(y_test,axis =1 ).tolist()\n",
    "yhat = np.argmax(yhat,axis = 1 ).tolist()\n",
    "\n",
    "multilabel_confusion_matrix(ytrue, yhat)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8095238095238095"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(ytrue , yhat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "model = tf.keras.models.load_model(\"label_8_3.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Good\n",
      "Hello\n",
      "You\n",
      "Good\n",
      "To_save\n",
      "Thankyou\n",
      "Hello\n",
      "Thankyou\n",
      "To_save\n",
      "Thankyou\n",
      "Help\n",
      "Thankyou\n",
      "You\n"
     ]
    }
   ],
   "source": [
    "##Final testing\n",
    "\n",
    "import tensorflow as tf\n",
    "sequences = []#concatenating 30 frames\n",
    "sentences = []\n",
    "predictions = []\n",
    "threshold = 0.45\n",
    "\n",
    "cap = cv2.VideoCapture(0)\n",
    "with mp_holistic.Holistic(min_detection_confidence=0.5,min_tracking_confidence=0.5) as holistic:\n",
    "    while cap.isOpened():\n",
    "        ret, frame = cap.read()\n",
    "        #Using Mediapipe\n",
    "\n",
    "        image, results = mediapipe_detection(frame, holistic)   \n",
    "        \n",
    "        #Drawing Ladmarks\n",
    "        draw_landmarks2(image,results)\n",
    "        \n",
    "        ##making predictions\n",
    "        \n",
    "        keypoints = extract_keypoints(results)\n",
    "        #sequences.insert(0,keypoints)\n",
    "        sequences.append(keypoints)\n",
    "        sequences = sequences[-30:]\n",
    "        \n",
    "        if len(sequences)==30:\n",
    "            res = model.predict(np.expand_dims(sequences,axis = 0))[0]\n",
    "            predictions.append(np.argmax(res))\n",
    "            \n",
    "            if np.unique(predictions[-10:])[0]==np.argmax(res):\n",
    "                if res[np.argmax(res)]>threshold:\n",
    "                    \n",
    "                    if len(sentences)>0:\n",
    "                        \n",
    "                        if actions[np.argmax(res)]!=sentences[-1]:\n",
    "                            print(actions[np.argmax(res)])\n",
    "                            sentences.append(actions[np.argmax(res)])\n",
    "                            \n",
    "                    else:\n",
    "                        sentences.append(actions[np.argmax(res)])\n",
    "            if len(sentences)>5:\n",
    "                sentences = sentences[-5:]\n",
    "    \n",
    "        cv2.rectangle(image, (0,0),(640,40), (245,117,16),-1 )\n",
    "        cv2.putText(image,\"\".join(sentences),(3,30),cv2.FONT_HERSHEY_SIMPLEX,1,(255,255,255),2,cv2.LINE_AA)\n",
    "        \n",
    "        \n",
    "        cv2.imshow(\"frame\" , image)\n",
    "       # cv2.imwrite(\"image.jpg\", image)\n",
    "        if cv2.waitKey(1)==ord('q'):\n",
    "            break\n",
    "    cap.release()\n",
    "    cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model1 = tf.model."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}