mini.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!pip install gtts\n",
    "!pip install playsound\n",
    "!pip install tensorflow\n",
    "!pip install opencv-python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "m2p8iXQygqRN",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import string\n",
    "import glob\n",
    "from tensorflow.keras.applications.inception_v3 import InceptionV3,preprocess_input\n",
    "#import tensorflow.keras.applications.inception_v3\n",
    "from tqdm import tqdm\n",
    "import tensorflow.keras.preprocessing.image\n",
    "import pickle\n",
    "from time import time\n",
    "import numpy as np\n",
    "#from PIL import Image\n",
    "import cv2\n",
    "from tensorflow.keras.models import Sequential, load_model,Model\n",
    "from tensorflow.keras.layers import (LSTM, Embedding,TimeDistributed, Dense, RepeatVector,Activation, Flatten, Reshape, concatenate,Dropout, BatchNormalization)\n",
    "from tensorflow.keras.optimizers import Adam, RMSprop\n",
    "from tensorflow.keras import Input, layers\n",
    "from tensorflow.keras import optimizers\n",
    "from tensorflow.keras.layers import add\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.preprocessing import image\n",
    "from tensorflow.keras.preprocessing.image import img_to_array\n",
    "import tensorflow as tf\n",
    "from twilio.rest import Client\n",
    "#import matplotlib.pyplot as plt\n",
    "from gtts import gTTS\n",
    "from playsound import playsound\n",
    "START = \"startseq\"\n",
    "STOP = \"endseq\"\n",
    "EPOCHS = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "hj4K-WHafser",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "root_captioning = \"M:\\mini\\data\\data\"\n",
    "files_path = \"M:\\mini\\jupyter\\pickles\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Ra-d7BLlf1nC",
    "outputId": "066b3c7b-4a2c-431e-c784-9c38db92d8b0",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#1\n",
    "null_punct = str.maketrans('', '', string.punctuation)\n",
    "lookup = dict()\n",
    "\n",
    "with open( os.path.join(root_captioning,'text','Flickr8k.token.txt'), 'r') as fp:\n",
    "  \n",
    "  max_length = 0\n",
    "  for line in fp.read().split('\\n'):\n",
    "    tok = line.split()\n",
    "    if len(line) >= 2:\n",
    "      id = tok[0].split('.')[0]\n",
    "      desc = tok[1:]\n",
    "      desc = [word.lower() for word in desc]\n",
    "      desc = [w.translate(null_punct) for w in desc]\n",
    "      desc = [word for word in desc if len(word)>1]\n",
    "      desc = [word for word in desc if word.isalpha()]\n",
    "      max_length = max(max_length,len(desc))\n",
    "      \n",
    "      if id not in lookup:\n",
    "        lookup[id] = list()\n",
    "      lookup[id].append(' '.join(desc))\n",
    "      \n",
    "lex = set()\n",
    "for key in lookup:\n",
    "  [lex.update(d.split()) for d in lookup[key]]\n",
    "print(\"done\")\n",
    "print(lookup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Gnq3ZjaSU79-",
    "outputId": "753b88ae-250e-451c-c398-24a2aed3b17e",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(len(lookup)) \n",
    "print(len(lex)) \n",
    "print(max_length) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "JYVZ83DLsb1r",
    "outputId": "3fe292ad-77a4-428e-cb80-4fa830b626cf",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#2\n",
    "img = glob.glob(os.path.join(root_captioning,'images', '*.jpg'))\n",
    "print(\"done\")\n",
    "print(img[0],\"    \",img[1])\n",
    "print(len(img))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "DosvCz6bVZA7",
    "outputId": "54ba5bb1-d466-4bfd-ac23-a1dd7eca1e7b",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "len(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lhJSo7czkW03",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_images_path = os.path.join(root_captioning,'text','Flickr_8k.trainImages.txt') \n",
    "train_images = set(open(train_images_path, 'r').read().strip().split('\\n'))\n",
    "test_images_path = os.path.join(root_captioning,'text','Flickr_8k.testImages.txt') \n",
    "test_images = set(open(test_images_path, 'r').read().strip().split('\\n'))\n",
    "\n",
    "train_img = []\n",
    "test_img = []\n",
    "\n",
    "for i in img:\n",
    "  f = os.path.split(i)[-1]\n",
    "  if f in train_images: \n",
    "    train_img.append(f) \n",
    "  elif f in test_images:\n",
    "    test_img.append(f) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ZZQZkT8X_qNt",
    "outputId": "e1f79faa-4ff1-4dbd-bc00-2d00c30a861e",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(len(train_images))\n",
    "print(len(test_images))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "cBX6TJbPF0nD",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#3\n",
    "train_descriptions = {k:v for k,v in lookup.items() if f'{k}.jpg' in train_images}\n",
    "for n,v in train_descriptions.items(): \n",
    "  for d in range(len(v)):\n",
    "    v[d] = f'{START} {v[d]} {STOP}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "v3_PCjddHcUe",
    "outputId": "4aff4d3d-e3fe-4535-ccf6-6874776f1512",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#print(train_descriptions)\n",
    "len(train_descriptions)\n",
    "#print(lookup)\n",
    "for n,v in train_descriptions.items(): \n",
    "  print(lookup[n])\n",
    "  break\n",
    "test_descriptions = {k:v for k,v in lookup.items() if f'{k}.jpg' in test_images}\n",
    "for n,v in test_descriptions.items():\n",
    "  print(lookup[n])\n",
    "  break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "DWOP58lrtGwt",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "encode_model = InceptionV3(weights='imagenet')\n",
    "encode_model = Model(encode_model.input, encode_model.layers[-2].output)\n",
    "WIDTH = 299\n",
    "HEIGHT = 299\n",
    "OUTPUT_DIM = 2048\n",
    "preprocess_input = tensorflow.keras.applications.inception_v3.preprocess_input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "_QQ-D9Rq9SwF",
    "outputId": "63ba9036-8dd0-4c3c-b77f-24d43080d719",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "encode_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "SeT4zJ8l9Ps9",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#5\n",
    "def encodeImage(img):\n",
    "\n",
    "  #img = img.resize(tuple(WIDTH, HEIGHT), Image.ANTIALIAS)\n",
    "  img=cv2.resize(image,(WIDTH, HEIGHT),3, fx=0.5, fy=0.5, interpolation = cv2.INTER_AREA)\n",
    "  x = tensorflow.keras.preprocessing.image.img_to_array(img)\n",
    "  x = np.expand_dims(x, axis=0)\n",
    "  x = preprocess_input(x)\n",
    "  x = encode_model.predict(x) \n",
    "  x = np.reshape(x, OUTPUT_DIM )\n",
    "  return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "OfusVb6uKREw",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "train_path = os.path.join(files_path,f'train{OUTPUT_DIM}.pkl')\n",
    "if not os.path.exists(train_path):\n",
    "  start = time()\n",
    "  encoding_train = {}\n",
    "  for id in tqdm(train_img):\n",
    "    image_path = os.path.join(root_captioning,'images', id)\n",
    "    img = tensorflow.keras.preprocessing.image.load_img(image_path, \\\n",
    "            target_size=(HEIGHT, WIDTH))\n",
    "    encoding_train[id] = encodeImage(img)\n",
    "  with open(train_path, \"wb\") as fp:\n",
    "    pickle.dump(encoding_train, fp)\n",
    "else:\n",
    "  with open(train_path, \"rb\") as fp:\n",
    "    encoding_train = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4mO2cNDOP64-",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "test_path = os.path.join(files_path,f'test{OUTPUT_DIM}.pkl')\n",
    "if not os.path.exists(test_path):\n",
    "  start = time()\n",
    "  encoding_test = {}\n",
    "  for id in tqdm(test_img):\n",
    "    image_path = os.path.join(root_captioning,'images', id)\n",
    "    img = tensorflow.keras.preprocessing.image.load_img(image_path, \\\n",
    "                target_size=(HEIGHT, WIDTH))\n",
    "    encoding_test[id] = encodeImage(img)\n",
    "  with open(test_path, \"wb\") as fp:\n",
    "    pickle.dump(encoding_test, fp)\n",
    "else:\n",
    "  with open(test_path, \"rb\") as fp:\n",
    "    encoding_test = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(encoding_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "C8DFzzPmXg-n",
    "outputId": "46ce43ed-e106-4891-baeb-e68f2e6b1003",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "all_train_captions = []\n",
    "for key, val in train_descriptions.items():\n",
    "    for cap in val:\n",
    "        all_train_captions.append(cap)\n",
    "len(all_train_captions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "-HxHSgLgYUUE",
    "outputId": "a458ecce-ede5-4281-df2e-4061d8c32461",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "word_count_threshold = 10\n",
    "word_counts = {}\n",
    "nsents = 0\n",
    "for sent in all_train_captions:\n",
    "    nsents += 1\n",
    "    for w in sent.split(' '):\n",
    "        word_counts[w] = word_counts.get(w, 0) + 1\n",
    "\n",
    "vocab = [w for w in word_counts if word_counts[w] >= word_count_threshold]\n",
    "print('preprocessed words %d ==> %d' % (len(word_counts), len(vocab)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Oi3zqAUhYcNP",
    "outputId": "21364e2f-808c-4814-93d9-4b87e00e00ea",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "idxtoword = {}\n",
    "wordtoidx = {}\n",
    "\n",
    "ix = 1\n",
    "for w in vocab:\n",
    "    wordtoidx[w] = ix\n",
    "    idxtoword[ix] = w\n",
    "    ix += 1\n",
    "    \n",
    "vocab_size = len(idxtoword) + 1 \n",
    "vocab_size\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "E4Z4hLNJUObA",
    "outputId": "ae38594d-3cd9-423b-dcb6-8d4abb56b1fa",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "print(idxtoword)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Rhhi4lRqVAMM",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "idxtoword_path = os.path.join(files_path,'idxtoword.pkl')\n",
    "with open(idxtoword_path, \"wb\") as fp:\n",
    "    pickle.dump(idxtoword, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Z3nW9VksW58_",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "wordtoidx_path = os.path.join(files_path,'wordtoidx.pkl')\n",
    "with open(wordtoidx_path, \"wb\") as fp:\n",
    "    pickle.dump(wordtoidx, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "KSohY53bYjTi",
    "outputId": "06fb2caf-a699-40a5-869f-8473c49d554c",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "max_length +=2\n",
    "print(max_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gZsbbCx6d04X",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def data_generator(descriptions, photos, wordtoidx, \\\n",
    "                   max_length, num_photos_per_batch):\n",
    "  x1, x2, y = [], [], []\n",
    "  n=0\n",
    "  while True:\n",
    "    for key, desc_list in descriptions.items():\n",
    "      n+=1\n",
    "      photo = photos[key+'.jpg']\n",
    "      for desc in desc_list:\n",
    "        seq = [wordtoidx[word] for word in desc.split(' ') \\\n",
    "               if word in wordtoidx]\n",
    "        for i in range(1, len(seq)):\n",
    "          in_seq, out_seq = seq[:i], seq[i]\n",
    "          in_seq = pad_sequences([in_seq], maxlen=max_length)[0]\n",
    "          out_seq = to_categorical([out_seq], num_classes=vocab_size)[0]\n",
    "          x1.append(photo)\n",
    "          x2.append(in_seq)\n",
    "          y.append(out_seq)\n",
    "      if n==num_photos_per_batch:\n",
    "        yield ([np.array(x1), np.array(x2)], np.array(y))\n",
    "        x1, x2, y = [], [], []\n",
    "        n=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "IY_9XZ4Hec73",
    "outputId": "a2e33145-9674-435d-a9a2-955916cb9d80",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "glove_dir = os.path.join(root_captioning)\n",
    "embeddings_index = {} \n",
    "f = open(os.path.join(glove_dir, 'glove.6B.200d.txt'), encoding=\"utf-8\")\n",
    "\n",
    "for line in tqdm(f):\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:], dtype='float32')\n",
    "    embeddings_index[word] = coefs\n",
    "\n",
    "f.close()\n",
    "print(f'Found {len(embeddings_index)} word vectors.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "adEmnYFEfog6",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "embedding_dim = 200\n",
    "embedding_matrix = np.zeros((vocab_size, embedding_dim))\n",
    "\n",
    "for word, i in wordtoidx.items():\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "86hPHHjSfx8p",
    "outputId": "0ee1be6d-3fe5-400e-de7f-dd6277fecbf0",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "embedding_matrix.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gRYqbogOXILG",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "embedding_matrix_path = os.path.join(files_path,'embedding_matrix.pkl')\n",
    "with open(embedding_matrix_path, \"wb\") as fp:\n",
    "    pickle.dump(embedding_matrix, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2homy6Znf7wL",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "inputs1 = Input(shape=(OUTPUT_DIM,))\n",
    "fe1 = Dropout(0.5)(inputs1)\n",
    "fe2 = Dense(256, activation='relu')(fe1)\n",
    "inputs2 = Input(shape=(max_length,))\n",
    "se1 = Embedding(vocab_size, embedding_dim, mask_zero=True)(inputs2)\n",
    "se2 = Dropout(0.5)(se1)\n",
    "se3 = LSTM(256)(se2)\n",
    "decoder1 = add([fe2, se3])\n",
    "decoder2 = Dense(256, activation='relu')(decoder1)\n",
    "outputs = Dense(vocab_size, activation='softmax')(decoder2)\n",
    "caption_model = Model(inputs=[inputs1, inputs2], outputs=outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "UcCe2n5pnGfo",
    "outputId": "b4ac0b0c-661e-4d1d-b58f-6bd80640a56f",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "embedding_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "asGo7TqRfgJp",
    "outputId": "5b387ba1-d36a-47e5-a99e-1db4e2f9738b",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "caption_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gze36nMIfoEg",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "caption_model.layers[2].set_weights([embedding_matrix])\n",
    "caption_model.layers[2].trainable = False\n",
    "caption_model.compile(loss='categorical_crossentropy', optimizer='adam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "0LXG3PE5fxzU",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "number_pics_per_bath = 3\n",
    "steps = len(train_descriptions)//number_pics_per_bath"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "dyCHBbjnf2b1",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model_path = os.path.join(\"M:\\mini\\jupyter\\model\",f'caption-model.hdf5')\n",
    "if not os.path.exists(model_path):\n",
    "  for i in tqdm(range(EPOCHS*2)):\n",
    "      generator = data_generator(train_descriptions, encoding_train, \n",
    "                    wordtoidx, max_length, number_pics_per_bath)\n",
    "      caption_model.fit_generator(generator, epochs=1,\n",
    "                    steps_per_epoch=steps, verbose=1)\n",
    "\n",
    "  caption_model.optimizer.lr = 1e-4\n",
    "  number_pics_per_bath = 6\n",
    "  steps = len(train_descriptions)//number_pics_per_bath\n",
    "\n",
    "  for i in range(EPOCHS):\n",
    "      generator = data_generator(train_descriptions, encoding_train, \n",
    "                    wordtoidx, max_length, number_pics_per_bath)\n",
    "      caption_model.fit_generator(generator, epochs=1, \n",
    "                            steps_per_epoch=steps, verbose=1)  \n",
    "  caption_model.save_weights(model_path)\n",
    "else:\n",
    "  caption_model.load_weights(model_path)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "JPKINJ3JMbSy",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def generateCaption(photo):\n",
    "    in_text = START\n",
    "    for i in range(max_length):\n",
    "        sequence = [wordtoidx[w] for w in in_text.split() if w in wordtoidx]\n",
    "        sequence = pad_sequences([sequence], maxlen=max_length)\n",
    "        yhat = caption_model.predict([photo,sequence], verbose=0)\n",
    "        yhat = np.argmax(yhat)\n",
    "        word = idxtoword[yhat]\n",
    "        in_text += ' ' + word\n",
    "        if word == STOP:\n",
    "            break\n",
    "    final = in_text.split()\n",
    "    final = final[1:-1]\n",
    "    final = ' '.join(final)\n",
    "    return final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model=load_model(r'M:\\mini\\jupyter\\model\\alert_model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import cv2\n",
    "from datetime import datetime\n",
    "vidcap = cv2.VideoCapture(r'C:\\Users\\yuvasri\\Downloads\\queda.mp4')\n",
    "count = 0\n",
    "success = True\n",
    "fps = int(vidcap.get(cv2.CAP_PROP_FPS))\n",
    "T = datetime.now()\n",
    "print(T)\n",
    "text=[]\n",
    "while success:\n",
    "    success,image = vidcap.read()\n",
    "    print(\"1\")\n",
    "    x1 = np.expand_dims(image, axis=0)\n",
    "    x1 = preprocess_input(x1)\n",
    "    outcome = [np.argmax(model.predict(x1))]\n",
    "    if outcome==[[1]]:\n",
    "        print(\"no\")\n",
    "        account_sid = \"AC99e30cc80bd4de0e2946118bf1d1d1b9\"\n",
    "        auth_token = \"5d737b2efe40561722780f53cd0fe1b0\"\n",
    "        client = Client(account_sid, auth_token)\n",
    "\n",
    "        call = client.calls.create(\n",
    "                        url='http://demo.twilio.com/docs/voice.xml',\n",
    "                        to='+919182544970',\n",
    "                        from_='+16503833718'\n",
    "                    )\n",
    "        print(call.sid)\n",
    "    if count%(2*fps) == 0 :\n",
    "         cv2.imwrite(r'M:\\mini\\jupyter\\images\\frame%d.jpg'%count,image)\n",
    "         #print(image.shape)\n",
    "         #print(type(image))\n",
    "         #cv2.imshow(\"image\"+str(count),image)\n",
    "         #print('successfully written 10th frame')\n",
    "         WIDTH = 299\n",
    "         HEIGHT = 299\n",
    "         a=(299,299)\n",
    "         OUTPUT_DIM = 2048\n",
    "         #img = np.resize(img,(299,299))\n",
    "         img=cv2.resize(image,(299,299),3, fx=0.5, fy=0.5, interpolation = cv2.INTER_AREA)\n",
    "         img=encodeImage(img).reshape((1,OUTPUT_DIM))\n",
    "         x=generateCaption(img)\n",
    "         seconds=datetime.now()-T\n",
    "         print(seconds.total_seconds())\n",
    "         #x = image.img_to_array(image)\n",
    "         if(seconds.total_seconds()>=10):\n",
    "                T=datetime.now()\n",
    "                print(x)\n",
    "                text.append(x)\n",
    "                print(\" \".join(text))\n",
    "                speech = gTTS(text=\"            \".join(text), lang='en', slow=False)\n",
    "                speech.save(r'M:\\mini\\jupyter\\audio\\audio%d.mp3'%count)\n",
    "                playsound(r'M:\\mini\\jupyter\\audio\\audio%d.mp3'%count)\n",
    "                text=[]\n",
    "         else:\n",
    "                print(x)\n",
    "                text.append(x)\n",
    "        \n",
    "    count+=1\n",
    "    if cv2.waitKey(20) & 0xFF == ord('q'):\n",
    "          break\n",
    "\n",
    "cap.release()\n",
    "cv2.destroyAllWindows()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "count=240\n",
    "playsound(r'M:\\mini\\jupyter\\audio\\audio%d.mp3'%count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from keras.models import load_model\n",
    "from keras.preprocessing import image\n",
    "from keras.preprocessing.image import img_to_array\n",
    "import tensorflow as tf\n",
    "model=load_model('M:\\mini\\jupyter\\model\\alert_model.h5')\n",
    "from twilio.rest import Client\n",
    "img = image.load_img(r\"C:\\Users\\jyoth\\Desktop\\6.jfif\", target_size=(224, 224))\n",
    "x = image.img_to_array(img)\n",
    "x = np.expand_dims(x, axis=0)\n",
    "x = preprocess_input(x)\n",
    "outcome = [np.argmax(model.predict(x))]\n",
    "if outcome==[[0]]:\n",
    "    print(\"no\")\n",
    "    account_sid = \"AC99e30cc80bd4de0e2946118bf1d1d1b9\"\n",
    "    auth_token = \"5d737b2efe40561722780f53cd0fe1b0\"\n",
    "    client = Client(account_sid, auth_token)\n",
    "\n",
    "    call = client.calls.create(\n",
    "                        url='http://demo.twilio.com/docs/voice.xml',\n",
    "                        to='+17704673802',\n",
    "                        from_='+919182544970'\n",
    "                    )\n",
    "\n",
    "    print(call.sid)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pip show twilio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "colab": {
   "name": "mini.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}