chatbot_v8_2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "language_info": {
      "name": "python",
      "version": "3.6.6",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "colab": {
      "name": "chatbot_v8.2.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/VGGatGitHub/AnswerFlow/blob/master/chatbot_v8_2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eIF2hQ5wGsqg",
        "colab_type": "text"
      },
      "source": [
        "https://www.kaggle.com/datasets?search=nq-train\n",
        "\n",
        "V2: added comads to look at the structure of the train.json file and to assess the %s.\n",
        "\n",
        "V3: changing the code to do tarining for long_answesrs or short_answers using training_for_long_answer switch.\n",
        "\n",
        "V4: reading in json file produced from jsonl using jsonl2json.ipynb\n",
        "\n",
        "V6: reading files from GitHub\n",
        "\n",
        "V7: using similarity function to assess the new answers \n",
        "\n",
        "V8: modified the process of geting the training data file\n",
        "\n",
        "V8.2: Added dropout that randomly drops 0.1 of the inputs and stoping after no-improvement for preset value of iterations.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9xPDM8ocY74i",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#VGG in case of using a Google Drive for your files\n",
        "#from google.colab import drive\n",
        "#drive.mount('/content/drive')\n",
        "#path='/content/drive/My Drive/Colab Notebooks/'\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "ed-1NnYIGsqj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import sys \n",
        "import os\n",
        "print(os.getcwd())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QByI39lkVt8B",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#VGG define the foldre to inspect for files \n",
        "path=os.getcwd()+\"/\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7_S1byB1WPph",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#VGG read in training data\n",
        "#you may have to adjust the BATCH_SIZE acordingly \n",
        "#path='/content/drive/My Drive/Colab Notebooks/'\n",
        "\n",
        "file_name='train200.json' #or train25.json or train200.json \n",
        "\n",
        "file_to_read=path+file_name"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vCUwJWTEms7O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Geting the training data file \n",
        "\n",
        "from pathlib import Path\n",
        "import requests\n",
        "\n",
        "try:\n",
        "  if not Path(file_to_read).is_file():\n",
        "    file_url='https://raw.githubusercontent.com/VGGatGitHub/natural-questions/master/'+file_name\n",
        "    print(\"Will try to fetch the file from:\\n\",file_url)\n",
        "\n",
        "    response = requests.get(file_url)\n",
        "    if response.status_code == 200:\n",
        "      print('Success!')\n",
        "      s=response.content\n",
        "      # Code for printing to a file \n",
        "      sample = open(file_to_read, 'w') \n",
        "      doc=s.decode()\n",
        "      print(doc, file = sample) \n",
        "      sample.close()\n",
        "    else:\n",
        "      print(\"Faild to fetch the file!\\n\",file_url)\n",
        "      print(response)\n",
        "except Exception:\n",
        "    print(\"Exception: Faild to find or fetch the training file needed:\",file_name)\n",
        "  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "3ej-f3iYGsqm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#make sure the file you what is in the correct directory\n",
        "#some possible files are train.json or train200.json \n",
        "\n",
        "for dirname, _, filenames in os.walk(path):\n",
        "    for filename in filenames:\n",
        "        print(os.path.join(dirname, filename))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cS2xGhZCoFxB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# VGG\n",
        "# you may need to get the file text_utils.py from \n",
        "# https://github.com/VGGatGitHub/natural-questions\n",
        "#\n",
        "\n",
        "# sys.path.append(os.path.abspath(path))\n",
        "# from text_utils import *\n",
        "\n",
        "# got import errors on colab using import code above\n",
        "# import text_utils"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5EhFsBX5aKR_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#VGG The cell has been removed since now the data is analized in the jsonl2json.ipynb"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5pgXArJmIaiv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
        "\n",
        "# Install TensorFlow\n",
        "try:\n",
        "  # %tensorflow_version only exists in Colab.\n",
        "  %tensorflow_version 2.x\n",
        "except Exception:\n",
        "  pass\n",
        "\n",
        "import tensorflow as tf\n",
        "print(tf.__version__)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g7bPY5ic2eC8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
        "\n",
        "import tensorflow as tf\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import matplotlib.ticker as ticker\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "import unicodedata\n",
        "import re\n",
        "import numpy as np\n",
        "import os\n",
        "import io\n",
        "import time\n",
        "import json\n",
        "\n",
        "# Converts the unicode file to ascii\n",
        "def unicode_to_ascii(s):\n",
        "    return ''.join(c for c in unicodedata.normalize('NFD', s)\n",
        "        if unicodedata.category(c) != 'Mn')\n",
        "\n",
        "\n",
        "def preprocess_sentence(w):\n",
        "    w = unicode_to_ascii(w.lower().strip())\n",
        "\n",
        "    # creating a space between a word and the punctuation following it\n",
        "    # eg: \"he is a boy.\" => \"he is a boy .\"\n",
        "    # Reference:- https://stackoverflow.com/questions/3645931/python-padding-punctuation-with-white-spaces-keeping-punctuation\n",
        "    w = re.sub(r\"([?.!,¿])\", r\" \\1 \", w)\n",
        "    w = re.sub(r'[\" \"]+', \" \", w)\n",
        "\n",
        "    # replacing everything with space except (0-9, a-z, A-Z, \".\", \"?\", \"!\", \",\")\n",
        "    w = re.sub(r\"[^0-9a-zA-Z?.!,¿]+\", \" \", w)\n",
        "\n",
        "    w = w.rstrip().strip()\n",
        "\n",
        "    # adding a start and an end token to the sentence\n",
        "    # so that the model know when to start and stop predicting.\n",
        "    w = '<start> ' + w + ' <end>'\n",
        "    return w\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NbW-oSFI2mFh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#VGG make sure the file_to_read has been defined above! \n",
        "\n",
        "UNKNOWN = \"<UNKNOWN>\"\n",
        "# 1. Remove the accents\n",
        "# 2. Clean the sentences\n",
        "# 3. Return word pairs in the format: [ENGLISH, SPANISH]\n",
        "def create_dataset():\n",
        "    source = []\n",
        "    target = []\n",
        "    context = []\n",
        "\n",
        "    n_short_answers=0 #VGG\n",
        "    n_long=0\n",
        "    \n",
        "    training_for_long_answer = False #True #False \n",
        "    #make sure to run furst for shor answers and then for long...\n",
        "\n",
        "    with open(file_to_read) as json_file: #VGG\n",
        "        data = json.load(json_file)\n",
        "\n",
        "        for nq_doc in data:\n",
        "            if filename == 'train200L.json':\n",
        "              doc = simplify_nq_example(nq_doc) #VGG for jsonl formated file\n",
        "            else:\n",
        "              doc=nq_doc\n",
        "\n",
        "            question_text = doc['question_text']\n",
        "            document_text = doc['document_text'].split()\n",
        "            long_answer_candidates = doc['long_answer_candidates']\n",
        "            annotations = doc['annotations'][0]\n",
        "            \n",
        "            if annotations['long_answer']['start_token'] < annotations['long_answer']['end_token']:\n",
        "                \n",
        "                n_long+=1\n",
        "                long_answer = \" \".join(document_text[annotations['long_answer']['start_token']:\n",
        "                                                     annotations['long_answer']['end_token']])\n",
        "                                      \n",
        "                if len(annotations['short_answers']) > 0:\n",
        "                    start_token = annotations['short_answers'][0]['start_token']\n",
        "                    end_token = annotations['short_answers'][0]['end_token']\n",
        "                    short_answer = \" \".join(document_text[start_token:end_token])\n",
        "                    n_short_answers+=1 #VGG\n",
        "                else:\n",
        "                    short_answer = UNKNOWN\n",
        "                \n",
        "                #VGG V3\n",
        "                if training_for_long_answer :\n",
        "                    short_answer=long_answer #VGG V3 change - make the target to be the long answer instead of the short answer \n",
        "                    for posibilities in long_answer_candidates:\n",
        "                        if posibilities[\"top_level\"]:\n",
        "                            start_token = posibilities['start_token']\n",
        "                            end_token = posibilities['end_token']                    \n",
        "                            posibility = \" \".join(document_text[start_token:end_token])\n",
        "                            context.append(preprocess_sentence(posibility))\n",
        "                else:\n",
        "                    context.append(preprocess_sentence(long_answer))\n",
        "                #VGG context = [] #VGG it seems to work better!\n",
        "\n",
        "                source.append(preprocess_sentence(question_text))\n",
        "                target.append(preprocess_sentence(short_answer))\n",
        "#VGG                \n",
        "        print(\"Data set of:\",len(data),\" elements. It contains:\",\n",
        "              n_short_answers,\"short answers out of\", n_long,\n",
        "              \"possible long answers, short/long rate is {:.0f}%\".format(\n",
        "                  100*n_short_answers/n_long))    \n",
        "    return target, source, context\n",
        " "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "_uuid": "4ecf3676-acd1-45e5-9459-3f9030b8b059",
        "_cell_guid": "3f83746e-4fa4-4e24-98cb-2f0df139a6af",
        "trusted": true,
        "id": "jevAjFFiGsqr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def max_length(tensor):\n",
        "    return max(len(t) for t in tensor)\n",
        "    \n",
        "def tokenize(lang):\n",
        "  lang_tokenizer = tf.keras.preprocessing.text.Tokenizer(\n",
        "      filters='')\n",
        "  lang_tokenizer.fit_on_texts(lang)\n",
        "  tensor = lang_tokenizer.texts_to_sequences(lang)\n",
        "  tensor = tf.keras.preprocessing.sequence.pad_sequences(tensor,\n",
        "                                                         padding='post')\n",
        "  return tensor, lang_tokenizer\n",
        "\n",
        "def load_dataset():\n",
        "    # creating cleaned input, output pairs\n",
        "    targ_lang, inp_lang, context_lang = create_dataset()\n",
        "\n",
        "    input_tensor, inp_lang_tokenizer = tokenize(inp_lang)\n",
        "    target_tensor, targ_lang_tokenizer = tokenize(targ_lang)\n",
        "    \n",
        "    return input_tensor, target_tensor, inp_lang_tokenizer, targ_lang_tokenizer\n",
        "\n",
        "def convert(lang, tensor):\n",
        "  for t in tensor:\n",
        "    if t!=0:\n",
        "      print (\"%d ----> %s\" % (t, lang.index_word[t]))\n",
        "\n",
        "class Encoder(tf.keras.Model):\n",
        "  def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):\n",
        "    super(Encoder, self).__init__()\n",
        "    self.batch_sz = batch_sz\n",
        "    self.enc_units = enc_units\n",
        "    self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
        "    self.gru = tf.keras.layers.GRU(self.enc_units,\n",
        "                                   return_sequences=True,\n",
        "                                   return_state=True,\n",
        "                                   recurrent_initializer='glorot_uniform')\n",
        "    self.dropout = tf.keras.layers.Dropout(0.2)\n",
        "\n",
        "  def call(self, x, hidden, training):\n",
        "    x = self.embedding(x)\n",
        "    output, state = self.gru(x, initial_state = hidden)\n",
        "    output = self.dropout(output, training=training)\n",
        "    return output, state\n",
        "\n",
        "  def initialize_hidden_state(self):\n",
        "    return tf.zeros((self.batch_sz, self.enc_units))\n",
        "    \n",
        "class BahdanauAttention(tf.keras.layers.Layer):\n",
        "  def __init__(self, units):\n",
        "    super(BahdanauAttention, self).__init__()\n",
        "    self.W1 = tf.keras.layers.Dense(units)\n",
        "    self.W2 = tf.keras.layers.Dense(units)\n",
        "    self.V = tf.keras.layers.Dense(1)\n",
        "\n",
        "  def call(self, query, values):\n",
        "    # hidden shape == (batch_size, hidden size)\n",
        "    # hidden_with_time_axis shape == (batch_size, 1, hidden size)\n",
        "    # we are doing this to perform addition to calculate the score\n",
        "    hidden_with_time_axis = tf.expand_dims(query, 1)\n",
        "\n",
        "    # score shape == (batch_size, max_length, 1)\n",
        "    # we get 1 at the last axis because we are applying score to self.V\n",
        "    # the shape of the tensor before applying self.V is (batch_size, max_length, units)\n",
        "    score = self.V(tf.nn.tanh(\n",
        "        self.W1(values) + self.W2(hidden_with_time_axis)))\n",
        "\n",
        "    # attention_weights shape == (batch_size, max_length, 1)\n",
        "    attention_weights = tf.nn.softmax(score, axis=1)\n",
        "\n",
        "    # context_vector shape after sum == (batch_size, hidden_size)\n",
        "    context_vector = attention_weights * values\n",
        "    context_vector = tf.reduce_sum(context_vector, axis=1)\n",
        "\n",
        "    return context_vector, attention_weights\n",
        "    \n",
        "class Decoder(tf.keras.Model):\n",
        "  def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):\n",
        "    super(Decoder, self).__init__()\n",
        "    self.batch_sz = batch_sz\n",
        "    self.dec_units = dec_units\n",
        "    self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
        "    self.gru = tf.keras.layers.GRU(self.dec_units,\n",
        "                                   return_sequences=True,\n",
        "                                   return_state=True,\n",
        "                                   recurrent_initializer='glorot_uniform')\n",
        "    self.fc = tf.keras.layers.Dense(vocab_size)\n",
        "\n",
        "    # used for attention\n",
        "    self.attention = BahdanauAttention(self.dec_units)\n",
        "\n",
        "    self.dropout = tf.keras.layers.Dropout(0.2)\n",
        "\n",
        "  def call(self, x, hidden, enc_output, training):\n",
        "    # enc_output shape == (batch_size, max_length, hidden_size)\n",
        "    context_vector, attention_weights = self.attention(hidden, enc_output)\n",
        "    context_vector = self.dropout(context_vector, training=training)\n",
        "\n",
        "    # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n",
        "    x = self.embedding(x)\n",
        "\n",
        "    # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n",
        "    x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n",
        "\n",
        "    # passing the concatenated vector to the GRU\n",
        "    output, state = self.gru(x)\n",
        "\n",
        "    # output shape == (batch_size * 1, hidden_size)\n",
        "    output = tf.reshape(output, (-1, output.shape[2]))\n",
        "\n",
        "    # output shape == (batch_size, vocab)\n",
        "    x = self.fc(output)\n",
        "\n",
        "    return x, state, attention_weights\n",
        "\n",
        "# Try experimenting with the size of that dataset\n",
        "input_tensor, target_tensor, inp_lang, targ_lang = load_dataset()\n",
        "\n",
        "# Calculate max_length of the target tensors\n",
        "max_length_targ, max_length_inp = max_length(target_tensor), max_length(input_tensor)    \n",
        "\n",
        "# Creating training and validation sets using an 80-20 split\n",
        "input_tensor_train, input_tensor_val, target_tensor_train, target_tensor_val = train_test_split(input_tensor, target_tensor, test_size=0.2)\n",
        "\n",
        "BUFFER_SIZE = len(input_tensor_train)\n",
        "BATCH_SIZE = 8 #VGG\n",
        "steps_per_epoch = len(input_tensor_train)//BATCH_SIZE\n",
        "embedding_dim = 1024\n",
        "units = 1024\n",
        "vocab_inp_size = len(inp_lang.word_index)+1\n",
        "vocab_tar_size = len(targ_lang.word_index)+1\n",
        "\n",
        "dataset = tf.data.Dataset.from_tensor_slices((input_tensor_train, target_tensor_train)).shuffle(BUFFER_SIZE)\n",
        "dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)\n",
        "\n",
        "example_input_batch, example_target_batch = next(iter(dataset))\n",
        "example_input_batch.shape, example_target_batch.shape\n",
        "\n",
        "encoder = Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE)    \n",
        "decoder = Decoder(vocab_tar_size, embedding_dim, units, BATCH_SIZE)\n",
        "optimizer = tf.keras.optimizers.Adam()\n",
        "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n",
        "    from_logits=True, reduction='none')\n",
        "    \n",
        "def loss_function(real, pred):\n",
        "  mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
        "  loss_ = loss_object(real, pred)\n",
        "  mask = tf.cast(mask, dtype=loss_.dtype)\n",
        "  loss_ = tf.multiply(loss_, mask)\n",
        "  return tf.reduce_mean(loss_)\n",
        "\n",
        "\n",
        "#VGG uncommented for possible checkpoint saving later \n",
        "\n",
        "checkpoint_dir = './training_checkpoints'\n",
        "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n",
        "checkpoint = tf.train.Checkpoint(optimizer=optimizer,\n",
        "                                 encoder=encoder,\n",
        "                                 decoder=decoder)  \n",
        "\n",
        "                                 \n",
        "@tf.function\n",
        "def train_step(inp, targ, enc_hidden):\n",
        "  loss = 0\n",
        "\n",
        "  with tf.GradientTape() as tape:\n",
        "    enc_output, enc_hidden = encoder(inp, enc_hidden, True)\n",
        "    dec_hidden = enc_hidden\n",
        "    dec_input = tf.expand_dims([targ_lang.word_index['<start>']] * BATCH_SIZE, 1)\n",
        "\n",
        "    # Teacher forcing - feeding the target as the next input\n",
        "    for t in range(1, targ.shape[1]):\n",
        "      # passing enc_output to the decoder\n",
        "      predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output, True)\n",
        "\n",
        "      loss += loss_function(targ[:, t], predictions)\n",
        "\n",
        "      # using teacher forcing\n",
        "      dec_input = tf.expand_dims(targ[:, t], 1)\n",
        "\n",
        "  batch_loss = (loss / int(targ.shape[1]))\n",
        "  variables = encoder.trainable_variables + decoder.trainable_variables\n",
        "  gradients = tape.gradient(loss, variables)\n",
        "  optimizer.apply_gradients(zip(gradients, variables))\n",
        "  return batch_loss\n",
        "  \n",
        "def evaluate(sentence):\n",
        "    attention_plot = np.zeros((max_length_targ, max_length_inp))\n",
        "    sentence = preprocess_sentence(sentence)\n",
        "\n",
        "    inputs = [inp_lang.word_index.get(i, 0) for i in sentence.split(' ')]\n",
        "    inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs],\n",
        "                                                           maxlen=max_length_inp,\n",
        "                                                           padding='post')\n",
        "    inputs = tf.convert_to_tensor(inputs)\n",
        "\n",
        "    result = ''\n",
        "    hidden = [tf.zeros((1, units))]\n",
        "    enc_out, enc_hidden = encoder(inputs, hidden, False)\n",
        "\n",
        "    dec_hidden = enc_hidden\n",
        "    dec_input = tf.expand_dims([targ_lang.word_index['<start>']], 0)\n",
        "\n",
        "    result ='<start> '#VGG \n",
        "    for t in range(max_length_targ):\n",
        "        predictions, dec_hidden, attention_weights = decoder(dec_input,\n",
        "                                                             dec_hidden,\n",
        "                                                             enc_out,\n",
        "                                                             False)\n",
        "\n",
        "        # storing the attention weights to plot later on\n",
        "        attention_weights = tf.reshape(attention_weights, (-1, ))\n",
        "        attention_plot[t] = attention_weights.numpy()\n",
        "\n",
        "        predicted_id = tf.argmax(predictions[0]).numpy()\n",
        "        result += targ_lang.index_word[predicted_id] + ' '\n",
        "\n",
        "        if targ_lang.index_word[predicted_id] == '<end>':\n",
        "            return result, sentence, attention_plot\n",
        "\n",
        "        # the predicted ID is fed back into the model\n",
        "        dec_input = tf.expand_dims([predicted_id], 0)\n",
        "    return result, sentence, attention_plot\n",
        "    \n",
        "#If you get error message about iretation problem -  check your BATCH_SIZE"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "OMsxPEhOGsqt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "EPOCHS = 128\n",
        "\n",
        "epoch=-1\n",
        "total_loss=1\n",
        "total_loss_cut=0.001*steps_per_epoch*BATCH_SIZE\n",
        "training_start_time=time.time()\n",
        "\n",
        "# early stopping parameters\n",
        "best_total_loss = sys.maxsize\n",
        "patience = 16\n",
        "early_stopping = False\n",
        "stopping_step = 0\n",
        "\n",
        "# print(\"\\nStarting training of at most {} epochs or until total loss is les than {:0.4f}\".format(EPOCHS,total_loss_cut))\n",
        "# while (epoch < EPOCHS) and (total_loss > total_loss_cut):\n",
        "print(\"\\nStarting training of at most {} epochs or if no improvement after {} epochs\".format(EPOCHS,patience))\n",
        "while (epoch < EPOCHS) and (early_stopping == False):  \n",
        "  epoch+=1\n",
        "\n",
        "  start = time.time()\n",
        "  enc_hidden = encoder.initialize_hidden_state()\n",
        "  total_loss = 0\n",
        "\n",
        "  for (batch, (inp, targ)) in enumerate(dataset.take(steps_per_epoch)):\n",
        "    batch_loss = train_step(inp, targ, enc_hidden)\n",
        "    total_loss += batch_loss\n",
        "\n",
        "    if batch%8 == 0:\n",
        "      print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1,\n",
        "                                                     batch,\n",
        "                                                     batch_loss.numpy()))\n",
        "  '''\n",
        "  # saving (checkpoint) the model every 2 epochs\n",
        "  if (epoch + 1) % 2 == 0:\n",
        "    checkpoint.save(file_prefix = checkpoint_prefix)\n",
        "  '''   \n",
        "\n",
        "  # early stopping\n",
        "  if (total_loss < best_total_loss):\n",
        "    stopping_step = 0\n",
        "    best_total_loss = total_loss\n",
        "  else:\n",
        "    stopping_step += 1\n",
        "  if stopping_step >= patience:\n",
        "    early_stopping = True\n",
        "\n",
        "  print('Epoch {} Total Loss {:.4f}'.format(epoch + 1, total_loss))\n",
        "  print('Time taken for this epoch {:.4f} sec\\n'.format(time.time() - start))\n",
        "\n",
        "print('BATCH_SIZE:{}, total training time {:.2f} minutes for {} epochs, final total_loss {:.4f}\\n'.format(\n",
        "    BATCH_SIZE,(time.time() - training_start_time)/60,epoch+1,total_loss))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MG-iceMSrYNF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print('BATCH_SIZE:{}, total training time {:.2f} minutes for {} epochs, final total_loss {:.4f}\\n'.format(\n",
        "    BATCH_SIZE,(time.time() - training_start_time)/60,epoch+1,total_loss))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "iI09zf-bGsqy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# function for plotting the attention weights\n",
        "def plot_attention(attention, sentence, predicted_sentence):\n",
        "    fig = plt.figure(figsize=(10,10))\n",
        "    ax = fig.add_subplot(1, 1, 1)\n",
        "    ax.matshow(attention, cmap='viridis')\n",
        "    fontdict = {'fontsize': 14}\n",
        "    ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n",
        "    ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n",
        "    ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n",
        "    ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n",
        "    plt.show()\n",
        "\n",
        "def ask(sentence):\n",
        "    result, sentence1, attention_plot = evaluate(sentence)\n",
        "    print('\\nQuestion: %s' % (sentence))\n",
        "    print('Predicted answer: {}'.format(result))\n",
        "    return result\n",
        "\n",
        "def show_attention_plot(sentence):\n",
        "  result, sentence1, attention_plot = evaluate(sentence)\n",
        "  attention_plot = attention_plot[:len(result.split(' ')), :len(sentence.split(' '))]\n",
        "  plot_attention(attention_plot, sentence.split(' '), result.split(' '))\n",
        "\n",
        "def is_it_known(sentence):\n",
        "    result, sentence, attention_plot = evaluate(sentence)\n",
        "    if result.split() != ['<start>', 'unknown', '<end>']: return True\n",
        "    return False\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "_uuid": "7a7fe15c-ca68-49f0-bec9-059d7ef47e6a",
        "_cell_guid": "4684e38d-ab84-492d-8e2f-d13b1a0924fd",
        "trusted": true,
        "id": "4hRJh7EFGsqw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ask('nnn')\n",
        "ask('how did I meet your mother')\n",
        "ask('who is your mother');\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dkzhBscq3zWi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "show_attention_plot(\"how did I meet your mother\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "VbL6X2bzGsq3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "show_attention_plot(\"who plays young flo in the progressive commercials\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PEdV3whqoTty",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import spacy.cli\n",
        "spacy.cli.download(\"en_core_web_md\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K_fGaAGKoVLj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import spacy as sp\n",
        "nlp = sp.load(\"en_core_web_md\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8IigyUmaoX3f",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Test the similarity measure and get some idea about the output values \n",
        "\n",
        "# sample text\n",
        "messages = [\n",
        "# Smartphones\n",
        "\"My phone is not good.\",\n",
        "\"Your cellphone looks great.\",\n",
        "# Weather\n",
        "\"Will it snow tomorrow?\",\n",
        "\"Recently a lot of hurricanes have hit the US\",\n",
        "# Food and health\n",
        "\"An apple a day, keeps the doctors away\",\n",
        "\"Eating strawberries is healthy\"\n",
        "]\n",
        "\n",
        "for text1 in messages:\n",
        "  doc1 = nlp(text1)\n",
        "  print()\n",
        "  for text2 in messages:\n",
        "    doc2 = nlp(text2)\n",
        "    print(doc1.similarity(doc2))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-iRhZwdJrWPG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "target, source, context = create_dataset()\n",
        "\n",
        "print(\"\\ngoing over all the questions and selecting those with answers ... \\n\")\n",
        "\n",
        "n_answers=0\n",
        "i=-1\n",
        "n_correct=0\n",
        "n_smlr=0\n",
        "smlrty_cut=85\n",
        "for question_text in source:\n",
        "    i+=1\n",
        "    TheAnswer=target[i]\n",
        "    TheAnswer=TheAnswer.replace('<start>',' ')\n",
        "    TheAnswer=TheAnswer.replace('<end>',' ')\n",
        "    doc1 = nlp(TheAnswer)\n",
        "    if is_it_known(question_text):\n",
        "        n_answers+=1\n",
        "        AFanswer=ask(question_text)\n",
        "        AFanswer=AFanswer.replace('<start>',' ')\n",
        "        AFanswer=AFanswer.replace('<end>',' ')\n",
        "        doc2 = nlp(AFanswer)\n",
        "        if AFanswer.split() == TheAnswer.split(): \n",
        "          n_correct+=1\n",
        "        else:\n",
        "          smlrty=100*doc1.similarity(doc2)\n",
        "          if smlrty > smlrty_cut: n_smlr+=1\n",
        "          print(\"The answer was:\",target[i])\n",
        "          print(\"Similarity:{:0.2f}%\\n\".format(smlrty))\n",
        "\n",
        "print(\"\\n{} answers out of {} possible, rate is {:.0f}%\".format(n_answers,len(source),100*n_answers/len(source)))\n",
        "if n_answers >0:\n",
        "  print(\"At least {} correct answers out of {} possible, rate is {:.0f}%\".format(n_correct,n_answers,100*n_correct/n_answers))\n",
        "if n_smlr >0:\n",
        "  print(\"There were {} similar answers with  similarity above {}%\\n\".format(n_smlr,smlrty_cut))\n",
        "           "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SXQ0s6F9nsH9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "target, source, context = create_dataset()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lyw_WyJB0tZ5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(\"Short Answer training results:\")\n",
        "print(\"\\n{} answers out of {} possible, rate is {:.0f}%\".format(n_answers,len(source),100*n_answers/len(source)))\n",
        "if n_answers >0:\n",
        "  print(\"At least {} correct answers out of {} possible, rate is {:.0f}%\".format(n_correct,n_answers,100*n_correct/n_answers))\n",
        "if n_smlr >0:\n",
        "  print(\"There were {} similar answers with  similarity above {}%\\n\".format(n_smlr,smlrty_cut))"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}