Bert-Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      Weitergehende Sicherungsmaßnahmen können eine ...\n",
       "1      In der großen Kasernenanlage im Norden Kiels k...\n",
       "2      Premierminister David Lloyd George honorierte ...\n",
       "3      Der Beitrag der Truppen dieser Dominions währe...\n",
       "4      Eine Balance zwischen verschiedenen Lebensbere...\n",
       "                             ...                        \n",
       "895    Zwischen 1901 und 2010 ist er um ca 1,7 cm pro...\n",
       "896    Aus Furcht vor einem Bürgerkrieg wollte sie – ...\n",
       "897    In den meisten Politikfeldern gilt dafür seit ...\n",
       "898    Aufgrund der Wärmekapazität des Gesteins, und ...\n",
       "899    Die Klinge ist zumeist aus nicht rostfreiem Ko...\n",
       "Name: Sentence, Length: 900, dtype: object"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load data and write out sentence and target\n",
    "import pandas as pd\n",
    "\n",
    "loaded_set = pd.read_excel(\"Dataset/\"+\"training.xlsx\")\n",
    "loaded_set['Sentence']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[CLS]',\n",
       " 'Weiter',\n",
       " '##gehende',\n",
       " 'Sicherungs',\n",
       " '##maßnahmen',\n",
       " 'können',\n",
       " 'eine',\n",
       " 'Video',\n",
       " '##überwachung',\n",
       " 'und',\n",
       " 'eine',\n",
       " 'Zugangs',\n",
       " '##kontrolle',\n",
       " 'durch',\n",
       " 'einen',\n",
       " 'Tür',\n",
       " '##öff',\n",
       " '##ner',\n",
       " 'sein',\n",
       " ',',\n",
       " 'denn',\n",
       " 'viele',\n",
       " 'G',\n",
       " '##AA',\n",
       " 'befinden',\n",
       " 'sich',\n",
       " 'in',\n",
       " 'Vor',\n",
       " '##räumen',\n",
       " 'der',\n",
       " 'Geschäfts',\n",
       " '##stellen',\n",
       " 'der',\n",
       " 'Banken',\n",
       " ',',\n",
       " 'sodass',\n",
       " 'sie',\n",
       " 'auch',\n",
       " 'außerhalb',\n",
       " 'der',\n",
       " 'Schalter',\n",
       " '##öffnungs',\n",
       " '##zeiten',\n",
       " 'zugänglich',\n",
       " 'sind',\n",
       " '.',\n",
       " '[SEP]']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\r\n",
    "from transformers import AutoModel, AutoTokenizer\r\n",
    "# german tokens for bert\r\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"dbmdz/bert-base-german-cased\")\r\n",
    "#model = AutoModel.from_pretrained(\"dbmdz/bert-base-german-cased\")\r\n",
    "\r\n",
    "\r\n",
    "\r\n",
    "tokens_num=[]\r\n",
    "for sen in loaded_set['Sentence']:\r\n",
    "    tokenized = (tokenizer.tokenize(sen)) \r\n",
    "    tokens_num.append( ['[CLS]'] + tokenized + ['[SEP]']) \r\n",
    "    \r\n",
    "# get max_seq length    \r\n",
    "lens = [len(i) for i in tokens_num]\r\n",
    "max_seq_length = max(lens)\r\n",
    "max_seq_length = int(1.5*max_seq_length)\r\n",
    "#max_seq_length = 256\r\n",
    "tokens_num[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[102,\n",
       " 1784,\n",
       " 13183,\n",
       " 28847,\n",
       " 4686,\n",
       " 618,\n",
       " 261,\n",
       " 4770,\n",
       " 20815,\n",
       " 136,\n",
       " 261,\n",
       " 21093,\n",
       " 11600,\n",
       " 387,\n",
       " 397,\n",
       " 2451,\n",
       " 706,\n",
       " 432,\n",
       " 290,\n",
       " 818,\n",
       " 1398,\n",
       " 1358,\n",
       " 159,\n",
       " 10695,\n",
       " 3857,\n",
       " 251,\n",
       " 153,\n",
       " 445,\n",
       " 7721,\n",
       " 125,\n",
       " 2484,\n",
       " 984,\n",
       " 125,\n",
       " 8232,\n",
       " 818,\n",
       " 7415,\n",
       " 307,\n",
       " 313,\n",
       " 5729,\n",
       " 125,\n",
       " 28802,\n",
       " 17893,\n",
       " 4083,\n",
       " 10370,\n",
       " 341,\n",
       " 566,\n",
       " 103]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.convert_tokens_to_ids(tokens_num[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def manual_features(x):\r\n",
    "    letter_count = []\r\n",
    "    avarange_letter_per_word = []\r\n",
    "    num_words = []\r\n",
    "    num_letters_array = []\r\n",
    "    longest_word_length = []\r\n",
    "    shortest_word_length = []\r\n",
    "    genitiv = []\r\n",
    "    akkusativ = []\r\n",
    "    dativ = []\r\n",
    "    dass = []\r\n",
    "\r\n",
    "    for sen in x:\r\n",
    "        current_sen_split = sen.split()\r\n",
    "        num_words.append(len(current_sen_split))\r\n",
    "        num_letters = []\r\n",
    "            \r\n",
    "        if \"des\" in sen:\r\n",
    "            genitiv.append(1)\r\n",
    "        else:\r\n",
    "            genitiv.append(0)\r\n",
    "\r\n",
    "        if \"dem\" in sen:\r\n",
    "            akkusativ.append(1)\r\n",
    "        else:\r\n",
    "            akkusativ.append(0)\r\n",
    "\r\n",
    "        if \"den\" in sen:\r\n",
    "            dativ.append(1)\r\n",
    "        else:\r\n",
    "            dativ.append(0)\r\n",
    "\r\n",
    "        if \"dass\" in sen:\r\n",
    "            dass.append(1)\r\n",
    "        else:\r\n",
    "                dass.append(0)\r\n",
    "\r\n",
    "        for y in range(len(current_sen_split)):\r\n",
    "            current_word = current_sen_split[y]\r\n",
    "    \r\n",
    "            num_letters.append(len(current_word))\r\n",
    "    \r\n",
    "        current_lettercount = sum(num_letters)\r\n",
    "        letter_count.append(current_lettercount) \r\n",
    "        avarange_letter_per_word.append(current_lettercount/len(current_sen_split))\r\n",
    "        longest_word_length.append(max(num_letters)) \r\n",
    "        shortest_word_length.append(min(num_letters)) \r\n",
    "    \r\n",
    "\r\n",
    "\r\n",
    "\r\n",
    "    feature_dict = {\r\n",
    "        'dativ':dativ, \r\n",
    "        'akkusativ': akkusativ, \r\n",
    "        'genitiv': genitiv, \r\n",
    "        'dass': dass,\r\n",
    "        'num_words':num_words,\r\n",
    "        'letter_count':letter_count,\r\n",
    "        'avarange_letter_per_word':avarange_letter_per_word,\r\n",
    "        'longest_word_length':longest_word_length,\r\n",
    "        'shortest_word_length':shortest_word_length, \r\n",
    "        }\r\n",
    "\r\n",
    "    feature_dataframe = pd.DataFrame(data=feature_dict)\r\n",
    "    scaler = StandardScaler()\r\n",
    "\r\n",
    "    feature_dataframe[['num_words', 'longest_word_length', 'shortest_word_length', 'letter_count', 'avarange_letter_per_word']] = scaler.fit_transform(feature_dataframe[['num_words', 'longest_word_length', 'shortest_word_length', 'letter_count', 'avarange_letter_per_word']])\r\n",
    "\r\n",
    "    feature_dataframe[['num_words', 'longest_word_length', 'shortest_word_length', 'letter_count', 'avarange_letter_per_word']] = scaler.transform(feature_dataframe[['num_words', 'longest_word_length', 'shortest_word_length', 'letter_count', 'avarange_letter_per_word']])\r\n",
    "\r\n",
    "\r\n",
    "    tensorX = tf.constant(feature_dataframe.values)\r\n",
    "\r\n",
    "    return tensorX"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\r\n",
    "from sklearn.preprocessing import StandardScaler\r\n",
    "scaler = StandardScaler()\r\n",
    "def encode_names(n, tokenizer):\r\n",
    "   tokens = list(tokenizer.tokenize(n))\r\n",
    "   tokens.append('[SEP]')\r\n",
    "   return tokenizer.convert_tokens_to_ids(tokens)\r\n",
    "\r\n",
    "def bert_encode(string_list, tokenizer, max_seq_length):\r\n",
    "    num_examples = len(string_list)\r\n",
    "  \r\n",
    "\r\n",
    "\r\n",
    "    letter_count = []\r\n",
    "    avarange_letter_per_word = []\r\n",
    "    num_words = []\r\n",
    "    num_letters_array = []\r\n",
    "    longest_word_length = []\r\n",
    "    shortest_word_length = []\r\n",
    "    genitiv = []\r\n",
    "    akkusativ = []\r\n",
    "    dativ = []\r\n",
    "    dass = []\r\n",
    "\r\n",
    "    for sen in string_list:\r\n",
    "        current_sen_split = sen.split()\r\n",
    "        num_words.append(len(current_sen_split))\r\n",
    "        num_letters = []\r\n",
    "            \r\n",
    "        if \"des\" in sen:\r\n",
    "            genitiv.append(1)\r\n",
    "        else:\r\n",
    "            genitiv.append(0)\r\n",
    "\r\n",
    "        if \"dem\" in sen:\r\n",
    "            akkusativ.append(1)\r\n",
    "        else:\r\n",
    "            akkusativ.append(0)\r\n",
    "\r\n",
    "        if \"den\" in sen:\r\n",
    "            dativ.append(1)\r\n",
    "        else:\r\n",
    "            dativ.append(0)\r\n",
    "\r\n",
    "        if \"dass\" in sen:\r\n",
    "            dass.append(1)\r\n",
    "        else:\r\n",
    "                dass.append(0)\r\n",
    "\r\n",
    "        for y in range(len(current_sen_split)):\r\n",
    "            current_word = current_sen_split[y]\r\n",
    "    \r\n",
    "            num_letters.append(len(current_word))\r\n",
    "    \r\n",
    "        current_lettercount = sum(num_letters)\r\n",
    "        letter_count.append(current_lettercount) \r\n",
    "        avarange_letter_per_word.append(current_lettercount/len(current_sen_split))\r\n",
    "        longest_word_length.append(max(num_letters)) \r\n",
    "        shortest_word_length.append(min(num_letters)) \r\n",
    "    \r\n",
    "\r\n",
    "\r\n",
    "\r\n",
    "    feature_dict = {\r\n",
    "        'num_words':num_words,\r\n",
    "        'avarange_letter_per_word':avarange_letter_per_word,\r\n",
    "        'longest_word_length':longest_word_length,\r\n",
    "        }\r\n",
    "\r\n",
    "    feature_dataframe = pd.DataFrame(data=feature_dict)\r\n",
    "    scaler = StandardScaler()\r\n",
    "\r\n",
    "    feature_dataframe[['num_words', 'longest_word_length',  'avarange_letter_per_word']] = scaler.fit_transform(feature_dataframe[['num_words', 'longest_word_length', 'avarange_letter_per_word']])\r\n",
    "\r\n",
    "    feature_dataframe[['num_words', 'longest_word_length', 'avarange_letter_per_word']] = scaler.transform(feature_dataframe[['num_words', 'longest_word_length', 'avarange_letter_per_word']])\r\n",
    "\r\n",
    "\r\n",
    "    X_train_mF = tf.constant(feature_dataframe.values)\r\n",
    "\r\n",
    "  \r\n",
    "\r\n",
    "\r\n",
    "\r\n",
    "    string_tokens = tf.ragged.constant([\r\n",
    "      encode_names(n, tokenizer) for n in np.array(string_list)])\r\n",
    "\r\n",
    "    cls = [tokenizer.convert_tokens_to_ids(['[CLS]'])]*string_tokens.shape[0]\r\n",
    "    input_word_ids = tf.concat([cls, string_tokens], axis=-1)\r\n",
    "\r\n",
    "    input_mask = tf.ones_like(input_word_ids).to_tensor(shape=(None, max_seq_length))\r\n",
    "\r\n",
    "    type_cls = tf.zeros_like(cls)\r\n",
    "    type_tokens = tf.ones_like(string_tokens)\r\n",
    "    input_type_ids = tf.concat(\r\n",
    "      [type_cls, type_tokens], axis=-1).to_tensor(shape=(None, max_seq_length))\r\n",
    "    scaler_input_word_ids = scaler.fit_transform(input_type_ids)  \r\n",
    "\r\n",
    "    inputs = {\r\n",
    "      #'sc': scaler_input_word_ids,\r\n",
    "      #'input_word_ids': input_word_ids,\r\n",
    "      'input_word_ids': input_word_ids.to_tensor(shape=(None, max_seq_length)),\r\n",
    "      'input_mask': input_mask,\r\n",
    "      'input_type_ids': input_type_ids,\r\n",
    "      'X_train_mF': X_train_mF\r\n",
    "      }\r\n",
    "\r\n",
    "    return inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\r\n",
    "\r\n",
    "x = loaded_set['Sentence']\r\n",
    "y = loaded_set['MOS']\r\n",
    "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.20, random_state=32)\r\n",
    "y_train = round(y_train, 2)\r\n",
    "y_test = round(y_test, 2)\r\n",
    "\r\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Enabling eager execution\n",
      "INFO:tensorflow:Enabling v2 tensorshape\n",
      "INFO:tensorflow:Enabling resource variables\n",
      "INFO:tensorflow:Enabling tensor equality\n",
      "INFO:tensorflow:Enabling control flow v2\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\r\n",
    "X_train = bert_encode(x_train, tokenizer, max_seq_length)\r\n",
    "X_test = bert_encode(x_test, tokenizer, max_seq_length)\r\n",
    "\r\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:absl:Using C:\\Users\\phili\\AppData\\Local\\Temp\\tfhub_modules to cache modules.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow_hub as hub\r\n",
    "bert_layer = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/2\",\r\n",
    "                            trainable=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_size = 768\r\n",
    "max_seq_length = max_seq_length  #length of the tokenised tensor\r\n",
    "\r\n",
    "input_word_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\r\n",
    "                                       name=\"input_word_ids\")\r\n",
    "input_mask = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\r\n",
    "                                   name=\"input_mask\")\r\n",
    "segment_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\r\n",
    "                                    name=\"segment_ids\")\r\n",
    "\r\n",
    "X_train_mF = tf.keras.layers.Input(shape=(3,), dtype=tf.int32,\r\n",
    "                                    name=\"X_train_mF\")\r\n",
    "\r\n",
    "pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])   \r\n",
    "dropout = tf.keras.layers.Dropout(0.2)(pooled_output)\r\n",
    "reshaped_bert = tf.keras.layers.Reshape((6,128))(dropout)\r\n",
    "\r\n",
    "dense_mf_1 = tf.keras.layers.Dense(20)(X_train_mF)\r\n",
    "dense_mf_2 = tf.keras.layers.Dense(128)(dense_mf_1)\r\n",
    "#dense_mf_3 = tf.keras.layers.Dense(24)(dropout_mf)\r\n",
    "dropout_mf = tf.keras.layers.Dropout(0.3)(dense_mf_2)\r\n",
    "reshaped_mf = tf.keras.layers.Reshape((1,128))(dense_mf_2)\r\n",
    "\r\n",
    "#concatinated_3 = tf.concat([concatinated_1, concatinated_2 ], 1)\r\n",
    "#reshaped_mf = tf.keras.layers.Reshape((1,24))(dense_mf_3)\r\n",
    "\r\n",
    "concatinated = tf.concat([reshaped_bert, reshaped_mf], 1)\r\n",
    "\r\n",
    "gru_1_out = tf.keras.layers.GRU(200, return_sequences=True, activation='relu')(concatinated)\r\n",
    "gru_2_out = tf.keras.layers.GRU(100, return_sequences=True, activation='relu')(gru_1_out)\r\n",
    "\r\n",
    "flat = tf.keras.layers.Flatten()(gru_2_out)\r\n",
    "dropout_2 = tf.keras.layers.Dropout(0.3)(flat)\r\n",
    "dense_2 = tf.keras.layers.Dense(300)(dropout_2)\r\n",
    "dense_3 = tf.keras.layers.Dense(100)(dense_2)\r\n",
    "dense_4 = tf.keras.layers.Dense(50)(dense_3)\r\n",
    "\r\n",
    "pred = tf.keras.layers.Dense(1)(dense_2)\r\n",
    "     \r\n",
    "\r\n",
    "\r\n",
    "\r\n",
    "model = tf.keras.Model(\r\n",
    "    inputs={\r\n",
    "        'input_word_ids': input_word_ids,\r\n",
    "        'input_mask': input_mask,\r\n",
    "        'input_type_ids': segment_ids,\r\n",
    "        'X_train_mF':X_train_mF\r\n",
    "        },\r\n",
    "        outputs=pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_13\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_word_ids (InputLayer)     [(None, 144)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_mask (InputLayer)         [(None, 144)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "segment_ids (InputLayer)        [(None, 144)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "X_train_mF (InputLayer)         [(None, 3)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "keras_layer (KerasLayer)        [(None, 768), (None, 177853441   input_word_ids[0][0]             \n",
      "                                                                 input_mask[0][0]                 \n",
      "                                                                 segment_ids[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_61 (Dense)                (None, 20)           80          X_train_mF[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_39 (Dropout)            (None, 768)          0           keras_layer[13][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_62 (Dense)                (None, 128)          2688        dense_61[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "reshape_26 (Reshape)            (None, 6, 128)       0           dropout_39[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "reshape_27 (Reshape)            (None, 1, 128)       0           dense_62[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "tf.concat_13 (TFOpLambda)       (None, 7, 128)       0           reshape_26[0][0]                 \n",
      "                                                                 reshape_27[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "gru_22 (GRU)                    (None, 7, 200)       198000      tf.concat_13[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "gru_23 (GRU)                    (None, 7, 100)       90600       gru_22[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "flatten_13 (Flatten)            (None, 700)          0           gru_23[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dropout_41 (Dropout)            (None, 700)          0           flatten_13[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_63 (Dense)                (None, 300)          210300      dropout_41[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_66 (Dense)                (None, 1)            301         dense_63[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 178,355,410\n",
      "Trainable params: 501,969\n",
      "Non-trainable params: 177,853,441\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer= tf.keras.optimizers.Adam(0.001),\r\n",
    "              loss= \"mean_absolute_error\",\r\n",
    "              metrics= [\"mean_squared_error\"])\r\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "48/48 [==============================] - 131s 3s/step - loss: 1.1114 - mean_squared_error: 1.9650\n",
      "Epoch 2/50\n",
      "48/48 [==============================] - 125s 3s/step - loss: 0.9327 - mean_squared_error: 1.3308\n",
      "Epoch 3/50\n",
      "48/48 [==============================] - 129s 3s/step - loss: 0.8104 - mean_squared_error: 0.9939\n",
      "Epoch 4/50\n",
      "48/48 [==============================] - 126s 3s/step - loss: 0.7773 - mean_squared_error: 0.9162\n",
      "Epoch 5/50\n",
      "48/48 [==============================] - 120s 3s/step - loss: 0.7950 - mean_squared_error: 0.9642\n",
      "Epoch 6/50\n",
      "48/48 [==============================] - 117s 2s/step - loss: 0.7236 - mean_squared_error: 0.8378\n",
      "Epoch 7/50\n",
      "48/48 [==============================] - 120s 2s/step - loss: 0.7562 - mean_squared_error: 0.8821\n",
      "Epoch 8/50\n",
      "48/48 [==============================] - 119s 2s/step - loss: 0.7168 - mean_squared_error: 0.8207\n",
      "Epoch 9/50\n",
      "48/48 [==============================] - 123s 3s/step - loss: 0.6983 - mean_squared_error: 0.7799\n",
      "Epoch 10/50\n",
      "48/48 [==============================] - 125s 3s/step - loss: 0.6990 - mean_squared_error: 0.7690\n",
      "Epoch 11/50\n",
      "48/48 [==============================] - 124s 3s/step - loss: 0.6872 - mean_squared_error: 0.7229\n",
      "Epoch 12/50\n",
      "48/48 [==============================] - 120s 2s/step - loss: 0.6864 - mean_squared_error: 0.7431\n",
      "Epoch 13/50\n",
      "48/48 [==============================] - 119s 2s/step - loss: 0.6904 - mean_squared_error: 0.7499\n",
      "Epoch 14/50\n",
      "48/48 [==============================] - 118s 2s/step - loss: 0.6447 - mean_squared_error: 0.6648\n",
      "Epoch 15/50\n",
      "48/48 [==============================] - 119s 2s/step - loss: 0.6578 - mean_squared_error: 0.7066\n",
      "Epoch 16/50\n",
      "48/48 [==============================] - 120s 3s/step - loss: 0.6496 - mean_squared_error: 0.6580\n",
      "Epoch 17/50\n",
      "48/48 [==============================] - 119s 2s/step - loss: 0.6586 - mean_squared_error: 0.6766\n",
      "Epoch 18/50\n",
      "48/48 [==============================] - 119s 2s/step - loss: 0.6253 - mean_squared_error: 0.6247\n",
      "Epoch 19/50\n",
      "48/48 [==============================] - 108s 2s/step - loss: 0.5878 - mean_squared_error: 0.5620\n",
      "Epoch 20/50\n",
      "48/48 [==============================] - 107s 2s/step - loss: 0.6064 - mean_squared_error: 0.5986\n",
      "Epoch 21/50\n",
      "48/48 [==============================] - 108s 2s/step - loss: 0.6574 - mean_squared_error: 0.6864\n",
      "Epoch 22/50\n",
      "48/48 [==============================] - 108s 2s/step - loss: 0.6055 - mean_squared_error: 0.6027\n",
      "Epoch 23/50\n",
      "48/48 [==============================] - 108s 2s/step - loss: 0.6012 - mean_squared_error: 0.5816\n",
      "Epoch 24/50\n",
      "48/48 [==============================] - 108s 2s/step - loss: 0.6010 - mean_squared_error: 0.5996\n",
      "Epoch 25/50\n",
      "48/48 [==============================] - 113s 2s/step - loss: 0.5823 - mean_squared_error: 0.5467\n",
      "Epoch 26/50\n",
      "48/48 [==============================] - 115s 2s/step - loss: 0.5872 - mean_squared_error: 0.5605\n",
      "Epoch 27/50\n",
      "48/48 [==============================] - 113s 2s/step - loss: 0.6192 - mean_squared_error: 0.6304\n",
      "Epoch 28/50\n",
      "48/48 [==============================] - 115s 2s/step - loss: 0.6294 - mean_squared_error: 0.6239\n",
      "Epoch 29/50\n",
      "48/48 [==============================] - 114s 2s/step - loss: 0.5584 - mean_squared_error: 0.5142\n",
      "Epoch 30/50\n",
      "48/48 [==============================] - 114s 2s/step - loss: 0.5839 - mean_squared_error: 0.5680\n",
      "Epoch 31/50\n",
      "48/48 [==============================] - 113s 2s/step - loss: 0.5518 - mean_squared_error: 0.5112\n",
      "Epoch 32/50\n",
      "48/48 [==============================] - 112s 2s/step - loss: 0.5660 - mean_squared_error: 0.5212\n",
      "Epoch 33/50\n",
      "48/48 [==============================] - 111s 2s/step - loss: 0.5438 - mean_squared_error: 0.5015\n",
      "Epoch 34/50\n",
      "48/48 [==============================] - 111s 2s/step - loss: 0.5630 - mean_squared_error: 0.5459\n",
      "Epoch 35/50\n",
      "48/48 [==============================] - 112s 2s/step - loss: 0.5288 - mean_squared_error: 0.4852\n",
      "Epoch 36/50\n",
      "48/48 [==============================] - 114s 2s/step - loss: 0.5252 - mean_squared_error: 0.4758\n",
      "Epoch 37/50\n",
      "48/48 [==============================] - 116s 2s/step - loss: 0.5466 - mean_squared_error: 0.4966\n",
      "Epoch 38/50\n",
      "48/48 [==============================] - 114s 2s/step - loss: 0.5199 - mean_squared_error: 0.4665\n",
      "Epoch 39/50\n",
      "48/48 [==============================] - 115s 2s/step - loss: 0.4954 - mean_squared_error: 0.4215\n",
      "Epoch 40/50\n",
      "48/48 [==============================] - 114s 2s/step - loss: 0.5140 - mean_squared_error: 0.4403\n",
      "Epoch 41/50\n",
      "48/48 [==============================] - 119s 2s/step - loss: 0.5205 - mean_squared_error: 0.4563\n",
      "Epoch 42/50\n",
      "48/48 [==============================] - 114s 2s/step - loss: 0.5129 - mean_squared_error: 0.4345\n",
      "Epoch 43/50\n",
      "48/48 [==============================] - 113s 2s/step - loss: 0.5243 - mean_squared_error: 0.4710\n",
      "Epoch 44/50\n",
      "48/48 [==============================] - 114s 2s/step - loss: 0.5421 - mean_squared_error: 0.5013\n",
      "Epoch 45/50\n",
      "48/48 [==============================] - 113s 2s/step - loss: 0.4887 - mean_squared_error: 0.4161\n",
      "Epoch 46/50\n",
      "48/48 [==============================] - 115s 2s/step - loss: 0.5105 - mean_squared_error: 0.4430\n",
      "Epoch 47/50\n",
      "48/48 [==============================] - 116s 2s/step - loss: 0.4901 - mean_squared_error: 0.4222\n",
      "Epoch 48/50\n",
      "26/48 [===============>..............] - ETA: 54s - loss: 0.4655 - mean_squared_error: 0.3741"
     ]
    }
   ],
   "source": [
    "epochs =  50\r\n",
    "batch_size = 15\r\n",
    "\r\n",
    "model.fit(X_train, y_train.values, epochs=epochs, batch_size=batch_size)\r\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2.78],\n",
       "       [4.23],\n",
       "       [2.64],\n",
       "       [3.45],\n",
       "       [2.32],\n",
       "       [3.12],\n",
       "       [4.66],\n",
       "       [4.44],\n",
       "       [3.72],\n",
       "       [4.27],\n",
       "       [2.76],\n",
       "       [4.57],\n",
       "       [3.96],\n",
       "       [2.84],\n",
       "       [3.12],\n",
       "       [4.35],\n",
       "       [2.57],\n",
       "       [3.31],\n",
       "       [3.79],\n",
       "       [2.49],\n",
       "       [4.07],\n",
       "       [2.41],\n",
       "       [3.98],\n",
       "       [2.86],\n",
       "       [2.62],\n",
       "       [3.5 ],\n",
       "       [3.88],\n",
       "       [3.42],\n",
       "       [3.  ],\n",
       "       [2.57],\n",
       "       [2.61],\n",
       "       [3.6 ],\n",
       "       [4.22],\n",
       "       [4.52],\n",
       "       [3.42],\n",
       "       [3.47],\n",
       "       [3.74],\n",
       "       [4.83],\n",
       "       [4.47],\n",
       "       [4.97],\n",
       "       [3.31],\n",
       "       [3.11],\n",
       "       [3.17],\n",
       "       [2.59],\n",
       "       [3.91],\n",
       "       [4.13],\n",
       "       [3.47],\n",
       "       [3.31],\n",
       "       [3.62],\n",
       "       [3.91],\n",
       "       [5.07],\n",
       "       [4.02],\n",
       "       [2.89],\n",
       "       [3.52],\n",
       "       [3.03],\n",
       "       [1.72],\n",
       "       [2.87],\n",
       "       [3.55],\n",
       "       [2.36],\n",
       "       [4.07],\n",
       "       [2.99],\n",
       "       [2.75],\n",
       "       [3.38],\n",
       "       [4.67],\n",
       "       [4.56],\n",
       "       [3.36],\n",
       "       [3.21],\n",
       "       [4.04],\n",
       "       [4.46],\n",
       "       [3.34],\n",
       "       [3.29],\n",
       "       [4.61],\n",
       "       [4.92],\n",
       "       [2.29],\n",
       "       [3.22],\n",
       "       [3.49],\n",
       "       [1.63],\n",
       "       [2.63],\n",
       "       [2.86],\n",
       "       [2.27],\n",
       "       [2.53],\n",
       "       [3.15],\n",
       "       [2.38],\n",
       "       [2.28],\n",
       "       [3.2 ],\n",
       "       [2.97],\n",
       "       [3.41],\n",
       "       [3.55],\n",
       "       [2.51],\n",
       "       [4.66],\n",
       "       [4.67],\n",
       "       [1.94],\n",
       "       [1.24],\n",
       "       [2.73],\n",
       "       [3.63],\n",
       "       [3.33],\n",
       "       [2.24],\n",
       "       [3.89],\n",
       "       [4.96],\n",
       "       [4.15],\n",
       "       [4.4 ],\n",
       "       [2.78],\n",
       "       [3.72],\n",
       "       [3.15],\n",
       "       [2.04],\n",
       "       [2.79],\n",
       "       [4.08],\n",
       "       [3.69],\n",
       "       [2.83],\n",
       "       [4.28],\n",
       "       [3.58],\n",
       "       [3.13],\n",
       "       [3.14],\n",
       "       [4.9 ],\n",
       "       [3.48],\n",
       "       [3.48],\n",
       "       [4.01],\n",
       "       [3.23],\n",
       "       [3.77],\n",
       "       [4.3 ],\n",
       "       [4.11],\n",
       "       [3.92],\n",
       "       [3.09],\n",
       "       [3.61],\n",
       "       [4.78],\n",
       "       [3.7 ],\n",
       "       [3.74],\n",
       "       [2.44],\n",
       "       [4.33],\n",
       "       [3.87],\n",
       "       [3.92],\n",
       "       [4.1 ],\n",
       "       [3.4 ],\n",
       "       [3.67],\n",
       "       [3.44],\n",
       "       [3.98],\n",
       "       [4.08],\n",
       "       [4.53],\n",
       "       [3.38],\n",
       "       [4.36],\n",
       "       [3.14],\n",
       "       [3.88],\n",
       "       [4.06],\n",
       "       [4.42],\n",
       "       [3.21],\n",
       "       [2.42],\n",
       "       [3.  ],\n",
       "       [4.46],\n",
       "       [4.65],\n",
       "       [3.29],\n",
       "       [3.64],\n",
       "       [3.33],\n",
       "       [1.45],\n",
       "       [3.32],\n",
       "       [4.76],\n",
       "       [3.9 ],\n",
       "       [2.46],\n",
       "       [3.05],\n",
       "       [4.78],\n",
       "       [2.82],\n",
       "       [1.94],\n",
       "       [4.17],\n",
       "       [4.1 ],\n",
       "       [3.96],\n",
       "       [3.55],\n",
       "       [3.16],\n",
       "       [4.51],\n",
       "       [4.3 ],\n",
       "       [1.46],\n",
       "       [3.43],\n",
       "       [4.43],\n",
       "       [4.06],\n",
       "       [2.65],\n",
       "       [2.69],\n",
       "       [3.55],\n",
       "       [3.65],\n",
       "       [3.31],\n",
       "       [3.  ],\n",
       "       [2.77],\n",
       "       [3.63]], dtype=float32)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\r\n",
    "pred = model.predict(X_test)\r\n",
    "rounded_pred = np.around(pred, decimals=2)\r\n",
    "rounded_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.067631593325197"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def rmse(predictions, targets):\r\n",
    "    return np.sqrt(((predictions - targets) ** 2).mean())\r\n",
    "\r\n",
    "rmse(rounded_pred.transpose(), y_test.values)\r\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "4c2c6958f73f61d7b8bffd4193a0d9584c9ef39256921fe54ce430734ddbdc1d"
  },
  "kernelspec": {
   "display_name": "Python 3.9.4 64-bit ('QandU': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  },
  "orig_nbformat": 2
 },
 "nbformat": 4,
 "nbformat_minor": 2
}