dependency-parser/5.biaffine-attention-cross-entropy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://raw.githubusercontent.com/UniversalDependencies/UD_English-EWT/master/en_ewt-ud-dev.conllu\n",
    "# !wget https://raw.githubusercontent.com/UniversalDependencies/UD_English-EWT/master/en_ewt-ud-train.conllu\n",
    "# !wget https://raw.githubusercontent.com/UniversalDependencies/UD_English-EWT/master/en_ewt-ud-test.conllu\n",
    "# !pip install malaya -U"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import malaya\n",
    "import re\n",
    "from malaya.texts._text_functions import split_into_sentences\n",
    "from malaya.texts import _regex\n",
    "import numpy as np\n",
    "import itertools\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "tokenizer = malaya.preprocessing._tokenizer\n",
    "splitter = split_into_sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_number_regex(s):\n",
    "    if re.match(\"^\\d+?\\.\\d+?$\", s) is None:\n",
    "        return s.isdigit()\n",
    "    return True\n",
    "\n",
    "def preprocessing(w):\n",
    "    if is_number_regex(w):\n",
    "        return '<NUM>'\n",
    "    elif re.match(_regex._money, w):\n",
    "        return '<MONEY>'\n",
    "    elif re.match(_regex._date, w):\n",
    "        return '<DATE>'\n",
    "    elif re.match(_regex._expressions['email'], w):\n",
    "        return '<EMAIL>'\n",
    "    elif re.match(_regex._expressions['url'], w):\n",
    "        return '<URL>'\n",
    "    else:\n",
    "        w = ''.join(''.join(s)[:2] for _, s in itertools.groupby(w))\n",
    "        return w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'PAD': 0,\n",
       "  'UNK': 1,\n",
       "  '_ROOT': 2,\n",
       "  '<NUM>': 3,\n",
       "  '<MONEY>': 4,\n",
       "  '<DATE>': 5,\n",
       "  '<URL>': 6,\n",
       "  '<EMAIL>': 7},\n",
       " {'PAD': 0,\n",
       "  'UNK': 1,\n",
       "  '_ROOT': 2,\n",
       "  '<NUM>': 3,\n",
       "  '<MONEY>': 4,\n",
       "  '<DATE>': 5,\n",
       "  '<URL>': 6,\n",
       "  '<EMAIL>': 7})"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word2idx = {'PAD': 0,'UNK':1, '_ROOT': 2}\n",
    "tag2idx = {'PAD': 0, '_<ROOT>': 1}\n",
    "char2idx = {'PAD': 0,'UNK':1, '_ROOT': 2}\n",
    "word_idx = 3\n",
    "tag_idx = 2\n",
    "char_idx = 3\n",
    "\n",
    "special_tokens = ['<NUM>', '<MONEY>', '<DATE>', '<URL>', '<EMAIL>']\n",
    "\n",
    "for t in special_tokens:\n",
    "    word2idx[t] = word_idx\n",
    "    word_idx += 1\n",
    "    char2idx[t] = char_idx\n",
    "    char_idx += 1\n",
    "    \n",
    "word2idx, char2idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "PAD = \"_PAD\"\n",
    "PAD_POS = \"_PAD_POS\"\n",
    "PAD_TYPE = \"_<PAD>\"\n",
    "PAD_CHAR = \"_PAD_CHAR\"\n",
    "ROOT = \"_ROOT\"\n",
    "ROOT_POS = \"_ROOT_POS\"\n",
    "ROOT_TYPE = \"_<ROOT>\"\n",
    "ROOT_CHAR = \"_ROOT_CHAR\"\n",
    "END = \"_END\"\n",
    "END_POS = \"_END_POS\"\n",
    "END_TYPE = \"_<END>\"\n",
    "END_CHAR = \"_END_CHAR\"\n",
    "\n",
    "def process_corpus(corpus, until = None):\n",
    "    global word2idx, tag2idx, char2idx, word_idx, tag_idx, char_idx\n",
    "    sentences, words, depends, labels, pos, chars = [], [], [], [], [], []\n",
    "    temp_sentence, temp_word, temp_depend, temp_label, temp_pos = [], [], [], [], []\n",
    "    first_time = True\n",
    "    for sentence in corpus:\n",
    "        try:\n",
    "            if len(sentence):\n",
    "                if sentence[0] == '#':\n",
    "                    continue\n",
    "                if first_time:\n",
    "                    print(sentence)\n",
    "                    first_time = False\n",
    "                sentence = sentence.split('\\t')\n",
    "                for c in sentence[1]:\n",
    "                    if c not in char2idx:\n",
    "                        char2idx[c] = char_idx\n",
    "                        char_idx += 1\n",
    "                if sentence[7] not in tag2idx:\n",
    "                    tag2idx[sentence[7]] = tag_idx\n",
    "                    tag_idx += 1\n",
    "                sentence[1] = preprocessing(sentence[1])\n",
    "                if sentence[1] not in word2idx:\n",
    "                    word2idx[sentence[1]] = word_idx\n",
    "                    word_idx += 1\n",
    "                temp_word.append(word2idx[sentence[1]])\n",
    "                temp_depend.append(int(sentence[6]))\n",
    "                temp_label.append(tag2idx[sentence[7]])\n",
    "                temp_sentence.append(sentence[1])\n",
    "                temp_pos.append(sentence[3])\n",
    "            else:\n",
    "                if len(temp_sentence) < 2 or len(temp_word) != len(temp_label):\n",
    "                    temp_word = []\n",
    "                    temp_depend = []\n",
    "                    temp_label = []\n",
    "                    temp_sentence = []\n",
    "                    temp_pos = []\n",
    "                    continue\n",
    "                words.append(temp_word)\n",
    "                depends.append(temp_depend)\n",
    "                labels.append(temp_label)\n",
    "                sentences.append( temp_sentence)\n",
    "                pos.append(temp_pos)\n",
    "                char_ = [[char2idx['_ROOT']]]\n",
    "                for w in temp_sentence:\n",
    "                    if w in char2idx:\n",
    "                        char_.append([char2idx[w]])\n",
    "                    else:\n",
    "                        char_.append([char2idx[c] for c in w])\n",
    "                chars.append(char_)\n",
    "                temp_word = []\n",
    "                temp_depend = []\n",
    "                temp_label = []\n",
    "                temp_sentence = []\n",
    "                temp_pos = []\n",
    "        except Exception as e:\n",
    "            print(e, sentence)\n",
    "    return sentences[:-1], words[:-1], depends[:-1], labels[:-1], pos[:-1], chars[:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\tFrom\tfrom\tADP\tIN\t_\t3\tcase\t3:case\t_\n",
      "invalid literal for int() with base 10: '_' ['10.1', 'has', 'have', 'VERB', 'VBZ', '_', '_', '_', '8:parataxis', 'CopyOf=-1']\n",
      "invalid literal for int() with base 10: '_' ['21.1', 'has', 'have', 'VERB', 'VBZ', '_', '_', '_', '16:conj:and', 'CopyOf=-1']\n"
     ]
    }
   ],
   "source": [
    "with open('en_ewt-ud-dev.conllu') as fopen:\n",
    "    dev = fopen.read().split('\\n')\n",
    "\n",
    "sentences_dev, words_dev, depends_dev, labels_dev, _, _ = process_corpus(dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\tWhat\twhat\tPRON\tWP\tPronType=Int\t0\troot\t0:root\t_\n",
      "invalid literal for int() with base 10: '_' ['24.1', 'left', 'left', 'VERB', 'VBN', 'Tense=Past|VerbForm=Part', '_', '_', '6:parataxis', 'CopyOf=6']\n"
     ]
    }
   ],
   "source": [
    "with open('en_ewt-ud-test.conllu') as fopen:\n",
    "    test = fopen.read().split('\\n')\n",
    "\n",
    "sentences_test, words_test, depends_test, labels_test, _, _ = process_corpus(test)\n",
    "sentences_test.extend(sentences_dev)\n",
    "words_test.extend(words_dev)\n",
    "depends_test.extend(depends_dev)\n",
    "labels_test.extend(labels_dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\tAl\tAl\tPROPN\tNNP\tNumber=Sing\t0\troot\t0:root\tSpaceAfter=No\n",
      "invalid literal for int() with base 10: '_' ['8.1', 'reported', 'report', 'VERB', 'VBN', 'Tense=Past|VerbForm=Part|Voice=Pass', '_', '_', '5:conj:and', 'CopyOf=5']\n",
      "invalid literal for int() with base 10: '_' ['22.1', 'used', 'use', 'VERB', 'VBN', 'Tense=Past|VerbForm=Part', '_', '_', '13:advcl:with|17:conj:and', 'CopyOf=17']\n",
      "invalid literal for int() with base 10: '_' ['22.1', 'used', 'use', 'VERB', 'VBN', 'Tense=Past|VerbForm=Part', '_', '_', '13:advcl:with|17:conj:and', 'CopyOf=17']\n",
      "invalid literal for int() with base 10: '_' ['11.1', 'called', 'call', 'VERB', 'VBN', 'Tense=Past|VerbForm=Part|Voice=Pass', '_', '_', '3:conj:and', 'CopyOf=3']\n",
      "invalid literal for int() with base 10: '_' ['14.1', 'is', 'be', 'VERB', 'VBZ', '_', '_', '_', '1:conj:and', 'CopyOf=1']\n",
      "invalid literal for int() with base 10: '_' ['20.1', 'reflect', 'reflect', 'VERB', 'VBP', 'Mood=Ind|Tense=Pres|VerbForm=Fin', '_', '_', '7:acl:relcl|9:conj', 'CopyOf=9']\n",
      "invalid literal for int() with base 10: '_' ['21.1', 'recruited', 'recruit', 'VERB', 'VBD', 'Mood=Ind|Tense=Past|VerbForm=Fin', '_', '_', '9:conj:and', 'CopyOf=9']\n",
      "invalid literal for int() with base 10: '_' ['9.1', 'wish', 'wish', 'VERB', 'VBP', 'Mood=Ind|Tense=Pres|VerbForm=Fin', '_', '_', '2:conj:and', 'CopyOf=2']\n",
      "invalid literal for int() with base 10: '_' ['38.1', 'supplied', 'supply', 'VERB', 'VBN', 'Tense=Past|VerbForm=Part|Voice=Pass', '_', '_', '16:conj:and', 'CopyOf=16']\n",
      "invalid literal for int() with base 10: '_' ['18.1', 'keep', 'keep', 'VERB', 'VB', 'Mood=Imp|VerbForm=Fin', '_', '_', '14:conj:and', 'CopyOf=14']\n",
      "invalid literal for int() with base 10: '_' ['21.1', 'keep', 'keep', 'VERB', 'VB', 'Mood=Imp|VerbForm=Fin', '_', '_', '14:conj:and', 'CopyOf=14']\n",
      "invalid literal for int() with base 10: '_' ['18.1', 'mean', 'mean', 'VERB', 'VB', 'VerbForm=Inf', '_', '_', '8:conj', 'CopyOf=8']\n",
      "invalid literal for int() with base 10: '_' ['30.1', 'play', 'play', 'VERB', 'VBP', 'Mood=Ind|Tense=Pres|VerbForm=Fin', '_', '_', '18:acl:relcl|27:conj:but', 'CopyOf=27']\n",
      "invalid literal for int() with base 10: '_' ['22.1', 'have', 'have', 'VERB', 'VBP', 'Mood=Ind|Tense=Pres|VerbForm=Fin', '_', '_', '17:conj', 'CopyOf=17']\n",
      "invalid literal for int() with base 10: '_' ['27.1', 'have', 'have', 'VERB', 'VBP', 'Mood=Ind|Tense=Pres|VerbForm=Fin', '_', '_', '17:conj', 'CopyOf=17']\n",
      "invalid literal for int() with base 10: '_' ['49.1', 'helped', 'help', 'VERB', 'VBD', '_', '_', '_', '38:conj:but', 'CopyOf=38']\n",
      "invalid literal for int() with base 10: '_' ['7.1', 'found', 'find', 'VERB', 'VBD', 'Mood=Ind|Tense=Past|VerbForm=Fin', '_', '_', '3:conj', 'CopyOf=3']\n",
      "invalid literal for int() with base 10: '_' ['10.1', 'excited', 'excited', 'ADJ', 'JJ', 'Degree=Pos', '_', '_', '4:advcl', 'CopyOf=4']\n",
      "invalid literal for int() with base 10: '_' ['15.1', \"'s\", 'be', 'VERB', 'VBZ', '_', '_', '_', '2:conj:and', 'CopyOf=2']\n",
      "invalid literal for int() with base 10: '_' ['25.1', 'took', 'take', 'VERB', 'VBD', 'Mood=Ind|Tense=Past|VerbForm=Fin', '_', '_', '17:conj:and', 'CopyOf=17']\n",
      "invalid literal for int() with base 10: '_' ['10.1', 'loss', 'lose', 'VERB', 'VBD', 'Mood=Ind|Tense=Past|VerbForm=Fin', '_', '_', '3:conj:and', 'CopyOf=3']\n",
      "invalid literal for int() with base 10: '_' ['11.1', 'leave', 'leave', 'VERB', 'VB', 'VerbForm=Inf', '_', '_', '7:parataxis', 'CopyOf=7']\n",
      "invalid literal for int() with base 10: '_' ['24.1', 'charge', 'charge', 'VERB', 'VBP', 'Mood=Ind|Tense=Pres|VerbForm=Fin', '_', '_', '16:conj:and', 'CopyOf=16']\n"
     ]
    }
   ],
   "source": [
    "with open('en_ewt-ud-train.conllu') as fopen:\n",
    "    train = fopen.read().split('\\n')\n",
    "\n",
    "sentences_train, words_train, depends_train, labels_train, _, _ = process_corpus(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(12000, 3824)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sentences_train), len(sentences_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21974"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx2word = {v:k for k, v in word2idx.items()}\n",
    "idx2tag = {v:k for k, v in tag2idx.items()}\n",
    "len(idx2word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_char_seq(batch, UNK = 2):\n",
    "    maxlen_c = max([len(k) for k in batch])\n",
    "    x = [[len(i) for i in k] for k in batch]\n",
    "    maxlen = max([j for i in x for j in i])\n",
    "    temp = np.zeros((len(batch),maxlen_c,maxlen),dtype=np.int32)\n",
    "    for i in range(len(batch)):\n",
    "        for k in range(len(batch[i])):\n",
    "            for no, c in enumerate(batch[i][k]):\n",
    "                temp[i,k,-1-no] = char2idx.get(c, UNK)\n",
    "    return temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X = words_train\n",
    "train_Y = labels_train\n",
    "train_depends = depends_train\n",
    "train_char = sentences_train\n",
    "\n",
    "test_X = words_test\n",
    "test_Y = labels_test\n",
    "test_depends = depends_test\n",
    "test_char = sentences_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BiAAttention:\n",
    "    def __init__(self, input_size_encoder, input_size_decoder, num_labels):\n",
    "        self.input_size_encoder = input_size_encoder\n",
    "        self.input_size_decoder = input_size_decoder\n",
    "        self.num_labels = num_labels\n",
    "        \n",
    "        self.W_d = tf.get_variable(\"W_d\", shape=[self.num_labels, self.input_size_decoder],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.W_e = tf.get_variable(\"W_e\", shape=[self.num_labels, self.input_size_encoder],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.U = tf.get_variable(\"U\", shape=[self.num_labels, self.input_size_decoder, self.input_size_encoder],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        \n",
    "    def forward(self, input_d, input_e, mask_d=None, mask_e=None):\n",
    "        batch = tf.shape(input_d)[0]\n",
    "        length_decoder = tf.shape(input_d)[1]\n",
    "        length_encoder = tf.shape(input_e)[1]\n",
    "        out_d = tf.expand_dims(tf.matmul(self.W_d, tf.transpose(input_d, [0, 2, 1])), 3)\n",
    "        out_e = tf.expand_dims(tf.matmul(self.W_e, tf.transpose(input_e, [0, 2, 1])), 2)\n",
    "        output = tf.matmul(tf.expand_dims(input_d, 1), self.U)\n",
    "        output = tf.matmul(output, tf.transpose(tf.expand_dims(input_e, 1), [0, 1, 3, 2]))\n",
    "        \n",
    "        output = output + out_d + out_e\n",
    "        \n",
    "        if mask_d is not None:\n",
    "            d = tf.expand_dims(tf.expand_dims(mask_d, 1), 3)\n",
    "            e = tf.expand_dims(tf.expand_dims(mask_e, 1), 2)\n",
    "            output = output * d * e\n",
    "            \n",
    "        return output\n",
    "    \n",
    "class BiLinear:\n",
    "    def __init__(self, left_features, right_features, out_features):\n",
    "        self.left_features = left_features\n",
    "        self.right_features = right_features\n",
    "        self.out_features = out_features\n",
    "        \n",
    "        self.U = tf.get_variable(\"U-bi\", shape=[out_features, left_features, right_features],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.W_l = tf.get_variable(\"Wl\", shape=[out_features, left_features],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.W_r = tf.get_variable(\"Wr\", shape=[out_features, right_features],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    \n",
    "    def forward(self, input_left, input_right):\n",
    "        left_size = tf.shape(input_left)\n",
    "        output_shape = tf.concat([left_size[:-1], [self.out_features]], axis = 0)\n",
    "        batch = tf.cast(tf.reduce_prod(left_size[:-1]), tf.int32)\n",
    "        input_left = tf.reshape(input_left, (batch, self.left_features))\n",
    "        input_right = tf.reshape(input_right, (batch, self.right_features))\n",
    "        tiled = tf.tile(tf.expand_dims(input_left, axis = 0), (self.out_features,1,1))\n",
    "        output = tf.transpose(tf.reduce_sum(tf.matmul(tiled, self.U), axis = 2))\n",
    "        output = output + tf.matmul(input_left, tf.transpose(self.W_l))\\\n",
    "        + tf.matmul(input_right, tf.transpose(self.W_r))\n",
    "        \n",
    "        return tf.reshape(output, output_shape)\n",
    "\n",
    "class Attention:\n",
    "    def __init__(self, word_dim, num_words, char_dim, num_chars, num_filters, kernel_size,\n",
    "                 hidden_size, encoder_layers, num_labels, arc_space, type_space):\n",
    "        \n",
    "        def cells(size, reuse=False):\n",
    "            return tf.nn.rnn_cell.LSTMCell(size,\n",
    "                                           initializer=tf.orthogonal_initializer(),reuse=reuse)\n",
    "        \n",
    "        self.word_embedd = tf.Variable(tf.random_uniform([num_words, word_dim], -1, 1))\n",
    "        self.char_embedd = tf.Variable(tf.random_uniform([num_chars, char_dim], -1, 1))\n",
    "        self.conv1d = tf.layers.Conv1D(num_filters, kernel_size, 1, padding='VALID')\n",
    "        self.num_labels = num_labels\n",
    "        self.encoder = tf.nn.rnn_cell.MultiRNNCell([cells(hidden_size) for _ in range(encoder_layers)])\n",
    "\n",
    "        \n",
    "        \n",
    "    def encode(self, input_word, input_char):\n",
    "        word = tf.nn.embedding_lookup(self.word_embedd, input_word)\n",
    "        char = tf.nn.embedding_lookup(self.char_embedd, input_char)\n",
    "        b = tf.shape(char)[0]\n",
    "        wl = tf.shape(char)[1]\n",
    "        cl = tf.shape(char)[2]\n",
    "        d = char.shape[3]\n",
    "        char = tf.reshape(char, [b * wl, cl, d])\n",
    "        char = tf.reduce_max(self.conv1d(char), axis = 1)\n",
    "        char = tf.nn.tanh(char)\n",
    "        d = char.shape[-1]\n",
    "        char = tf.reshape(char, [b, wl, d])\n",
    "        \n",
    "        src_encoding = tf.concat([word, char], axis=2)\n",
    "        output, hn = tf.nn.dynamic_rnn(self.encoder, src_encoding, dtype = tf.float32,\n",
    "                                      scope = 'encoder')\n",
    "        arc_h = tf.nn.elu(self.arc_h(output))\n",
    "        arc_c = tf.nn.elu(self.arc_c(output))\n",
    "        \n",
    "        type_h = tf.nn.elu(self.type_h(output))\n",
    "        type_c = tf.nn.elu(self.type_c(output))\n",
    "        \n",
    "        return (arc_h, arc_c), (type_h, type_c), hn\n",
    "    \n",
    "    def forward(self, input_word, input_char, mask):\n",
    "        arcs, types, _ = self.encode(input_word, input_char)\n",
    "        \n",
    "        out_arc = tf.squeeze(self.attention.forward(arcs[0], arcs[1], mask_d=mask, mask_e=mask), axis = 1)\n",
    "        return out_arc, types, mask\n",
    "    \n",
    "    def loss(self, input_word, input_char, mask, heads, types):\n",
    "        out_arc, out_type, _ = self.forward(input_word, input_char, mask)\n",
    "        type_h, type_c = out_type\n",
    "        batch = tf.shape(out_arc)[0]\n",
    "        max_len = tf.shape(out_arc)[1]\n",
    "        batch_index = tf.range(0, batch)\n",
    "        t = tf.transpose(heads)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0), \n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        type_h = tf.gather_nd(type_h, concatenated)\n",
    "        out_type = self.bilinear.forward(type_h, type_c)\n",
    "        minus_inf = -1e8\n",
    "        minus_mask = (1 - mask) * minus_inf\n",
    "        out_arc = out_arc + tf.expand_dims(minus_mask, axis = 2) + tf.expand_dims(minus_mask, axis = 1)\n",
    "        loss_arc = tf.nn.log_softmax(out_arc, dim=1)\n",
    "        loss_type = tf.nn.log_softmax(out_type, dim=2)\n",
    "        loss_arc = loss_arc * tf.expand_dims(mask, axis = 2) * tf.expand_dims(mask, axis = 1)\n",
    "        loss_type = loss_type * tf.expand_dims(mask, axis = 2)\n",
    "        num = tf.reduce_sum(mask) - tf.cast(batch, tf.float32)\n",
    "        child_index = tf.tile(tf.expand_dims(tf.range(0, max_len), 1), [1, batch])\n",
    "        t = tf.transpose(heads)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0),\n",
    "                                               tf.expand_dims(t, axis = 0),\n",
    "                                               tf.expand_dims(child_index, axis = 0)], axis = 0))\n",
    "        loss_arc = tf.gather_nd(loss_arc, concatenated)\n",
    "        loss_arc = tf.transpose(loss_arc, [1, 0])\n",
    "        \n",
    "        t = tf.transpose(types)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0),\n",
    "                                               tf.expand_dims(child_index, axis = 0),\n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        loss_type = tf.gather_nd(loss_type, concatenated)\n",
    "        loss_type = tf.transpose(loss_type, [1, 0])\n",
    "        return tf.reduce_sum(-loss_arc) / num, tf.reduce_sum(-loss_type) / num\n",
    "    \n",
    "    def decode(self, input_word, input_char, mask, leading_symbolic=0):\n",
    "        out_arc, out_type, _ = self.forward(input_word, input_char, mask)\n",
    "        batch = tf.shape(out_arc)[0]\n",
    "        max_len = tf.shape(out_arc)[1]\n",
    "        sec_max_len = tf.shape(out_arc)[2]\n",
    "        out_arc = out_arc + tf.linalg.diag(tf.fill([max_len], -np.inf))\n",
    "        minus_mask = tf.expand_dims(tf.cast(1 - mask, tf.bool), axis = 2)\n",
    "        minus_mask = tf.tile(minus_mask, [1, 1, sec_max_len])\n",
    "        out_arc = tf.where(minus_mask, tf.fill(tf.shape(out_arc), -np.inf), out_arc)\n",
    "        heads = tf.argmax(out_arc, axis = 1)\n",
    "        type_h, type_c = out_type\n",
    "        batch = tf.shape(type_h)[0]\n",
    "        max_len = tf.shape(type_h)[1]\n",
    "        batch_index = tf.range(0, batch)\n",
    "        t = tf.cast(tf.transpose(heads), tf.int32)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0), \n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        type_h = tf.gather_nd(type_h, concatenated)\n",
    "        out_type = self.bilinear.forward(type_h, type_c)\n",
    "        out_type = out_type[:, :, leading_symbolic:]\n",
    "        types = tf.argmax(out_type, axis = 2)\n",
    "        return heads, types\n",
    "    \n",
    "class Model:\n",
    "    def __init__(\n",
    "        self, \n",
    "        dim_word,\n",
    "        dim_char,\n",
    "        dropout,\n",
    "        learning_rate,\n",
    "        hidden_size_char,\n",
    "        hidden_size_word,\n",
    "        num_layers,\n",
    "        cov = 0.0):\n",
    "        \n",
    "        def cells(size, reuse = False):\n",
    "            return tf.contrib.rnn.DropoutWrapper(\n",
    "                tf.nn.rnn_cell.LSTMCell(\n",
    "                    size,\n",
    "                    initializer = tf.orthogonal_initializer(),\n",
    "                    reuse = reuse,\n",
    "                ),\n",
    "                output_keep_prob = dropout,\n",
    "            )\n",
    "        \n",
    "        self.words = tf.placeholder(tf.int32, (None, None))\n",
    "        self.chars = tf.placeholder(tf.int32, (None, None, None))\n",
    "        self.heads = tf.placeholder(tf.int32, (None, None))\n",
    "        self.types = tf.placeholder(tf.int32, (None, None))\n",
    "        self.mask = tf.cast(tf.math.not_equal(self.words, 0), tf.float32)\n",
    "        self.maxlen = tf.shape(self.words)[1]\n",
    "        self.lengths = tf.count_nonzero(self.words, 1)\n",
    "        mask = self.mask\n",
    "        heads = self.heads\n",
    "        types = self.types\n",
    "        \n",
    "        self.arc_h = tf.layers.Dense(hidden_size_word)\n",
    "        self.arc_c = tf.layers.Dense(hidden_size_word)\n",
    "        self.attention = BiAAttention(hidden_size_word, hidden_size_word, 1)\n",
    "\n",
    "        self.type_h = tf.layers.Dense(hidden_size_word)\n",
    "        self.type_c = tf.layers.Dense(hidden_size_word)\n",
    "        self.bilinear = BiLinear(hidden_size_word, hidden_size_word, len(tag2idx))\n",
    "        \n",
    "        self.word_embeddings = tf.Variable(\n",
    "            tf.truncated_normal(\n",
    "                [len(word2idx), dim_word], stddev = 1.0 / np.sqrt(dim_word)\n",
    "            )\n",
    "        )\n",
    "        self.char_embeddings = tf.Variable(\n",
    "            tf.truncated_normal(\n",
    "                [len(char2idx), dim_char], stddev = 1.0 / np.sqrt(dim_char)\n",
    "            )\n",
    "        )\n",
    "\n",
    "        word_embedded = tf.nn.embedding_lookup(\n",
    "            self.word_embeddings, self.words\n",
    "        )\n",
    "        char_embedded = tf.nn.embedding_lookup(\n",
    "            self.char_embeddings, self.chars\n",
    "        )\n",
    "        s = tf.shape(char_embedded)\n",
    "        char_embedded = tf.reshape(\n",
    "            char_embedded, shape = [s[0] * s[1], s[-2], dim_char]\n",
    "        )\n",
    "\n",
    "        for n in range(num_layers):\n",
    "            (out_fw, out_bw), (\n",
    "                state_fw,\n",
    "                state_bw,\n",
    "            ) = tf.nn.bidirectional_dynamic_rnn(\n",
    "                cell_fw = cells(hidden_size_char),\n",
    "                cell_bw = cells(hidden_size_char),\n",
    "                inputs = char_embedded,\n",
    "                dtype = tf.float32,\n",
    "                scope = 'bidirectional_rnn_char_%d' % (n),\n",
    "            )\n",
    "            char_embedded = tf.concat((out_fw, out_bw), 2)\n",
    "        output = tf.reshape(\n",
    "            char_embedded[:, -1], shape = [s[0], s[1], 2 * hidden_size_char]\n",
    "        )\n",
    "        word_embedded = tf.concat([word_embedded, output], axis = -1)\n",
    "\n",
    "        for n in range(num_layers):\n",
    "            (out_fw, out_bw), (\n",
    "                state_fw,\n",
    "                state_bw,\n",
    "            ) = tf.nn.bidirectional_dynamic_rnn(\n",
    "                cell_fw = cells(hidden_size_word),\n",
    "                cell_bw = cells(hidden_size_word),\n",
    "                inputs = word_embedded,\n",
    "                dtype = tf.float32,\n",
    "                scope = 'bidirectional_rnn_word_%d' % (n),\n",
    "            )\n",
    "            word_embedded = tf.concat((out_fw, out_bw), 2)\n",
    "        \n",
    "        \n",
    "        arc_h = tf.nn.elu(self.arc_h(word_embedded))\n",
    "        arc_c = tf.nn.elu(self.arc_c(word_embedded))\n",
    "        \n",
    "        type_h = tf.nn.elu(self.type_h(word_embedded))\n",
    "        type_c = tf.nn.elu(self.type_c(word_embedded))\n",
    "        \n",
    "        out_arc = tf.squeeze(self.attention.forward(arc_h, arc_h, mask_d=self.mask, \n",
    "                                                    mask_e=self.mask), axis = 1)\n",
    "        \n",
    "        batch = tf.shape(out_arc)[0]\n",
    "        max_len = tf.shape(out_arc)[1]\n",
    "        sec_max_len = tf.shape(out_arc)[2]\n",
    "        batch_index = tf.range(0, batch)\n",
    "        \n",
    "        decode_arc = out_arc + tf.linalg.diag(tf.fill([max_len], -np.inf))\n",
    "        minus_mask = tf.expand_dims(tf.cast(1 - mask, tf.bool), axis = 2)\n",
    "        minus_mask = tf.tile(minus_mask, [1, 1, sec_max_len])\n",
    "        decode_arc = tf.where(minus_mask, tf.fill(tf.shape(decode_arc), -np.inf), decode_arc)\n",
    "        self.heads_seq = tf.argmax(decode_arc, axis = 1)\n",
    "        \n",
    "        t = tf.cast(tf.transpose(self.heads_seq), tf.int32)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0), \n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        type_h = tf.gather_nd(type_h, concatenated)\n",
    "        out_type = self.bilinear.forward(type_h, type_c)\n",
    "        self.tags_seq = tf.argmax(out_type, axis = 2)\n",
    "        \n",
    "        batch = tf.shape(out_arc)[0]\n",
    "        max_len = tf.shape(out_arc)[1]\n",
    "        batch_index = tf.range(0, batch)\n",
    "        t = tf.transpose(heads)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0), \n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        type_h = tf.gather_nd(type_h, concatenated)\n",
    "        out_type = self.bilinear.forward(type_h, type_c)\n",
    "        minus_inf = -1e8\n",
    "        minus_mask = (1 - mask) * minus_inf\n",
    "        out_arc = out_arc + tf.expand_dims(minus_mask, axis = 2) + tf.expand_dims(minus_mask, axis = 1)\n",
    "        loss_arc = tf.nn.log_softmax(out_arc, dim=1)\n",
    "        loss_type = tf.nn.log_softmax(out_type, dim=2)\n",
    "        loss_arc = loss_arc * tf.expand_dims(mask, axis = 2) * tf.expand_dims(mask, axis = 1)\n",
    "        loss_type = loss_type * tf.expand_dims(mask, axis = 2)\n",
    "        num = tf.reduce_sum(mask) - tf.cast(batch, tf.float32)\n",
    "        child_index = tf.tile(tf.expand_dims(tf.range(0, max_len), 1), [1, batch])\n",
    "        t = tf.transpose(heads)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0),\n",
    "                                               tf.expand_dims(t, axis = 0),\n",
    "                                               tf.expand_dims(child_index, axis = 0)], axis = 0))\n",
    "        loss_arc = tf.gather_nd(loss_arc, concatenated)\n",
    "        loss_arc = tf.transpose(loss_arc, [1, 0])\n",
    "        \n",
    "        t = tf.transpose(types)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0),\n",
    "                                               tf.expand_dims(child_index, axis = 0),\n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        loss_type = tf.gather_nd(loss_type, concatenated)\n",
    "        loss_type = tf.transpose(loss_type, [1, 0])\n",
    "        self.cost = (tf.reduce_sum(-loss_arc) / num) + (tf.reduce_sum(-loss_type) / num)\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        \n",
    "        mask = tf.sequence_mask(self.lengths, maxlen = self.maxlen)\n",
    "        \n",
    "        self.prediction = tf.boolean_mask(self.tags_seq, mask)\n",
    "        mask_label = tf.boolean_mask(self.types, mask)\n",
    "        correct_pred = tf.equal(tf.cast(self.prediction, tf.int32), mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "        \n",
    "        self.prediction = tf.cast(tf.boolean_mask(self.heads_seq, mask), tf.int32)\n",
    "        mask_label = tf.boolean_mask(self.heads, mask)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy_depends = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "reduction_indices is deprecated, use axis instead\n",
      "WARNING:tensorflow:From <ipython-input-14-f4c58ffdfb95>:183: LSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:From <ipython-input-14-f4c58ffdfb95>:238: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow/python/ops/rnn.py:464: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow/python/ops/rnn_cell_impl.py:961: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From <ipython-input-14-f4c58ffdfb95>:277: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From <ipython-input-14-f4c58ffdfb95>:300: calling log_softmax (from tensorflow.python.ops.nn_ops) with dim is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "dim is deprecated, use axis instead\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "dim_word = 128\n",
    "dim_char = 256\n",
    "dropout = 1.0\n",
    "learning_rate = 1e-3\n",
    "hidden_size_char = 128\n",
    "hidden_size_word = 128\n",
    "num_layers = 2\n",
    "\n",
    "model = Model(dim_word,dim_char,dropout,learning_rate,hidden_size_char,hidden_size_word,num_layers)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_x = train_X[:5]\n",
    "batch_x = pad_sequences(batch_x,padding='post')\n",
    "batch_char = train_char[:5]\n",
    "batch_char = generate_char_seq(batch_char)\n",
    "batch_y = train_Y[:5]\n",
    "batch_y = pad_sequences(batch_y,padding='post')\n",
    "batch_depends = train_depends[:5]\n",
    "batch_depends = pad_sequences(batch_depends,padding='post')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.0, 0.05172414, 7.4798884]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run([model.accuracy, model.accuracy_depends, model.cost],\n",
    "        feed_dict = {model.words: batch_x,\n",
    "                model.chars: batch_char,\n",
    "                model.types: batch_y,\n",
    "                model.heads: batch_depends})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([19, 19, 19, 23, 23, 19, 19, 19, 19, 23, 23, 23, 23, 23, 23, 23, 23,\n",
       "        23, 23, 23, 17, 17, 17, 17, 35, 35, 35, 43, 43, 43, 43, 43, 43, 43,\n",
       "        35, 35]),\n",
       " array([2, 2, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "        2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0]),\n",
       " array([ 0,  1,  1,  1,  6,  7,  1,  7,  8,  8,  8,  8,  8, 15,  8, 18, 18,\n",
       "         7, 21, 21, 18, 23, 21, 21, 28, 28, 28, 21,  1,  0,  0,  0,  0,  0,\n",
       "         0,  0], dtype=int32))"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags_seq, heads = sess.run(\n",
    "    [model.tags_seq, model.heads_seq],\n",
    "    feed_dict = {\n",
    "        model.words: batch_x,\n",
    "        model.chars: batch_char\n",
    "    },\n",
    ")\n",
    "tags_seq[0], heads[0], batch_depends[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 375/375 [01:07<00:00,  5.53it/s, accuracy=0.675, accuracy_depends=0.664, cost=2]    \n",
      "test minibatch loop: 100%|██████████| 120/120 [00:08<00:00, 14.80it/s, accuracy=0.688, accuracy_depends=0.66, cost=1.91] \n",
      "train minibatch loop:   0%|          | 1/375 [00:00<01:09,  5.39it/s, accuracy=0.641, accuracy_depends=0.606, cost=2.29]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, training loss: 4.134478, training acc: 0.409135, training depends: 0.386963, valid loss: 2.286293, valid acc: 0.647242, valid depends: 0.598646\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 375/375 [01:07<00:00,  5.57it/s, accuracy=0.782, accuracy_depends=0.767, cost=1.27] \n",
      "test minibatch loop: 100%|██████████| 120/120 [00:08<00:00, 14.59it/s, accuracy=0.789, accuracy_depends=0.761, cost=1.13]\n",
      "train minibatch loop:   0%|          | 1/375 [00:00<01:11,  5.20it/s, accuracy=0.752, accuracy_depends=0.759, cost=1.31]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, training loss: 1.745138, training acc: 0.708187, training depends: 0.671352, valid loss: 1.594311, valid acc: 0.737908, valid depends: 0.680541\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 375/375 [01:07<00:00,  5.59it/s, accuracy=0.796, accuracy_depends=0.802, cost=1.02] \n",
      "test minibatch loop: 100%|██████████| 120/120 [00:08<00:00, 14.77it/s, accuracy=0.818, accuracy_depends=0.806, cost=0.888]\n",
      "train minibatch loop:   0%|          | 0/375 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 2, training loss: 1.160756, training acc: 0.782296, training depends: 0.746494, valid loss: 1.441877, valid acc: 0.755935, valid depends: 0.702582\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 375/375 [01:07<00:00,  5.55it/s, accuracy=0.84, accuracy_depends=0.818, cost=0.81]  \n",
      "test minibatch loop: 100%|██████████| 120/120 [00:08<00:00, 14.75it/s, accuracy=0.834, accuracy_depends=0.814, cost=0.777]\n",
      "train minibatch loop:   0%|          | 1/375 [00:00<01:11,  5.27it/s, accuracy=0.842, accuracy_depends=0.842, cost=0.637]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 3, training loss: 0.858341, training acc: 0.820096, training depends: 0.786564, valid loss: 1.435897, valid acc: 0.776457, valid depends: 0.710300\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 375/375 [01:06<00:00,  5.60it/s, accuracy=0.84, accuracy_depends=0.844, cost=0.666] \n",
      "test minibatch loop: 100%|██████████| 120/120 [00:07<00:00, 15.36it/s, accuracy=0.85, accuracy_depends=0.814, cost=0.692]\n",
      "train minibatch loop:   0%|          | 1/375 [00:00<01:08,  5.45it/s, accuracy=0.872, accuracy_depends=0.877, cost=0.448]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 4, training loss: 0.668751, training acc: 0.842080, training depends: 0.813836, valid loss: 1.501069, valid acc: 0.782813, valid depends: 0.712437\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 375/375 [01:07<00:00,  5.59it/s, accuracy=0.846, accuracy_depends=0.854, cost=0.568]\n",
      "test minibatch loop: 100%|██████████| 120/120 [00:08<00:00, 14.78it/s, accuracy=0.834, accuracy_depends=0.818, cost=0.709]\n",
      "train minibatch loop:   0%|          | 1/375 [00:00<01:07,  5.52it/s, accuracy=0.877, accuracy_depends=0.879, cost=0.351]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 5, training loss: 0.542109, training acc: 0.860690, training depends: 0.831518, valid loss: 1.586633, valid acc: 0.784649, valid depends: 0.711521\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 375/375 [01:06<00:00,  5.61it/s, accuracy=0.871, accuracy_depends=0.863, cost=0.493]\n",
      "test minibatch loop: 100%|██████████| 120/120 [00:08<00:00, 14.64it/s, accuracy=0.842, accuracy_depends=0.826, cost=0.704]\n",
      "train minibatch loop:   0%|          | 1/375 [00:00<01:08,  5.43it/s, accuracy=0.888, accuracy_depends=0.891, cost=0.271]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 6, training loss: 0.442386, training acc: 0.872789, training depends: 0.845544, valid loss: 1.746875, valid acc: 0.800454, valid depends: 0.713102\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 375/375 [01:07<00:00,  5.55it/s, accuracy=0.88, accuracy_depends=0.872, cost=0.452] \n",
      "test minibatch loop: 100%|██████████| 120/120 [00:08<00:00, 14.75it/s, accuracy=0.846, accuracy_depends=0.81, cost=0.884]\n",
      "train minibatch loop:   0%|          | 1/375 [00:00<01:11,  5.25it/s, accuracy=0.912, accuracy_depends=0.916, cost=0.222]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 7, training loss: 0.372068, training acc: 0.886267, training depends: 0.856076, valid loss: 1.880266, valid acc: 0.801846, valid depends: 0.711724\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 375/375 [01:07<00:00,  5.60it/s, accuracy=0.881, accuracy_depends=0.883, cost=0.341] \n",
      "test minibatch loop: 100%|██████████| 120/120 [00:08<00:00, 14.50it/s, accuracy=0.862, accuracy_depends=0.83, cost=0.752]\n",
      "train minibatch loop:   0%|          | 1/375 [00:00<01:09,  5.40it/s, accuracy=0.912, accuracy_depends=0.909, cost=0.2]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 8, training loss: 0.306797, training acc: 0.896594, training depends: 0.864416, valid loss: 2.109596, valid acc: 0.804174, valid depends: 0.714363\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 375/375 [01:06<00:00,  5.61it/s, accuracy=0.889, accuracy_depends=0.896, cost=0.268] \n",
      "test minibatch loop: 100%|██████████| 120/120 [00:08<00:00, 14.85it/s, accuracy=0.842, accuracy_depends=0.838, cost=0.824]\n",
      "train minibatch loop:   0%|          | 1/375 [00:00<01:13,  5.12it/s, accuracy=0.921, accuracy_depends=0.906, cost=0.206]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 9, training loss: 0.248885, training acc: 0.905460, training depends: 0.872795, valid loss: 2.231252, valid acc: 0.811433, valid depends: 0.713417\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 375/375 [01:07<00:00,  5.60it/s, accuracy=0.9, accuracy_depends=0.904, cost=0.236]   \n",
      "test minibatch loop: 100%|██████████| 120/120 [00:08<00:00, 14.58it/s, accuracy=0.85, accuracy_depends=0.826, cost=0.933]\n",
      "train minibatch loop:   0%|          | 1/375 [00:00<01:06,  5.60it/s, accuracy=0.919, accuracy_depends=0.912, cost=0.136]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 10, training loss: 0.209872, training acc: 0.910505, training depends: 0.877886, valid loss: 2.240191, valid acc: 0.811876, valid depends: 0.717278\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 375/375 [01:06<00:00,  5.62it/s, accuracy=0.904, accuracy_depends=0.904, cost=0.177] \n",
      "test minibatch loop: 100%|██████████| 120/120 [00:08<00:00, 14.63it/s, accuracy=0.846, accuracy_depends=0.814, cost=1.07]\n",
      "train minibatch loop:   0%|          | 1/375 [00:00<01:10,  5.33it/s, accuracy=0.927, accuracy_depends=0.927, cost=0.0913]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 11, training loss: 0.179435, training acc: 0.917127, training depends: 0.882472, valid loss: 2.472229, valid acc: 0.808497, valid depends: 0.717172\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 375/375 [01:07<00:00,  5.57it/s, accuracy=0.909, accuracy_depends=0.908, cost=0.19]  \n",
      "test minibatch loop: 100%|██████████| 120/120 [00:08<00:00, 14.70it/s, accuracy=0.858, accuracy_depends=0.798, cost=1.54]\n",
      "train minibatch loop:   0%|          | 1/375 [00:00<01:12,  5.15it/s, accuracy=0.938, accuracy_depends=0.931, cost=0.0815]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 12, training loss: 0.146958, training acc: 0.924886, training depends: 0.886345, valid loss: 2.617582, valid acc: 0.816131, valid depends: 0.719541\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 375/375 [01:07<00:00,  5.58it/s, accuracy=0.921, accuracy_depends=0.907, cost=0.151] \n",
      "test minibatch loop: 100%|██████████| 120/120 [00:08<00:00, 14.43it/s, accuracy=0.879, accuracy_depends=0.806, cost=1.17]\n",
      "train minibatch loop:   0%|          | 1/375 [00:00<01:12,  5.17it/s, accuracy=0.954, accuracy_depends=0.924, cost=0.0877]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 13, training loss: 0.124930, training acc: 0.929599, training depends: 0.888695, valid loss: 2.686270, valid acc: 0.823643, valid depends: 0.720902\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 375/375 [01:07<00:00,  5.56it/s, accuracy=0.921, accuracy_depends=0.925, cost=0.108] \n",
      "test minibatch loop: 100%|██████████| 120/120 [00:07<00:00, 16.10it/s, accuracy=0.87, accuracy_depends=0.818, cost=1.4]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 14, training loss: 0.106369, training acc: 0.938695, training depends: 0.891401, valid loss: 2.815747, valid acc: 0.824535, valid depends: 0.724233\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "batch_size = 32\n",
    "epoch = 15\n",
    "\n",
    "for e in range(epoch):\n",
    "    train_acc, train_loss = [], []\n",
    "    test_acc, test_loss = [], []\n",
    "    train_acc_depends, test_acc_depends = [], []\n",
    "    \n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x = train_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_char = train_char[i: index]\n",
    "        batch_char = generate_char_seq(batch_char)\n",
    "        batch_y = train_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = train_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        \n",
    "        acc_depends, acc, cost, _ = sess.run(\n",
    "            [model.accuracy_depends, model.accuracy, model.cost, model.optimizer],\n",
    "            feed_dict = {\n",
    "                model.words: batch_x,\n",
    "                model.chars: batch_char,\n",
    "                model.types: batch_y,\n",
    "                model.heads: batch_depends\n",
    "            },\n",
    "        )\n",
    "        train_loss.append(cost)\n",
    "        train_acc.append(acc)\n",
    "        train_acc_depends.append(acc_depends)\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "        \n",
    "    pbar = tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'test minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x = test_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_char = test_char[i: index]\n",
    "        batch_char = generate_char_seq(batch_char)\n",
    "        batch_y = test_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = test_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        \n",
    "        acc_depends, acc, cost = sess.run(\n",
    "            [model.accuracy_depends, model.accuracy, model.cost],\n",
    "            feed_dict = {\n",
    "                model.words: batch_x,\n",
    "                model.chars: batch_char,\n",
    "                model.types: batch_y,\n",
    "                model.heads: batch_depends\n",
    "            },\n",
    "        )\n",
    "        test_loss.append(cost)\n",
    "        test_acc.append(acc)\n",
    "        test_acc_depends.append(acc_depends)\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "    \n",
    "    \n",
    "    print(\n",
    "    'epoch: %d, training loss: %f, training acc: %f, training depends: %f, valid loss: %f, valid acc: %f, valid depends: %f\\n'\n",
    "    % (e, np.mean(train_loss), \n",
    "       np.mean(train_acc), \n",
    "       np.mean(train_acc_depends), \n",
    "       np.mean(test_loss), \n",
    "       np.mean(test_acc), \n",
    "       np.mean(test_acc_depends)\n",
    "    ))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 3,  6, 22, 26,  6, 18, 16,  5,  3, 13, 10, 11,  6, 12, 31, 10, 16,\n",
       "         7,  7,  7,  7,  7,  7,  7,  7,  7,  7,  7,  7,  7,  7,  7,  7,  7,\n",
       "        22]),\n",
       " array([ 2,  6,  5,  5,  2,  8,  8,  0, 11, 11,  8, 14, 14,  8, 16, 14, 14,\n",
       "         8,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "         0]),\n",
       " array([ 2,  8,  5,  5,  2,  8,  8,  0, 11, 11,  8, 14, 14,  8, 16, 14, 14,\n",
       "         8,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "         0], dtype=int32))"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags_seq, heads = sess.run(\n",
    "    [model.tags_seq, model.heads_seq],\n",
    "    feed_dict = {\n",
    "        model.words: batch_x,\n",
    "        model.chars: batch_char\n",
    "    },\n",
    ")\n",
    "tags_seq[0], heads[0], batch_depends[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(heads_pred, types_pred, heads, types, lengths,\n",
    "             symbolic_root=False, symbolic_end=False):\n",
    "    batch_size, _ = heads_pred.shape\n",
    "    ucorr = 0.\n",
    "    lcorr = 0.\n",
    "    total = 0.\n",
    "    ucomplete_match = 0.\n",
    "    lcomplete_match = 0.\n",
    "\n",
    "    corr_root = 0.\n",
    "    total_root = 0.\n",
    "    start = 1 if symbolic_root else 0\n",
    "    end = 1 if symbolic_end else 0\n",
    "    for i in range(batch_size):\n",
    "        ucm = 1.\n",
    "        lcm = 1.\n",
    "        for j in range(start, lengths[i] - end):\n",
    "\n",
    "            total += 1\n",
    "            if heads[i, j] == heads_pred[i, j]:\n",
    "                ucorr += 1\n",
    "                if types[i, j] == types_pred[i, j]:\n",
    "                    lcorr += 1\n",
    "                else:\n",
    "                    lcm = 0\n",
    "            else:\n",
    "                ucm = 0\n",
    "                lcm = 0\n",
    "\n",
    "            if heads[i, j] == 0:\n",
    "                total_root += 1\n",
    "                corr_root += 1 if heads_pred[i, j] == 0 else 0\n",
    "\n",
    "        ucomplete_match += ucm\n",
    "        lcomplete_match += lcm\n",
    "    \n",
    "    return ucorr / total, lcorr / total, corr_root / total_root"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "arcs, types, roots = [], [], []\n",
    "\n",
    "for i in range(0, len(test_X), batch_size):\n",
    "    index = min(i + batch_size, len(test_X))\n",
    "    batch_x = test_X[i: index]\n",
    "    batch_x = pad_sequences(batch_x,padding='post')\n",
    "    batch_char = test_char[i: index]\n",
    "    batch_char = generate_char_seq(batch_char)\n",
    "    batch_y = test_Y[i: index]\n",
    "    batch_y = pad_sequences(batch_y,padding='post')\n",
    "    batch_depends = test_depends[i: index]\n",
    "    batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "    \n",
    "    tags_seq, heads = sess.run(\n",
    "        [model.tags_seq, model.heads_seq],\n",
    "        feed_dict = {\n",
    "            model.words: batch_x,\n",
    "            model.chars: batch_char\n",
    "        },\n",
    "    )\n",
    "    \n",
    "    arc_accuracy, type_accuracy, root_accuracy = evaluate(heads, tags_seq, batch_depends, batch_y, \n",
    "            np.count_nonzero(batch_x, axis = 1))\n",
    "    arcs.append(arc_accuracy)\n",
    "    types.append(type_accuracy)\n",
    "    roots.append(root_accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "arc accuracy: 0.7242327707332225\n",
      "types accuracy: 0.6353447083607996\n",
      "root accuracy: 0.68515625\n"
     ]
    }
   ],
   "source": [
    "print('arc accuracy:', np.mean(arcs))\n",
    "print('types accuracy:', np.mean(types))\n",
    "print('root accuracy:', np.mean(roots))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}