dependency-parser/1.lstm-birnn-crf-biaffine.ipynb

{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!wget https://raw.githubusercontent.com/UniversalDependencies/UD_English-EWT/master/en_ewt-ud-dev.conllu\n!wget https://raw.githubusercontent.com/UniversalDependencies/UD_English-EWT/master/en_ewt-ud-train.conllu\n!wget https://raw.githubusercontent.com/UniversalDependencies/UD_English-EWT/master/en_ewt-ud-test.conllu\n!pip install malaya -U","execution_count":1,"outputs":[{"output_type":"stream","text":"--2019-09-30 05:05:04--  https://raw.githubusercontent.com/UniversalDependencies/UD_English-EWT/master/en_ewt-ud-dev.conllu\nResolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\nConnecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 1668174 (1.6M) [text/plain]\nSaving to: ‘en_ewt-ud-dev.conllu’\n\nen_ewt-ud-dev.conll 100%[===================>]   1.59M  --.-KB/s    in 0.05s   \n\n2019-09-30 05:05:05 (30.7 MB/s) - ‘en_ewt-ud-dev.conllu’ saved [1668174/1668174]\n\n--2019-09-30 05:05:05--  https://raw.githubusercontent.com/UniversalDependencies/UD_English-EWT/master/en_ewt-ud-train.conllu\nResolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\nConnecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 13303045 (13M) [text/plain]\nSaving to: ‘en_ewt-ud-train.conllu’\n\nen_ewt-ud-train.con 100%[===================>]  12.69M  --.-KB/s    in 0.1s    \n\n2019-09-30 05:05:06 (102 MB/s) - ‘en_ewt-ud-train.conllu’ saved [13303045/13303045]\n\n--2019-09-30 05:05:06--  https://raw.githubusercontent.com/UniversalDependencies/UD_English-EWT/master/en_ewt-ud-test.conllu\nResolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\nConnecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 1661985 (1.6M) [text/plain]\nSaving to: ‘en_ewt-ud-test.conllu’\n\nen_ewt-ud-test.conl 100%[===================>]   1.58M  --.-KB/s    in 0.05s   \n\n2019-09-30 05:05:07 (31.0 MB/s) - ‘en_ewt-ud-test.conllu’ saved [1661985/1661985]\n\nCollecting malaya\n\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b1/11/5f8ea8da94136d1fb4db39931d4ed55ae51655a3212b33e5bf607271646e/malaya-2.7.7.0-py3-none-any.whl (2.1MB)\n\u001b[K     |████████████████████████████████| 2.1MB 4.9MB/s eta 0:00:01\n\u001b[?25hRequirement already satisfied, skipping upgrade: scipy in /opt/conda/lib/python3.6/site-packages (from malaya) (1.2.1)\nCollecting dateparser (from malaya)\n\u001b[?25l  Downloading https://files.pythonhosted.org/packages/82/9d/51126ac615bbc4418478d725a5fa1a0f112059f6f111e4b48cfbe17ef9d0/dateparser-0.7.2-py2.py3-none-any.whl (352kB)\n\u001b[K     |████████████████████████████████| 358kB 43.4MB/s eta 0:00:01\n\u001b[?25hRequirement already satisfied, skipping upgrade: sentencepiece in /opt/conda/lib/python3.6/site-packages (from malaya) (0.1.83)\nCollecting bert-tensorflow (from malaya)\n\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a6/66/7eb4e8b6ea35b7cc54c322c816f976167a43019750279a8473d355800a93/bert_tensorflow-1.0.1-py2.py3-none-any.whl (67kB)\n\u001b[K     |████████████████████████████████| 71kB 26.2MB/s eta 0:00:01\n\u001b[?25hRequirement already satisfied, skipping upgrade: tensorflow in /opt/conda/lib/python3.6/site-packages (from malaya) (1.14.0)\nRequirement already satisfied, skipping upgrade: networkx in /opt/conda/lib/python3.6/site-packages (from malaya) (2.3)\nRequirement already satisfied, skipping upgrade: ftfy in /opt/conda/lib/python3.6/site-packages (from malaya) (5.6)\nRequirement already satisfied, skipping upgrade: xgboost in /opt/conda/lib/python3.6/site-packages (from malaya) (0.90)\nRequirement already satisfied, skipping upgrade: sklearn in /opt/conda/lib/python3.6/site-packages (from malaya) (0.0)\nRequirement already satisfied, skipping upgrade: requests in /opt/conda/lib/python3.6/site-packages (from malaya) (2.22.0)\nCollecting PySastrawi (from malaya)\n\u001b[?25l  Downloading https://files.pythonhosted.org/packages/61/84/b0a5454a040f81e81e6a95a5d5635f20ad43cc0c288f8b4966b339084962/PySastrawi-1.2.0-py2.py3-none-any.whl (210kB)\n\u001b[K     |████████████████████████████████| 215kB 44.6MB/s eta 0:00:01\n\u001b[?25hRequirement already satisfied, skipping upgrade: scikit-learn in /opt/conda/lib/python3.6/site-packages (from malaya) (0.21.3)\nRequirement already satisfied, skipping upgrade: numpy in /opt/conda/lib/python3.6/site-packages (from malaya) (1.16.4)\nRequirement already satisfied, skipping upgrade: unidecode in /opt/conda/lib/python3.6/site-packages (from malaya) (1.1.1)\nRequirement already satisfied, skipping upgrade: tzlocal in /opt/conda/lib/python3.6/site-packages (from dateparser->malaya) (2.0.0)\nRequirement already satisfied, skipping upgrade: pytz in /opt/conda/lib/python3.6/site-packages (from dateparser->malaya) (2019.2)\nRequirement already satisfied, skipping upgrade: regex in /opt/conda/lib/python3.6/site-packages (from dateparser->malaya) (2019.8.19)\nRequirement already satisfied, skipping upgrade: python-dateutil in /opt/conda/lib/python3.6/site-packages (from dateparser->malaya) (2.8.0)\nRequirement already satisfied, skipping upgrade: six in /opt/conda/lib/python3.6/site-packages (from bert-tensorflow->malaya) (1.12.0)\nRequirement already satisfied, skipping upgrade: wrapt>=1.11.1 in /opt/conda/lib/python3.6/site-packages (from tensorflow->malaya) (1.11.2)\nRequirement already satisfied, skipping upgrade: astor>=0.6.0 in /opt/conda/lib/python3.6/site-packages (from tensorflow->malaya) (0.8.0)\nRequirement already satisfied, skipping upgrade: wheel>=0.26 in /opt/conda/lib/python3.6/site-packages (from tensorflow->malaya) (0.33.6)\nRequirement already satisfied, skipping upgrade: absl-py>=0.7.0 in /opt/conda/lib/python3.6/site-packages (from tensorflow->malaya) (0.8.0)\nRequirement already satisfied, skipping upgrade: gast>=0.2.0 in /opt/conda/lib/python3.6/site-packages (from tensorflow->malaya) (0.3.2)\nRequirement already satisfied, skipping upgrade: keras-applications>=1.0.6 in /opt/conda/lib/python3.6/site-packages (from tensorflow->malaya) (1.0.8)\nRequirement already satisfied, skipping upgrade: termcolor>=1.1.0 in /opt/conda/lib/python3.6/site-packages (from tensorflow->malaya) (1.1.0)\nRequirement already satisfied, skipping upgrade: tensorflow-estimator<1.15.0rc0,>=1.14.0rc0 in /opt/conda/lib/python3.6/site-packages (from tensorflow->malaya) (1.14.0)\nRequirement already satisfied, skipping upgrade: grpcio>=1.8.6 in /opt/conda/lib/python3.6/site-packages (from tensorflow->malaya) (1.24.0)\nRequirement already satisfied, skipping upgrade: keras-preprocessing>=1.0.5 in /opt/conda/lib/python3.6/site-packages (from tensorflow->malaya) (1.1.0)\nRequirement already satisfied, skipping upgrade: protobuf>=3.6.1 in /opt/conda/lib/python3.6/site-packages (from tensorflow->malaya) (3.7.1)\nRequirement already satisfied, skipping upgrade: tensorboard<1.15.0,>=1.14.0 in /opt/conda/lib/python3.6/site-packages (from tensorflow->malaya) (1.14.0)\nRequirement already satisfied, skipping upgrade: google-pasta>=0.1.6 in /opt/conda/lib/python3.6/site-packages (from tensorflow->malaya) (0.1.7)\nRequirement already satisfied, skipping upgrade: decorator>=4.3.0 in /opt/conda/lib/python3.6/site-packages (from networkx->malaya) (4.4.0)\nRequirement already satisfied, skipping upgrade: wcwidth in /opt/conda/lib/python3.6/site-packages (from ftfy->malaya) (0.1.7)\nRequirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /opt/conda/lib/python3.6/site-packages (from requests->malaya) (3.0.4)\nRequirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /opt/conda/lib/python3.6/site-packages (from requests->malaya) (2019.9.11)\nRequirement already satisfied, skipping upgrade: idna<2.9,>=2.5 in /opt/conda/lib/python3.6/site-packages (from requests->malaya) (2.8)\nRequirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.6/site-packages (from requests->malaya) (1.24.2)\nRequirement already satisfied, skipping upgrade: joblib>=0.11 in /opt/conda/lib/python3.6/site-packages (from scikit-learn->malaya) (0.13.2)\nRequirement already satisfied, skipping upgrade: h5py in /opt/conda/lib/python3.6/site-packages (from keras-applications>=1.0.6->tensorflow->malaya) (2.9.0)\nRequirement already satisfied, skipping upgrade: setuptools in /opt/conda/lib/python3.6/site-packages (from protobuf>=3.6.1->tensorflow->malaya) (41.2.0)\nRequirement already satisfied, skipping upgrade: markdown>=2.6.8 in /opt/conda/lib/python3.6/site-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow->malaya) (3.1.1)\nRequirement already satisfied, skipping upgrade: werkzeug>=0.11.15 in /opt/conda/lib/python3.6/site-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow->malaya) (0.16.0)\n","name":"stdout"},{"output_type":"stream","text":"Installing collected packages: dateparser, bert-tensorflow, PySastrawi, malaya\nSuccessfully installed PySastrawi-1.2.0 bert-tensorflow-1.0.1 dateparser-0.7.2 malaya-2.7.7.0\n","name":"stdout"}]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import malaya\nimport re\nfrom malaya.texts._text_functions import split_into_sentences\nfrom malaya.texts import _regex\nimport numpy as np\nimport itertools\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\n\ntokenizer = malaya.preprocessing._tokenizer\nsplitter = split_into_sentences","execution_count":2,"outputs":[{"output_type":"stream","text":"not found any version, deleting previous version models..\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"def is_number_regex(s):\n    if re.match(\"^\\d+?\\.\\d+?$\", s) is None:\n        return s.isdigit()\n    return True\n\ndef preprocessing(w):\n    if is_number_regex(w):\n        return '<NUM>'\n    elif re.match(_regex._money, w):\n        return '<MONEY>'\n    elif re.match(_regex._date, w):\n        return '<DATE>'\n    elif re.match(_regex._expressions['email'], w):\n        return '<EMAIL>'\n    elif re.match(_regex._expressions['url'], w):\n        return '<URL>'\n    else:\n        w = ''.join(''.join(s)[:2] for _, s in itertools.groupby(w))\n        return w","execution_count":3,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word2idx = {'PAD': 0,'UNK':1, '_ROOT': 2}\ntag2idx = {'PAD': 0, '_<ROOT>': 1}\nchar2idx = {'PAD': 0,'UNK':1, '_ROOT': 2}\nword_idx = 3\ntag_idx = 2\nchar_idx = 3\n\nspecial_tokens = ['<NUM>', '<MONEY>', '<DATE>', '<URL>', '<EMAIL>']\n\nfor t in special_tokens:\n    word2idx[t] = word_idx\n    word_idx += 1\n    char2idx[t] = char_idx\n    char_idx += 1\n    \nword2idx, char2idx","execution_count":4,"outputs":[{"output_type":"execute_result","execution_count":4,"data":{"text/plain":"({'PAD': 0,\n  'UNK': 1,\n  '_ROOT': 2,\n  '<NUM>': 3,\n  '<MONEY>': 4,\n  '<DATE>': 5,\n  '<URL>': 6,\n  '<EMAIL>': 7},\n {'PAD': 0,\n  'UNK': 1,\n  '_ROOT': 2,\n  '<NUM>': 3,\n  '<MONEY>': 4,\n  '<DATE>': 5,\n  '<URL>': 6,\n  '<EMAIL>': 7})"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"PAD = \"_PAD\"\nPAD_POS = \"_PAD_POS\"\nPAD_TYPE = \"_<PAD>\"\nPAD_CHAR = \"_PAD_CHAR\"\nROOT = \"_ROOT\"\nROOT_POS = \"_ROOT_POS\"\nROOT_TYPE = \"_<ROOT>\"\nROOT_CHAR = \"_ROOT_CHAR\"\nEND = \"_END\"\nEND_POS = \"_END_POS\"\nEND_TYPE = \"_<END>\"\nEND_CHAR = \"_END_CHAR\"\n\ndef process_corpus(corpus, until = None):\n    global word2idx, tag2idx, char2idx, word_idx, tag_idx, char_idx\n    sentences, words, depends, labels, pos, chars = [], [], [], [], [], []\n    temp_sentence, temp_word, temp_depend, temp_label, temp_pos = [], [], [], [], []\n    first_time = True\n    for sentence in corpus:\n        try:\n            if len(sentence):\n                if sentence[0] == '#':\n                    continue\n                if first_time:\n                    print(sentence)\n                    first_time = False\n                sentence = sentence.split('\\t')\n                for c in sentence[1]:\n                    if c not in char2idx:\n                        char2idx[c] = char_idx\n                        char_idx += 1\n                if sentence[7] not in tag2idx:\n                    tag2idx[sentence[7]] = tag_idx\n                    tag_idx += 1\n                sentence[1] = preprocessing(sentence[1])\n                if sentence[1] not in word2idx:\n                    word2idx[sentence[1]] = word_idx\n                    word_idx += 1\n                temp_word.append(word2idx[sentence[1]])\n                temp_depend.append(int(sentence[6]))\n                temp_label.append(tag2idx[sentence[7]])\n                temp_sentence.append(sentence[1])\n                temp_pos.append(sentence[3])\n            else:\n                if len(temp_sentence) < 2 or len(temp_word) != len(temp_label):\n                    temp_word = []\n                    temp_depend = []\n                    temp_label = []\n                    temp_sentence = []\n                    temp_pos = []\n                    continue\n                words.append(temp_word)\n                depends.append(temp_depend)\n                labels.append(temp_label)\n                sentences.append( temp_sentence)\n                pos.append(temp_pos)\n                char_ = [[char2idx['_ROOT']]]\n                for w in temp_sentence:\n                    if w in char2idx:\n                        char_.append([char2idx[w]])\n                    else:\n                        char_.append([char2idx[c] for c in w])\n                chars.append(char_)\n                temp_word = []\n                temp_depend = []\n                temp_label = []\n                temp_sentence = []\n                temp_pos = []\n        except Exception as e:\n            print(e, sentence)\n    return sentences[:-1], words[:-1], depends[:-1], labels[:-1], pos[:-1], chars[:-1]","execution_count":5,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('en_ewt-ud-dev.conllu') as fopen:\n    dev = fopen.read().split('\\n')\n\nsentences_dev, words_dev, depends_dev, labels_dev, _, _ = process_corpus(dev)","execution_count":6,"outputs":[{"output_type":"stream","text":"1\tFrom\tfrom\tADP\tIN\t_\t3\tcase\t3:case\t_\ninvalid literal for int() with base 10: '_' ['10.1', 'has', 'have', 'VERB', 'VBZ', '_', '_', '_', '8:parataxis', 'CopyOf=-1']\ninvalid literal for int() with base 10: '_' ['21.1', 'has', 'have', 'VERB', 'VBZ', '_', '_', '_', '16:conj:and', 'CopyOf=-1']\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('en_ewt-ud-test.conllu') as fopen:\n    test = fopen.read().split('\\n')\n\nsentences_test, words_test, depends_test, labels_test, _, _ = process_corpus(test)\nsentences_test.extend(sentences_dev)\nwords_test.extend(words_dev)\ndepends_test.extend(depends_dev)\nlabels_test.extend(labels_dev)","execution_count":7,"outputs":[{"output_type":"stream","text":"1\tWhat\twhat\tPRON\tWP\tPronType=Int\t0\troot\t0:root\t_\ninvalid literal for int() with base 10: '_' ['24.1', 'left', 'left', 'VERB', 'VBN', 'Tense=Past|VerbForm=Part', '_', '_', '6:parataxis', 'CopyOf=6']\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('en_ewt-ud-train.conllu') as fopen:\n    train = fopen.read().split('\\n')\n\nsentences_train, words_train, depends_train, labels_train, _, _ = process_corpus(train)","execution_count":8,"outputs":[{"output_type":"stream","text":"1\tAl\tAl\tPROPN\tNNP\tNumber=Sing\t0\troot\t0:root\tSpaceAfter=No\ninvalid literal for int() with base 10: '_' ['8.1', 'reported', 'report', 'VERB', 'VBN', 'Tense=Past|VerbForm=Part|Voice=Pass', '_', '_', '5:conj:and', 'CopyOf=5']\ninvalid literal for int() with base 10: '_' ['22.1', 'used', 'use', 'VERB', 'VBN', 'Tense=Past|VerbForm=Part', '_', '_', '13:advcl:with|17:conj:and', 'CopyOf=17']\ninvalid literal for int() with base 10: '_' ['22.1', 'used', 'use', 'VERB', 'VBN', 'Tense=Past|VerbForm=Part', '_', '_', '13:advcl:with|17:conj:and', 'CopyOf=17']\ninvalid literal for int() with base 10: '_' ['11.1', 'called', 'call', 'VERB', 'VBN', 'Tense=Past|VerbForm=Part|Voice=Pass', '_', '_', '3:conj:and', 'CopyOf=3']\ninvalid literal for int() with base 10: '_' ['14.1', 'is', 'be', 'VERB', 'VBZ', '_', '_', '_', '1:conj:and', 'CopyOf=1']\ninvalid literal for int() with base 10: '_' ['20.1', 'reflect', 'reflect', 'VERB', 'VBP', 'Mood=Ind|Tense=Pres|VerbForm=Fin', '_', '_', '7:acl:relcl|9:conj', 'CopyOf=9']\ninvalid literal for int() with base 10: '_' ['21.1', 'recruited', 'recruit', 'VERB', 'VBD', 'Mood=Ind|Tense=Past|VerbForm=Fin', '_', '_', '9:conj:and', 'CopyOf=9']\ninvalid literal for int() with base 10: '_' ['9.1', 'wish', 'wish', 'VERB', 'VBP', 'Mood=Ind|Tense=Pres|VerbForm=Fin', '_', '_', '2:conj:and', 'CopyOf=2']\ninvalid literal for int() with base 10: '_' ['38.1', 'supplied', 'supply', 'VERB', 'VBN', 'Tense=Past|VerbForm=Part|Voice=Pass', '_', '_', '16:conj:and', 'CopyOf=16']\ninvalid literal for int() with base 10: '_' ['18.1', 'keep', 'keep', 'VERB', 'VB', 'Mood=Imp|VerbForm=Fin', '_', '_', '14:conj:and', 'CopyOf=14']\ninvalid literal for int() with base 10: '_' ['21.1', 'keep', 'keep', 'VERB', 'VB', 'Mood=Imp|VerbForm=Fin', '_', '_', '14:conj:and', 'CopyOf=14']\ninvalid literal for int() with base 10: '_' ['18.1', 'mean', 'mean', 'VERB', 'VB', 'VerbForm=Inf', '_', '_', '8:conj', 'CopyOf=8']\ninvalid literal for int() with base 10: '_' ['30.1', 'play', 'play', 'VERB', 'VBP', 'Mood=Ind|Tense=Pres|VerbForm=Fin', '_', '_', '18:acl:relcl|27:conj:but', 'CopyOf=27']\ninvalid literal for int() with base 10: '_' ['22.1', 'have', 'have', 'VERB', 'VBP', 'Mood=Ind|Tense=Pres|VerbForm=Fin', '_', '_', '17:conj', 'CopyOf=17']\ninvalid literal for int() with base 10: '_' ['27.1', 'have', 'have', 'VERB', 'VBP', 'Mood=Ind|Tense=Pres|VerbForm=Fin', '_', '_', '17:conj', 'CopyOf=17']\ninvalid literal for int() with base 10: '_' ['49.1', 'helped', 'help', 'VERB', 'VBD', '_', '_', '_', '38:conj:but', 'CopyOf=38']\ninvalid literal for int() with base 10: '_' ['7.1', 'found', 'find', 'VERB', 'VBD', 'Mood=Ind|Tense=Past|VerbForm=Fin', '_', '_', '3:conj', 'CopyOf=3']\ninvalid literal for int() with base 10: '_' ['10.1', 'excited', 'excited', 'ADJ', 'JJ', 'Degree=Pos', '_', '_', '4:advcl', 'CopyOf=4']\ninvalid literal for int() with base 10: '_' ['15.1', \"'s\", 'be', 'VERB', 'VBZ', '_', '_', '_', '2:conj:and', 'CopyOf=2']\ninvalid literal for int() with base 10: '_' ['25.1', 'took', 'take', 'VERB', 'VBD', 'Mood=Ind|Tense=Past|VerbForm=Fin', '_', '_', '17:conj:and', 'CopyOf=17']\ninvalid literal for int() with base 10: '_' ['10.1', 'loss', 'lose', 'VERB', 'VBD', 'Mood=Ind|Tense=Past|VerbForm=Fin', '_', '_', '3:conj:and', 'CopyOf=3']\ninvalid literal for int() with base 10: '_' ['11.1', 'leave', 'leave', 'VERB', 'VB', 'VerbForm=Inf', '_', '_', '7:parataxis', 'CopyOf=7']\ninvalid literal for int() with base 10: '_' ['24.1', 'charge', 'charge', 'VERB', 'VBP', 'Mood=Ind|Tense=Pres|VerbForm=Fin', '_', '_', '16:conj:and', 'CopyOf=16']\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(sentences_train), len(sentences_test)","execution_count":9,"outputs":[{"output_type":"execute_result","execution_count":9,"data":{"text/plain":"(12000, 3824)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"idx2word = {v:k for k, v in word2idx.items()}\nidx2tag = {v:k for k, v in tag2idx.items()}\nlen(idx2word)","execution_count":10,"outputs":[{"output_type":"execute_result","execution_count":10,"data":{"text/plain":"21974"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generate_char_seq(batch, UNK = 2):\n    maxlen_c = max([len(k) for k in batch])\n    x = [[len(i) for i in k] for k in batch]\n    maxlen = max([j for i in x for j in i])\n    temp = np.zeros((len(batch),maxlen_c,maxlen),dtype=np.int32)\n    for i in range(len(batch)):\n        for k in range(len(batch[i])):\n            for no, c in enumerate(batch[i][k]):\n                temp[i,k,-1-no] = char2idx.get(c, UNK)\n    return temp","execution_count":11,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"generate_char_seq(sentences_train[:5]).shape","execution_count":12,"outputs":[{"output_type":"execute_result","execution_count":12,"data":{"text/plain":"(5, 36, 11)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"pad_sequences(words_train[:5],padding='post').shape","execution_count":13,"outputs":[{"output_type":"execute_result","execution_count":13,"data":{"text/plain":"(5, 36)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X = words_train\ntrain_Y = labels_train\ntrain_depends = depends_train\ntrain_char = sentences_train\n\ntest_X = words_test\ntest_Y = labels_test\ntest_depends = depends_test\ntest_char = sentences_test","execution_count":14,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BiAAttention:\n    def __init__(self, input_size_encoder, input_size_decoder, num_labels):\n        self.input_size_encoder = input_size_encoder\n        self.input_size_decoder = input_size_decoder\n        self.num_labels = num_labels\n        \n        self.W_d = tf.get_variable(\"W_d\", shape=[self.num_labels, self.input_size_decoder],\n           initializer=tf.contrib.layers.xavier_initializer())\n        self.W_e = tf.get_variable(\"W_e\", shape=[self.num_labels, self.input_size_encoder],\n           initializer=tf.contrib.layers.xavier_initializer())\n        self.U = tf.get_variable(\"U\", shape=[self.num_labels, self.input_size_decoder, self.input_size_encoder],\n           initializer=tf.contrib.layers.xavier_initializer())\n        \n    def forward(self, input_d, input_e, mask_d=None, mask_e=None):\n        batch = tf.shape(input_d)[0]\n        length_decoder = tf.shape(input_d)[1]\n        length_encoder = tf.shape(input_e)[1]\n        out_d = tf.expand_dims(tf.matmul(self.W_d, tf.transpose(input_d, [0, 2, 1])), 3)\n        out_e = tf.expand_dims(tf.matmul(self.W_e, tf.transpose(input_e, [0, 2, 1])), 2)\n        output = tf.matmul(tf.expand_dims(input_d, 1), self.U)\n        output = tf.matmul(output, tf.transpose(tf.expand_dims(input_e, 1), [0, 1, 3, 2]))\n        \n        output = output + out_d + out_e\n        \n        if mask_d is not None:\n            d = tf.expand_dims(tf.expand_dims(mask_d, 1), 3)\n            e = tf.expand_dims(tf.expand_dims(mask_e, 1), 2)\n            output = output * d * e\n            \n        return output\n\nclass Model:\n    def __init__(\n        self,\n        dim_word,\n        dim_char,\n        dropout,\n        learning_rate,\n        hidden_size_char,\n        hidden_size_word,\n        num_layers\n    ):\n        def cells(size, reuse = False):\n            return tf.contrib.rnn.DropoutWrapper(\n                tf.nn.rnn_cell.LSTMCell(\n                    size,\n                    initializer = tf.orthogonal_initializer(),\n                    reuse = reuse,\n                ),\n                output_keep_prob = dropout,\n            )\n        \n        self.word_ids = tf.placeholder(tf.int32, shape = [None, None])\n        self.char_ids = tf.placeholder(tf.int32, shape = [None, None, None])\n        self.labels = tf.placeholder(tf.int32, shape = [None, None])\n        self.depends = tf.placeholder(tf.int32, shape = [None, None])\n        self.maxlen = tf.shape(self.word_ids)[1]\n        self.lengths = tf.count_nonzero(self.word_ids, 1)\n        self.mask = tf.math.not_equal(self.word_ids, 0)\n        float_mask = tf.cast(self.mask, tf.float32)\n        \n        self.arc_h = tf.layers.Dense(hidden_size_word)\n        self.arc_c = tf.layers.Dense(hidden_size_word)\n        self.attention = BiAAttention(hidden_size_word, hidden_size_word, 1)\n\n        self.word_embeddings = tf.Variable(\n            tf.truncated_normal(\n                [len(word2idx), dim_word], stddev = 1.0 / np.sqrt(dim_word)\n            )\n        )\n        self.char_embeddings = tf.Variable(\n            tf.truncated_normal(\n                [len(char2idx), dim_char], stddev = 1.0 / np.sqrt(dim_char)\n            )\n        )\n\n        word_embedded = tf.nn.embedding_lookup(\n            self.word_embeddings, self.word_ids\n        )\n        char_embedded = tf.nn.embedding_lookup(\n            self.char_embeddings, self.char_ids\n        )\n        s = tf.shape(char_embedded)\n        char_embedded = tf.reshape(\n            char_embedded, shape = [s[0] * s[1], s[-2], dim_char]\n        )\n\n        for n in range(num_layers):\n            (out_fw, out_bw), (\n                state_fw,\n                state_bw,\n            ) = tf.nn.bidirectional_dynamic_rnn(\n                cell_fw = cells(hidden_size_char),\n                cell_bw = cells(hidden_size_char),\n                inputs = char_embedded,\n                dtype = tf.float32,\n                scope = 'bidirectional_rnn_char_%d' % (n),\n            )\n            char_embedded = tf.concat((out_fw, out_bw), 2)\n        output = tf.reshape(\n            char_embedded[:, -1], shape = [s[0], s[1], 2 * hidden_size_char]\n        )\n        word_embedded = tf.concat([word_embedded, output], axis = -1)\n\n        for n in range(num_layers):\n            (out_fw, out_bw), (\n                state_fw,\n                state_bw,\n            ) = tf.nn.bidirectional_dynamic_rnn(\n                cell_fw = cells(hidden_size_word),\n                cell_bw = cells(hidden_size_word),\n                inputs = word_embedded,\n                dtype = tf.float32,\n                scope = 'bidirectional_rnn_word_%d' % (n),\n            )\n            word_embedded = tf.concat((out_fw, out_bw), 2)\n\n        logits = tf.layers.dense(word_embedded, len(idx2tag))\n        log_likelihood, transition_params = tf.contrib.crf.crf_log_likelihood(\n            logits, self.labels, self.lengths\n        )\n        arc_h = tf.nn.elu(self.arc_h(word_embedded))\n        arc_c = tf.nn.elu(self.arc_c(word_embedded))\n        out_arc = tf.squeeze(self.attention.forward(arc_h, arc_h, mask_d=float_mask, mask_e=float_mask), axis = 1)\n        \n        batch = tf.shape(out_arc)[0]\n        batch_index = tf.range(0, batch)\n        max_len = tf.shape(out_arc)[1]\n        sec_max_len = tf.shape(out_arc)[2]\n        \n        minus_inf = -1e8\n        minus_mask = (1 - float_mask) * minus_inf\n        out_arc = out_arc + tf.expand_dims(minus_mask, axis = 2) + tf.expand_dims(minus_mask, axis = 1)\n        loss_arc = tf.nn.log_softmax(out_arc, dim=1)\n        loss_arc = loss_arc * tf.expand_dims(float_mask, axis = 2) * tf.expand_dims(float_mask, axis = 1)\n        num = tf.reduce_sum(float_mask) - tf.cast(batch, tf.float32)\n        \n        child_index = tf.tile(tf.expand_dims(tf.range(0, max_len), 1), [1, batch])\n        t = tf.transpose(self.depends)\n        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0),\n                                               tf.expand_dims(t, axis = 0),\n                                               tf.expand_dims(child_index, axis = 0)], axis = 0))\n        loss_arc = tf.gather_nd(loss_arc, concatenated)\n        loss_arc = tf.transpose(loss_arc, [1, 0])[1:]\n        \n        loss_arc = tf.reduce_sum(-loss_arc) / num\n        \n        self.cost = tf.reduce_mean(-log_likelihood) + loss_arc\n        \n        self.optimizer = tf.train.AdamOptimizer(\n            learning_rate = learning_rate\n        ).minimize(self.cost)\n        \n        mask = tf.sequence_mask(self.lengths, maxlen = self.maxlen)\n        \n        self.tags_seq, _ = tf.contrib.crf.crf_decode(\n            logits, transition_params, self.lengths\n        )\n        \n        out_arc = out_arc + tf.linalg.diag(tf.fill([max_len], -np.inf))\n        minus_mask = tf.expand_dims(tf.cast(1.0 - float_mask, tf.bool), axis = 2)\n        minus_mask = tf.tile(minus_mask, [1, 1, sec_max_len])\n        out_arc = tf.where(minus_mask, tf.fill(tf.shape(out_arc), -np.inf), out_arc)\n        self.heads = tf.argmax(out_arc, axis = 1)\n        \n        self.prediction = tf.boolean_mask(self.tags_seq, mask)\n        mask_label = tf.boolean_mask(self.labels, mask)\n        correct_pred = tf.equal(self.prediction, mask_label)\n        correct_index = tf.cast(correct_pred, tf.float32)\n        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n        \n        self.prediction = tf.cast(tf.boolean_mask(self.heads, mask), tf.int32)\n        mask_label = tf.boolean_mask(self.depends, mask)\n        correct_pred = tf.equal(self.prediction, mask_label)\n        correct_index = tf.cast(correct_pred, tf.float32)\n        self.accuracy_depends = tf.reduce_mean(tf.cast(correct_pred, tf.float32))","execution_count":15,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.reset_default_graph()\nsess = tf.InteractiveSession()\n\ndim_word = 128\ndim_char = 256\ndropout = 1.0\nlearning_rate = 1e-3\nhidden_size_char = 128\nhidden_size_word = 128\nnum_layers = 2\n\nmodel = Model(dim_word,dim_char,dropout,learning_rate,hidden_size_char,hidden_size_word,num_layers)\nsess.run(tf.global_variables_initializer())","execution_count":16,"outputs":[{"output_type":"stream","text":"WARNING: Entity <bound method LSTMCell.call of <tensorflow.python.ops.rnn_cell_impl.LSTMCell object at 0x7f95a31d34e0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method LSTMCell.call of <tensorflow.python.ops.rnn_cell_impl.LSTMCell object at 0x7f95a31d34e0>>: AttributeError: module 'gast' has no attribute 'Num'\nWARNING: Entity <bound method LSTMCell.call of <tensorflow.python.ops.rnn_cell_impl.LSTMCell object at 0x7f95a3213160>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method LSTMCell.call of <tensorflow.python.ops.rnn_cell_impl.LSTMCell object at 0x7f95a3213160>>: AttributeError: module 'gast' has no attribute 'Num'\nWARNING: Entity <bound method LSTMCell.call of <tensorflow.python.ops.rnn_cell_impl.LSTMCell object at 0x7f95a31b0048>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method LSTMCell.call of <tensorflow.python.ops.rnn_cell_impl.LSTMCell object at 0x7f95a31b0048>>: AttributeError: module 'gast' has no attribute 'Num'\nWARNING: Entity <bound method LSTMCell.call of <tensorflow.python.ops.rnn_cell_impl.LSTMCell object at 0x7f95a31d54a8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method LSTMCell.call of <tensorflow.python.ops.rnn_cell_impl.LSTMCell object at 0x7f95a31d54a8>>: AttributeError: module 'gast' has no attribute 'Num'\nWARNING: Entity <bound method LSTMCell.call of <tensorflow.python.ops.rnn_cell_impl.LSTMCell object at 0x7f95a2fd1a20>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method LSTMCell.call of <tensorflow.python.ops.rnn_cell_impl.LSTMCell object at 0x7f95a2fd1a20>>: AttributeError: module 'gast' has no attribute 'Num'\nWARNING: Entity <bound method LSTMCell.call of <tensorflow.python.ops.rnn_cell_impl.LSTMCell object at 0x7f95a2fc8f98>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method LSTMCell.call of <tensorflow.python.ops.rnn_cell_impl.LSTMCell object at 0x7f95a2fc8f98>>: AttributeError: module 'gast' has no attribute 'Num'\nWARNING: Entity <bound method LSTMCell.call of <tensorflow.python.ops.rnn_cell_impl.LSTMCell object at 0x7f95a2f35c18>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method LSTMCell.call of <tensorflow.python.ops.rnn_cell_impl.LSTMCell object at 0x7f95a2f35c18>>: AttributeError: module 'gast' has no attribute 'Num'\nWARNING: Entity <bound method LSTMCell.call of <tensorflow.python.ops.rnn_cell_impl.LSTMCell object at 0x7f95a2e33710>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method LSTMCell.call of <tensorflow.python.ops.rnn_cell_impl.LSTMCell object at 0x7f95a2e33710>>: AttributeError: module 'gast' has no attribute 'Num'\nWARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7f95a2e9df60>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7f95a2e9df60>>: AssertionError: Bad argument number for Name: 3, expecting 4\nWARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7f95a3279e10>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7f95a3279e10>>: AssertionError: Bad argument number for Name: 3, expecting 4\nWARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7f95a327e438>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x7f95a327e438>>: AssertionError: Bad argument number for Name: 3, expecting 4\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_x = train_X[:5]\nbatch_x = pad_sequences(batch_x,padding='post')\nbatch_char = train_char[:5]\nbatch_char = generate_char_seq(batch_char)\nbatch_y = train_Y[:5]\nbatch_y = pad_sequences(batch_y,padding='post')\nbatch_depends = train_depends[:5]\nbatch_depends = pad_sequences(batch_depends,padding='post')","execution_count":17,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sess.run([model.accuracy, model.accuracy_depends, model.cost],\n        feed_dict = {model.word_ids: batch_x,\n                model.char_ids: batch_char,\n                model.labels: batch_y,\n                model.depends: batch_depends})","execution_count":18,"outputs":[{"output_type":"execute_result","execution_count":18,"data":{"text/plain":"[0.0, 0.00862069, 94.88574]"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\n\nbatch_size = 32\nepoch = 15\n\nfor e in range(epoch):\n    train_acc, train_loss = [], []\n    test_acc, test_loss = [], []\n    train_acc_depends, test_acc_depends = [], []\n    \n    pbar = tqdm(\n        range(0, len(train_X), batch_size), desc = 'train minibatch loop'\n    )\n    for i in pbar:\n        index = min(i + batch_size, len(train_X))\n        batch_x = train_X[i: index]\n        batch_x = pad_sequences(batch_x,padding='post')\n        batch_char = train_char[i: index]\n        batch_char = generate_char_seq(batch_char)\n        batch_y = train_Y[i: index]\n        batch_y = pad_sequences(batch_y,padding='post')\n        batch_depends = train_depends[i: index]\n        batch_depends = pad_sequences(batch_depends,padding='post')\n        \n        acc_depends, acc, cost, _ = sess.run(\n            [model.accuracy_depends, model.accuracy, model.cost, model.optimizer],\n            feed_dict = {\n                model.word_ids: batch_x,\n                model.char_ids: batch_char,\n                model.labels: batch_y,\n                model.depends: batch_depends\n            },\n        )\n        train_loss.append(cost)\n        train_acc.append(acc)\n        train_acc_depends.append(acc_depends)\n        pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n        \n    pbar = tqdm(\n        range(0, len(test_X), batch_size), desc = 'test minibatch loop'\n    )\n    for i in pbar:\n        index = min(i + batch_size, len(test_X))\n        batch_x = test_X[i: index]\n        batch_x = pad_sequences(batch_x,padding='post')\n        batch_char = test_char[i: index]\n        batch_char = generate_char_seq(batch_char)\n        batch_y = test_Y[i: index]\n        batch_y = pad_sequences(batch_y,padding='post')\n        batch_depends = test_depends[i: index]\n        batch_depends = pad_sequences(batch_depends,padding='post')\n        \n        acc_depends, acc, cost = sess.run(\n            [model.accuracy_depends, model.accuracy, model.cost],\n            feed_dict = {\n                model.word_ids: batch_x,\n                model.char_ids: batch_char,\n                model.labels: batch_y,\n                model.depends: batch_depends\n            },\n        )\n        test_loss.append(cost)\n        test_acc.append(acc)\n        test_acc_depends.append(acc_depends)\n        pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n    \n    \n    print(\n    'epoch: %d, training loss: %f, training acc: %f, training depends: %f, valid loss: %f, valid acc: %f, valid depends: %f\\n'\n    % (e, np.mean(train_loss), \n       np.mean(train_acc), \n       np.mean(train_acc_depends), \n       np.mean(test_loss), \n       np.mean(test_acc), \n       np.mean(test_acc_depends)\n    ))\n        ","execution_count":19,"outputs":[{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 375/375 [02:06<00:00,  2.96it/s, accuracy=0.79, accuracy_depends=0.607, cost=17.6]  \ntest minibatch loop: 100%|██████████| 120/120 [00:18<00:00,  6.36it/s, accuracy=0.822, accuracy_depends=0.599, cost=10.2]\ntrain minibatch loop:   0%|          | 0/375 [00:00<?, ?it/s]","name":"stderr"},{"output_type":"stream","text":"epoch: 0, training loss: 30.011385, training acc: 0.547456, training depends: 0.384560, valid loss: 12.160900, valid acc: 0.763054, valid depends: 0.570995\n\n","name":"stdout"},{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 375/375 [02:06<00:00,  2.97it/s, accuracy=0.871, accuracy_depends=0.731, cost=11.2]\ntest minibatch loop: 100%|██████████| 120/120 [00:17<00:00,  6.70it/s, accuracy=0.883, accuracy_depends=0.721, cost=7.19]\ntrain minibatch loop:   0%|          | 0/375 [00:00<?, ?it/s]","name":"stderr"},{"output_type":"stream","text":"epoch: 1, training loss: 11.472513, training acc: 0.819827, training depends: 0.622879, valid loss: 8.814706, valid acc: 0.820314, valid depends: 0.651465\n\n","name":"stdout"},{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 375/375 [02:04<00:00,  3.02it/s, accuracy=0.891, accuracy_depends=0.772, cost=8.89]\ntest minibatch loop: 100%|██████████| 120/120 [00:18<00:00,  6.66it/s, accuracy=0.899, accuracy_depends=0.773, cost=5.81]\ntrain minibatch loop:   0%|          | 0/375 [00:00<?, ?it/s]","name":"stderr"},{"output_type":"stream","text":"epoch: 2, training loss: 7.686915, training acc: 0.879738, training depends: 0.695450, valid loss: 7.770745, valid acc: 0.841929, valid depends: 0.677420\n\n","name":"stdout"},{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 375/375 [02:04<00:00,  3.02it/s, accuracy=0.928, accuracy_depends=0.792, cost=7.04]\ntest minibatch loop: 100%|██████████| 120/120 [00:18<00:00,  6.53it/s, accuracy=0.927, accuracy_depends=0.794, cost=5.42]\ntrain minibatch loop:   0%|          | 0/375 [00:00<?, ?it/s]","name":"stderr"},{"output_type":"stream","text":"epoch: 3, training loss: 5.874665, training acc: 0.909947, training depends: 0.731291, valid loss: 7.536985, valid acc: 0.848511, valid depends: 0.688594\n\n","name":"stdout"},{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 375/375 [02:05<00:00,  3.00it/s, accuracy=0.94, accuracy_depends=0.807, cost=5.93] \ntest minibatch loop: 100%|██████████| 120/120 [00:18<00:00,  6.54it/s, accuracy=0.907, accuracy_depends=0.794, cost=5.43]\ntrain minibatch loop:   0%|          | 0/375 [00:00<?, ?it/s]","name":"stderr"},{"output_type":"stream","text":"epoch: 4, training loss: 4.713706, training acc: 0.930022, training depends: 0.755921, valid loss: 7.551268, valid acc: 0.853732, valid depends: 0.697372\n\n","name":"stdout"},{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 375/375 [02:05<00:00,  2.99it/s, accuracy=0.948, accuracy_depends=0.823, cost=4.92]\ntest minibatch loop: 100%|██████████| 120/120 [00:18<00:00,  6.60it/s, accuracy=0.903, accuracy_depends=0.785, cost=5.59]\ntrain minibatch loop:   0%|          | 0/375 [00:00<?, ?it/s]","name":"stderr"},{"output_type":"stream","text":"epoch: 5, training loss: 3.865628, training acc: 0.944412, training depends: 0.772768, valid loss: 7.787340, valid acc: 0.854716, valid depends: 0.699638\n\n","name":"stdout"},{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 375/375 [02:04<00:00,  3.02it/s, accuracy=0.959, accuracy_depends=0.838, cost=3.99]\ntest minibatch loop: 100%|██████████| 120/120 [00:18<00:00,  6.34it/s, accuracy=0.903, accuracy_depends=0.789, cost=5.47]\ntrain minibatch loop:   0%|          | 0/375 [00:00<?, ?it/s]","name":"stderr"},{"output_type":"stream","text":"epoch: 6, training loss: 3.208910, training acc: 0.955319, training depends: 0.786367, valid loss: 8.006297, valid acc: 0.856437, valid depends: 0.703963\n\n","name":"stdout"},{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 375/375 [02:04<00:00,  3.01it/s, accuracy=0.968, accuracy_depends=0.832, cost=3.38] \ntest minibatch loop: 100%|██████████| 120/120 [00:18<00:00,  6.55it/s, accuracy=0.911, accuracy_depends=0.806, cost=5.37]\ntrain minibatch loop:   0%|          | 0/375 [00:00<?, ?it/s]","name":"stderr"},{"output_type":"stream","text":"epoch: 7, training loss: 2.692029, training acc: 0.963750, training depends: 0.797338, valid loss: 8.316906, valid acc: 0.855605, valid depends: 0.705780\n\n","name":"stdout"},{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 375/375 [02:05<00:00,  2.99it/s, accuracy=0.975, accuracy_depends=0.847, cost=2.66] \ntest minibatch loop: 100%|██████████| 120/120 [00:18<00:00,  6.64it/s, accuracy=0.915, accuracy_depends=0.806, cost=5.85]\ntrain minibatch loop:   0%|          | 0/375 [00:00<?, ?it/s]","name":"stderr"},{"output_type":"stream","text":"epoch: 8, training loss: 2.280745, training acc: 0.969272, training depends: 0.806022, valid loss: 8.649180, valid acc: 0.855627, valid depends: 0.704731\n\n","name":"stdout"},{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 375/375 [02:06<00:00,  2.97it/s, accuracy=0.979, accuracy_depends=0.844, cost=2.18] \ntest minibatch loop: 100%|██████████| 120/120 [00:18<00:00,  6.64it/s, accuracy=0.907, accuracy_depends=0.814, cost=6.05]\ntrain minibatch loop:   0%|          | 0/375 [00:00<?, ?it/s]","name":"stderr"},{"output_type":"stream","text":"epoch: 9, training loss: 1.914802, training acc: 0.974550, training depends: 0.812909, valid loss: 8.974008, valid acc: 0.855271, valid depends: 0.705615\n\n","name":"stdout"},{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 375/375 [02:04<00:00,  3.00it/s, accuracy=0.984, accuracy_depends=0.847, cost=1.87] \ntest minibatch loop: 100%|██████████| 120/120 [00:18<00:00,  6.57it/s, accuracy=0.911, accuracy_depends=0.81, cost=6.58] \ntrain minibatch loop:   0%|          | 0/375 [00:00<?, ?it/s]","name":"stderr"},{"output_type":"stream","text":"epoch: 10, training loss: 1.551014, training acc: 0.980733, training depends: 0.820803, valid loss: 9.371287, valid acc: 0.854632, valid depends: 0.704477\n\n","name":"stdout"},{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 375/375 [02:05<00:00,  2.98it/s, accuracy=0.983, accuracy_depends=0.854, cost=1.65] \ntest minibatch loop: 100%|██████████| 120/120 [00:18<00:00,  6.45it/s, accuracy=0.903, accuracy_depends=0.818, cost=6.59]\ntrain minibatch loop:   0%|          | 0/375 [00:00<?, ?it/s]","name":"stderr"},{"output_type":"stream","text":"epoch: 11, training loss: 1.224350, training acc: 0.986435, training depends: 0.829505, valid loss: 9.807080, valid acc: 0.853969, valid depends: 0.706970\n\n","name":"stdout"},{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 375/375 [02:06<00:00,  2.97it/s, accuracy=0.988, accuracy_depends=0.86, cost=1.34]  \ntest minibatch loop: 100%|██████████| 120/120 [00:18<00:00,  6.54it/s, accuracy=0.899, accuracy_depends=0.818, cost=7.63]\ntrain minibatch loop:   0%|          | 0/375 [00:00<?, ?it/s]","name":"stderr"},{"output_type":"stream","text":"epoch: 12, training loss: 0.981612, training acc: 0.989736, training depends: 0.837044, valid loss: 10.443762, valid acc: 0.851969, valid depends: 0.703867\n\n","name":"stdout"},{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 375/375 [02:06<00:00,  2.97it/s, accuracy=0.993, accuracy_depends=0.854, cost=1.17] \ntest minibatch loop: 100%|██████████| 120/120 [00:18<00:00,  6.52it/s, accuracy=0.911, accuracy_depends=0.826, cost=6.97]\ntrain minibatch loop:   0%|          | 0/375 [00:00<?, ?it/s]","name":"stderr"},{"output_type":"stream","text":"epoch: 13, training loss: 0.779776, training acc: 0.992719, training depends: 0.843190, valid loss: 10.895637, valid acc: 0.851806, valid depends: 0.703673\n\n","name":"stdout"},{"output_type":"stream","text":"train minibatch loop: 100%|██████████| 375/375 [02:05<00:00,  2.99it/s, accuracy=0.995, accuracy_depends=0.856, cost=0.899]\ntest minibatch loop: 100%|██████████| 120/120 [00:18<00:00,  6.52it/s, accuracy=0.907, accuracy_depends=0.81, cost=7.13] ","name":"stderr"},{"output_type":"stream","text":"epoch: 14, training loss: 0.616768, training acc: 0.994896, training depends: 0.848271, valid loss: 11.158718, valid acc: 0.853316, valid depends: 0.704753\n\n","name":"stdout"},{"output_type":"stream","text":"\n","name":"stderr"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"def evaluate(heads_pred, types_pred, heads, types, lengths,\n             symbolic_root=False, symbolic_end=False):\n    batch_size, _ = words.shape\n    ucorr = 0.\n    lcorr = 0.\n    total = 0.\n    ucomplete_match = 0.\n    lcomplete_match = 0.\n\n    corr_root = 0.\n    total_root = 0.\n    start = 1 if symbolic_root else 0\n    end = 1 if symbolic_end else 0\n    for i in range(batch_size):\n        ucm = 1.\n        lcm = 1.\n        for j in range(start, lengths[i] - end):\n\n            total += 1\n            if heads[i, j] == heads_pred[i, j]:\n                ucorr += 1\n                if types[i, j] == types_pred[i, j]:\n                    lcorr += 1\n                else:\n                    lcm = 0\n            else:\n                ucm = 0\n                lcm = 0\n\n            if heads[i, j] == 0:\n                total_root += 1\n                corr_root += 1 if heads_pred[i, j] == 0 else 0\n\n        ucomplete_match += ucm\n        lcomplete_match += lcm\n\n    return (ucorr, lcorr, total, ucomplete_match, lcomplete_match), \\\n           (corr_root, total_root), batch_size","execution_count":20,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tags_seq, heads = sess.run(\n    [model.tags_seq, model.heads],\n    feed_dict = {\n        model.word_ids: batch_x,\n        model.char_ids: batch_char\n    },\n)\ntags_seq[0], heads[0], batch_depends[0]","execution_count":21,"outputs":[{"output_type":"execute_result","execution_count":21,"data":{"text/plain":"(array([40,  6, 22, 26, 23, 18, 16,  5,  3, 13, 10, 11,  6, 12, 13, 10, 16,\n         7,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n         0], dtype=int32),\n array([ 2,  8,  5,  5,  2,  8,  8,  0, 11, 11,  8, 14, 14,  8, 16, 14, 14,\n         8,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n         0]),\n array([ 2,  8,  5,  5,  2,  8,  8,  0, 11, 11,  8, 14, 14,  8, 16, 14, 14,\n         8,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n         0], dtype=int32))"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"def evaluate(heads_pred, types_pred, heads, types, lengths,\n             symbolic_root=False, symbolic_end=False):\n    batch_size, _ = heads_pred.shape\n    ucorr = 0.\n    lcorr = 0.\n    total = 0.\n    ucomplete_match = 0.\n    lcomplete_match = 0.\n\n    corr_root = 0.\n    total_root = 0.\n    start = 1 if symbolic_root else 0\n    end = 1 if symbolic_end else 0\n    for i in range(batch_size):\n        ucm = 1.\n        lcm = 1.\n        for j in range(start, lengths[i] - end):\n\n            total += 1\n            if heads[i, j] == heads_pred[i, j]:\n                ucorr += 1\n                if types[i, j] == types_pred[i, j]:\n                    lcorr += 1\n                else:\n                    lcm = 0\n            else:\n                ucm = 0\n                lcm = 0\n\n            if heads[i, j] == 0:\n                total_root += 1\n                corr_root += 1 if heads_pred[i, j] == 0 else 0\n\n        ucomplete_match += ucm\n        lcomplete_match += lcm\n    \n    return ucorr / total, lcorr / total, corr_root / total_root","execution_count":22,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"arc_accuracy, type_accuracy, root_accuracy = evaluate(heads, tags_seq, batch_depends, batch_y, \n        np.count_nonzero(batch_x, axis = 1))\narc_accuracy, type_accuracy, root_accuracy","execution_count":23,"outputs":[{"output_type":"execute_result","execution_count":23,"data":{"text/plain":"(0.8097165991902834, 0.7692307692307693, 0.8125)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"arcs, types, roots = [], [], []\n\npbar = tqdm(\n    range(0, len(test_X), batch_size), desc = 'test minibatch loop'\n)\nfor i in pbar:\n    index = min(i + batch_size, len(test_X))\n    batch_x = test_X[i: index]\n    batch_x = pad_sequences(batch_x,padding='post')\n    batch_char = test_char[i: index]\n    batch_char = generate_char_seq(batch_char)\n    batch_y = test_Y[i: index]\n    batch_y = pad_sequences(batch_y,padding='post')\n    batch_depends = test_depends[i: index]\n    batch_depends = pad_sequences(batch_depends,padding='post')\n    \n    tags_seq, heads = sess.run(\n        [model.tags_seq, model.heads],\n        feed_dict = {\n            model.word_ids: batch_x,\n            model.char_ids: batch_char\n        },\n    )\n    \n    arc_accuracy, type_accuracy, root_accuracy = evaluate(heads, tags_seq, batch_depends, batch_y, \n            np.count_nonzero(batch_x, axis = 1))\n    pbar.set_postfix(arc_accuracy = arc_accuracy, type_accuracy = type_accuracy, \n                     root_accuracy = root_accuracy)\n    arcs.append(arc_accuracy)\n    types.append(type_accuracy)\n    roots.append(root_accuracy)","execution_count":24,"outputs":[{"output_type":"stream","text":"test minibatch loop: 100%|██████████| 120/120 [00:16<00:00,  7.09it/s, arc_accuracy=0.81, root_accuracy=0.812, type_accuracy=0.769] \n","name":"stderr"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('arc accuracy:', np.mean(arcs))\nprint('types accuracy:', np.mean(types))\nprint('root accuracy:', np.mean(roots))","execution_count":25,"outputs":[{"output_type":"stream","text":"arc accuracy: 0.7047525228808802\ntypes accuracy: 0.6518708550802323\nroot accuracy: 0.6640625\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}