flair-tiger-pos-tagging.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Predict German POS (TIGER corpus) with FLAIR ",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rf4cARh7Nt1N"
      },
      "source": [
        "! pip install flair"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gXuqFnuJPJDa"
      },
      "source": [
        "# Load data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JwGuMVL_SNdk"
      },
      "source": [
        "! wget https://www.ims.uni-stuttgart.de/documents/ressourcen/korpora/tiger-corpus/download/tigercorpus-2.2.conll09.tar.gz\n",
        "! tar -xvzf tigercorpus-2.2.conll09.tar.gz\n",
        "! ls"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NIrr-1ANoo_H"
      },
      "source": [
        "! wget https://sites.google.com/site/empirist2015/home/shared-task-data/empirist_gold_cmc.zip\n",
        "! wget https://sites.google.com/site/empirist2015/home/shared-task-data/empirist_gold_web.zip\n",
        "\n",
        "! unzip empirist_gold_cmc.zip\n",
        "! unzip empirist_gold_web.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S4c06hglpIrV"
      },
      "source": [
        "ls"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "feF5Z1Dpr-j4"
      },
      "source": [
        "from flair.data import Corpus\n",
        "from flair.datasets import ColumnCorpus\n",
        "\n",
        "\n",
        "# set seed to always get the same data splits \n",
        "import random\n",
        "random.seed(0)\n",
        "\n",
        "# define columns\n",
        "columns = {0: 'id', 1: 'text', 2: 'lemma', 3: 'placeholder', 4: 'pos'}\n",
        "\n",
        "# this is the folder in which train, test and dev files reside\n",
        "data_folder = '.'\n",
        "\n",
        "split = 0.7"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gEqUOzNQNrdo"
      },
      "source": [
        "# retrieve corpus using column format, data folder and the names of the train, dev and test files\n",
        "corpus: Corpus = ColumnCorpus(data_folder, columns, train_file='tiger_release_aug07.corrected.16012013.conll09',)\n",
        "#                                                            test_file='test.txt',\n",
        "#                                                            dev_file='dev.txt')\n",
        "\n",
        "corpus: Corpus = corpus.downsample(split, downsample_train=True, downsample_dev=False, downsample_test=True)\n",
        "corpus.obtain_statistics()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uxYvSsqBRbnn"
      },
      "source": [
        "print('EXAMPLE SEQUENCE', corpus.test[1].to_tagged_string('pos'))\n",
        "print('# TRAINING SEQUENCE', len(corpus.train))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HJRKm4s1PwtP"
      },
      "source": [
        "# Train Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "soIO2Ng3RoWQ"
      },
      "source": [
        "from flair.embeddings import TokenEmbeddings, WordEmbeddings, StackedEmbeddings, FlairEmbeddings\n",
        "from torch.optim.adam import Adam\n",
        "from typing import List\n",
        "\n",
        "from flair.models import SequenceTagger\n",
        "\n",
        "from flair.trainers import ModelTrainer\n",
        "from flair.training_utils import EvaluationMetric\n",
        "\n",
        "from flair.visual.training_curves import Plotter"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pkvdsJwgI3C2"
      },
      "source": [
        "# 1. get the corpus\n",
        "\n",
        "\n",
        "# 2. what tag do we want to predict?\n",
        "tag_type = 'pos'\n",
        "\n",
        "# 3. make the tag dictionary from the corpus\n",
        "tag_dictionary = corpus.make_tag_dictionary(tag_type=tag_type)\n",
        "print(tag_dictionary.idx2item)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kcLlFPpDWdwh"
      },
      "source": [
        "# 4. initialize embeddings\n",
        "embedding_types: List[TokenEmbeddings] = [\n",
        "    # WordEmbeddings('de'),\n",
        "    FlairEmbeddings('german-forward'),\n",
        "    FlairEmbeddings('german-backward'),\n",
        "]\n",
        "\n",
        "embeddings: StackedEmbeddings = StackedEmbeddings(embeddings=embedding_types)\n",
        "\n",
        "# 5. initialize sequence tagger\n",
        "tagger: SequenceTagger = SequenceTagger(hidden_size=256,\n",
        "                                        embeddings=embeddings,\n",
        "                                        tag_dictionary=tag_dictionary,\n",
        "                                        tag_type=tag_type,\n",
        "                                        use_crf=True)\n",
        "\n",
        "# 6. initialize trainer\n",
        "trainer: ModelTrainer = ModelTrainer(tagger, corpus)\n",
        "\n",
        "# 7. start training\n",
        "save_to = '{}/taggers/pos_tiger{}'.format(data_folder, split)\n",
        "trainer.train(save_to,\n",
        "              learning_rate=0.1,\n",
        "              mini_batch_size=32,\n",
        "              max_epochs=20,\n",
        "              embeddings_storage_mode='none',\n",
        "              # checkpoint=True\n",
        "             )\n",
        "\n",
        "# 8. plot training curves (optional)\n",
        "plotter = Plotter()\n",
        "plotter.plot_training_curves(save_to + '/loss.tsv')\n",
        "plotter.plot_weights(save_to + '/weights.txt')\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9bqf4TMSQ0yR"
      },
      "source": [
        "# Evaluation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "alqlyKE0Q8jq"
      },
      "source": [
        "### 1. prepare test data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JUN1VkYf00-2"
      },
      "source": [
        "from sklearn.metrics import classification_report\n",
        "import csv\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import re\n",
        "\n",
        "test_file = open(data_folder + \"/test/test.tsv\", encoding=\"utf-8\")\n",
        "test_cmc_file = open(data_folder + \"/test/cmc_test.tsv\", encoding=\"utf-8\")\n",
        "test_web_file = open(data_folder + \"/test/web_test.tsv\", encoding=\"utf-8\")\n",
        "\n",
        "test = pd.read_csv(test_file, sep=' ', names = [\"word\", \"label\", \"pred\"])\n",
        "cmc_test = pd.read_csv(test_cmc_file, sep='\\t', names = [\"word\", \"label\"])\n",
        "web_test = pd.read_csv(test_web_file, sep='\\t', names = [\"word\", \"label\"])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BIIe6SXDnzg0"
      },
      "source": [
        "! more taggers/pos_tiger0.7/test.tsv"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A1BvTdJ06awm"
      },
      "source": [
        "test[test.label != test.pred].head(5)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DOG5V0tP3und"
      },
      "source": [
        "print(test.shape)\n",
        "print(cmc_test.shape)\n",
        "print(web_test.shape)\n",
        "web_test.head(6)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5ZJ2YfDv7iaU"
      },
      "source": [
        "test_words = test.word.tolist()\n",
        "cmc_words = cmc_test.word.tolist()\n",
        "web_words = web_test.word.tolist()\n",
        "\n",
        "test_labels = test.label.tolist()\n",
        "cmc_labels = cmc_test.label.tolist()\n",
        "web_labels = web_test.label.tolist()\n",
        "\n",
        "print(len(test_labels))\n",
        "print(len(set(test_labels)))\n",
        "labels = list(set(test_labels))\n",
        "print(labels[:10])\n",
        "\n",
        "set_labels_cmc = list(set(cmc_labels))\n",
        "set_labels_web = list(set(web_labels))\n",
        "print(len(set_labels_cmc))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_-jGMc6L3ePI"
      },
      "source": [
        "from sklearn import preprocessing\n",
        "\n",
        "lb = preprocessing.LabelBinarizer(sparse_output=True)\n",
        "lb.fit(set_labels_cmc)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wqxR-gnaFOt3"
      },
      "source": [
        "y_test = lb.transform(test_labels)\n",
        "y_cmc = lb.transform(cmc_labels)\n",
        "y_web = lb.transform(web_labels)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oZfcMAVIF_o2"
      },
      "source": [
        "X_test = \" \".join(test_words)\n",
        "X_cmc = \" \".join(cmc_words)\n",
        "X_web = \" \".join(web_words)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L_NW72CHRG-q"
      },
      "source": [
        "### 2. evaluate the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fnH5npwOzehj"
      },
      "source": [
        "from flair.data import Sentence\n",
        "\n",
        "model_005 = SequenceTagger.load_from_file('{}/taggers/pos_tiger0.05/best-model.pt'.format(data_folder))\n",
        "model_01 = SequenceTagger.load_from_file('{}/taggers/pos_tiger0.1/best-model.pt'.format(data_folder))\n",
        "model_05 = SequenceTagger.load_from_file('{}/taggers/pos_tiger0.5/best-model.pt'.format(data_folder))\n",
        "model_1 = SequenceTagger.load_from_file('{}/taggers/pos_tiger1.0/best-model.pt'.format(data_folder))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0mciNeNDQtay"
      },
      "source": [
        "model = SequenceTagger.load_from_file('{}/taggers/pos_tiger0.5/best-model.pt'.format(data_folder))\n",
        "\n",
        "# create example sentence\n",
        "sentence1 = Sentence('Mit der Ablehnung des Scheidungsabkommens zwischen dem Vereinigten Königreich und der EU dauert die Ungewissheit an .')\n",
        "sentence2 = Sentence('Die zentrale Weichenstellung für das Land dürfte nun weiter aufgeschoben werden .')\n",
        "\n",
        "# predict tags and print\n",
        "model.predict(sentence1)\n",
        "model.predict(sentence2)\n",
        "\n",
        "print(sentence1.to_tagged_string())\n",
        "print(sentence2.to_tagged_string())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ehmCvgbbb8gZ"
      },
      "source": [
        "def predict(test_set, model):\n",
        "  if test_set == \"CMC\":\n",
        "    sentences = Sentence(X_cmc)\n",
        "    preds = model.predict(sentences)\n",
        "    return sentences.to_tagged_string()\n",
        "\n",
        "  if test_set == \"WEB\":   \n",
        "    sentences = Sentence(X_web)\n",
        "    preds = model.predict(sentences)\n",
        "    return sentences.to_tagged_string()\n",
        "\n",
        "  if test_set == \"TEST\":    \n",
        "    sentences = Sentence(X_test)\n",
        "    preds = model.predict(sentences)\n",
        "    return sentences.to_tagged_string()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UszVjkWkeBX3"
      },
      "source": [
        "print(predict(test_set=\"WEB\", model=model_1))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-JapliB5QNf_"
      },
      "source": [
        "def evaluate(test_set, model):\n",
        "  with open(data_folder + \"/test/eval_results/preds{}_tagger{}.txt\".format(test_set, model), \"r\") as fin:\n",
        "    preds_string = fin.read()\n",
        "    preds_list = re.split(\" <|> \", preds_string)\n",
        "    preds_array = np.array(preds_list).reshape(-1,2)\n",
        "    preds_df = pd.DataFrame(preds_array, columns = [\"word\", \"pred\"])\n",
        "    print(preds_df.head(3))\n",
        "    preds_labels = preds_df.pred.tolist()\n",
        "    \n",
        "    if test_set == \"CMC\":\n",
        "      print(len(preds_labels))\n",
        "      print(len(cmc_labels))\n",
        "      assert len(preds_labels) == len(cmc_labels)\n",
        "\n",
        "      y_pred = lb.transform(preds_labels)\n",
        "      print(classification_report(y_cmc, y_pred, target_names=set_labels_cmc))\n",
        "    \n",
        "    if test_set == \"WEB\":\n",
        "      print(len(preds_labels))\n",
        "      print(len(web_labels))\n",
        "      assert len(preds_labels) == len(web_labels)\n",
        "\n",
        "      y_pred = lb.transform(preds_labels)\n",
        "      print(classification_report(y_web, y_pred, target_names=set_labels_web))\n",
        "      \n",
        "    if test_set == \"TEST\":\n",
        "      print(len(preds_labels))\n",
        "      print(len(test_labels))\n",
        "      assert len(preds_labels) == len(test_labels)\n",
        "\n",
        "      y_pred = lb.transform(preds_labels)\n",
        "      print(classification_report(y_test, y_pred, target_names=labels))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "82w3EPv_Y2Wf"
      },
      "source": [
        "\"\"\"before running the function it is important to reassure that the LabelBinarizer\n",
        "is fitted on the \"right\" lable set. CMC and WEB test sets contain less number of\n",
        "lables than the original TEST set. Then y_preds and y_true must be transformed\n",
        "with the correct LabelBinarizer correspondently\"\"\"\n",
        "\n",
        "evaluate(test_set=\"CMC\", model=\"0.1\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ivA_xAlqSgR1"
      },
      "source": [
        "# Tuning the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U7DTv1H1SoXS"
      },
      "source": [
        "from hyperopt import hp\n",
        "from flair.hyperparameter.param_selection import SearchSpace, Parameter\n",
        "\n",
        "from flair.hyperparameter.param_selection import SequenceTaggerParamSelector, OptimizationValue"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ltsk_rDYSyrw"
      },
      "source": [
        "# 1. define your search space\n",
        "search_space = SearchSpace()\n",
        "search_space.add(Parameter.EMBEDDINGS, hp.choice, options=[\n",
        "    [\n",
        "    #WordEmbeddings('de'),\n",
        "    FlairEmbeddings('german-forward', use_cache=True),\n",
        "    FlairEmbeddings('german-backward', use_cache=True)],\n",
        "    #[FlairEmbeddings('german-forward', use_cache=True),\n",
        "    #FlairEmbeddings('german-backward', use_cache=True)]\n",
        "])\n",
        "search_space.add(Parameter.HIDDEN_SIZE, hp.choice, options=[32, 64, 128])\n",
        "#search_space.add(Parameter.RNN_LAYERS, hp.choice, options=[1, 2])\n",
        "#search_space.add(Parameter.DROPOUT, hp.uniform, low=0.0, high=0.5)\n",
        "search_space.add(Parameter.LEARNING_RATE, hp.choice, options=[0.05, 0.1, 0.15, 0.2])\n",
        "search_space.add(Parameter.MINI_BATCH_SIZE, hp.choice, options=[8, 16, 32])\n",
        "\n",
        "# 2. create the parameter selector\n",
        "save_optim_to = '{}/optim_results'.format(data_folder)\n",
        "param_selector = SequenceTaggerParamSelector(\n",
        "    corpus, \n",
        "    tag_type, \n",
        "    save_optim_to, \n",
        "    max_epochs=20,\n",
        "    evaluation_metric = EvaluationMetric.MICRO_F1_SCORE,\n",
        "    training_runs=3,\n",
        "    optimization_value=OptimizationValue.DEV_SCORE\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nyQcjM3pV1AZ"
      },
      "source": [
        "# 3. start the optimization\n",
        "param_selector.optimize(search_space, max_evals=100)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}