step4_apply_to_entire_text.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "metadata": {
    "collapsed": true,
    "id": "initial_id",
    "ExecuteTime": {
     "end_time": "2024-05-09T12:03:59.526919Z",
     "start_time": "2024-05-09T12:03:59.523451Z"
    }
   },
   "source": [
    "import json\n",
    "import os.path\n",
    "from typing import Union\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "from kediff_ner_system import KediffNerSystem"
   ],
   "id": "initial_id",
   "outputs": [],
   "execution_count": 27
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "id": "cc71f8550aacb4a4"
   },
   "source": [
    "# Load things"
   ],
   "id": "cc71f8550aacb4a4"
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "242ed9de"
   },
   "source": [
    "## Define `DATA_DIR`"
   ],
   "id": "242ed9de"
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "275f01f14b3aeed1",
    "outputId": "222a789a-43c7-4245-a1ed-4140a6d86011",
    "ExecuteTime": {
     "end_time": "2024-05-09T12:04:00.188044Z",
     "start_time": "2024-05-09T12:04:00.185028Z"
    }
   },
   "source": [
    "DATA_DIR: str\n",
    "try:\n",
    "    from google.colab import drive\n",
    "\n",
    "    print(\n",
    "        \"You work on Colab. Gentle as we are, we will mount Drive for you. \"\n",
    "        \"It'd help if you allowed this in the popup that opens.\"\n",
    "    )\n",
    "    drive.mount('/content/drive')\n",
    "    DATA_DIR = os.path.join('drive', 'MyDrive', 'KEDiff', 'data')\n",
    "except ModuleNotFoundError:\n",
    "    print(\"You do not work on Colab\")\n",
    "    DATA_DIR = os.path.join('data')\n",
    "print(f\"{DATA_DIR=}\", '-->', os.path.abspath(DATA_DIR))"
   ],
   "id": "275f01f14b3aeed1",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You do not work on Colab\n",
      "DATA_DIR='data' --> /Users/daniel/source/repos/lelvilamp/kediff-ner-training/data\n"
     ]
    }
   ],
   "execution_count": 28
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "id": "a1a91c9859dc7bd9"
   },
   "source": [
    "## Import NER Ensemble _`kners`_"
   ],
   "id": "a1a91c9859dc7bd9"
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "G7y8j8W27Xgh",
    "outputId": "d0bfddf5-aeb3-427e-921c-d3bfd5064770",
    "ExecuteTime": {
     "end_time": "2024-05-09T12:04:00.817799Z",
     "start_time": "2024-05-09T12:04:00.813102Z"
    }
   },
   "source": [
    "try:\n",
    "    from google.colab import drive\n",
    "\n",
    "    print(\"You work on Colab, so 'kediff_ner_system.py' needs to be copied over before importing it.\")\n",
    "\n",
    "    import shutil\n",
    "    shutil.copy(\n",
    "        src = os.path.join('drive', 'MyDrive', 'KEDiff', 'kediff_ner_system.py'),\n",
    "        dst = '.'\n",
    "    )\n",
    "except ModuleNotFoundError:\n",
    "    pass"
   ],
   "id": "G7y8j8W27Xgh",
   "outputs": [],
   "execution_count": 29
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "mVMR9J5I772j",
    "ExecuteTime": {
     "end_time": "2024-05-09T12:04:01.130285Z",
     "start_time": "2024-05-09T12:04:01.127498Z"
    }
   },
   "source": [
    "import kediff_ner_system as kners"
   ],
   "id": "mVMR9J5I772j",
   "outputs": [],
   "execution_count": 30
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 769
    },
    "id": "ac5576e56aeb517c",
    "outputId": "1d55e556-7606-42d2-d92c-75f00b918b13",
    "ExecuteTime": {
     "end_time": "2024-05-09T12:04:01.314149Z",
     "start_time": "2024-05-09T12:04:01.310712Z"
    }
   },
   "source": [
    "MODELS_DIR: str = os.path.join(DATA_DIR, 'trained_models')\n",
    "MODELS_DIR"
   ],
   "id": "ac5576e56aeb517c",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'data/trained_models'"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 31
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "517cb064d6c4684d",
    "outputId": "8070d590-3ef7-4cd9-a281-18cde79ccdd7",
    "ExecuteTime": {
     "end_time": "2024-05-09T12:04:01.474058Z",
     "start_time": "2024-05-09T12:04:01.471479Z"
    }
   },
   "source": [
    "classifier_paths: dict[str, str] = {\n",
    "    entity_type: os.path.join(MODELS_DIR,\n",
    "                              kners.CLASSIFIER_MODEL_VERSION,\n",
    "                              \"\".join([kners.CLASSIFIER_NAME_BASE, entity_type]),\n",
    "                              kners.SELECTED_EPOCHS[entity_type])\n",
    "    for entity_type in kners.ENTITY_TYPES\n",
    "}\n",
    "classifier_paths"
   ],
   "id": "517cb064d6c4684d",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'EVENT': 'data/trained_models/2024-01-15/oalz-1788-q1-ner-EVENT/checkpoint-1393',\n",
       " 'LOC': 'data/trained_models/2024-01-15/oalz-1788-q1-ner-LOC/checkpoint-1393',\n",
       " 'MISC': 'data/trained_models/2024-01-15/oalz-1788-q1-ner-MISC/checkpoint-2786',\n",
       " 'ORG': 'data/trained_models/2024-01-15/oalz-1788-q1-ner-ORG/checkpoint-1393',\n",
       " 'PER': 'data/trained_models/2024-01-15/oalz-1788-q1-ner-PER/checkpoint-2786',\n",
       " 'TIME': 'data/trained_models/2024-01-15/oalz-1788-q1-ner-TIME/checkpoint-1393'}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 32
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "cf8e7cf1",
    "outputId": "83262473-b12a-428b-f3aa-e06e25e97f62",
    "ExecuteTime": {
     "end_time": "2024-05-09T12:04:01.693260Z",
     "start_time": "2024-05-09T12:04:01.690012Z"
    }
   },
   "source": [
    "{k: os.path.abspath(v) for k, v in classifier_paths.items()}"
   ],
   "id": "cf8e7cf1",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'EVENT': '/Users/daniel/source/repos/lelvilamp/kediff-ner-training/data/trained_models/2024-01-15/oalz-1788-q1-ner-EVENT/checkpoint-1393',\n",
       " 'LOC': '/Users/daniel/source/repos/lelvilamp/kediff-ner-training/data/trained_models/2024-01-15/oalz-1788-q1-ner-LOC/checkpoint-1393',\n",
       " 'MISC': '/Users/daniel/source/repos/lelvilamp/kediff-ner-training/data/trained_models/2024-01-15/oalz-1788-q1-ner-MISC/checkpoint-2786',\n",
       " 'ORG': '/Users/daniel/source/repos/lelvilamp/kediff-ner-training/data/trained_models/2024-01-15/oalz-1788-q1-ner-ORG/checkpoint-1393',\n",
       " 'PER': '/Users/daniel/source/repos/lelvilamp/kediff-ner-training/data/trained_models/2024-01-15/oalz-1788-q1-ner-PER/checkpoint-2786',\n",
       " 'TIME': '/Users/daniel/source/repos/lelvilamp/kediff-ner-training/data/trained_models/2024-01-15/oalz-1788-q1-ner-TIME/checkpoint-1393'}"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 33
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "bf8fa51c1c7d8648",
    "outputId": "cffd73d1-7199-4955-bc78-2333bdd81aa5",
    "ExecuteTime": {
     "end_time": "2024-05-09T12:04:05.817744Z",
     "start_time": "2024-05-09T12:04:01.893125Z"
    }
   },
   "source": "ner: KediffNerSystem = kners.KediffNerSystem(classifier_paths, print_debug_messages_to_console=True)",
   "id": "bf8fa51c1c7d8648",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading tokeniser 'dbmdz/bert-base-historic-multilingual-cased'\n",
      "Initialising models. Received paths to 6 classifiers\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6/6 [00:03<00:00,  1.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Class initialised\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "execution_count": 34
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "id": "225969368eba1a48"
   },
   "source": [
    "## Import Text and Ground Truth"
   ],
   "id": "225969368eba1a48"
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "d202594e161d56d6",
    "outputId": "41dd6d2d-639f-45a6-d3e2-c3cb69161a24",
    "ExecuteTime": {
     "end_time": "2024-05-09T12:04:05.847463Z",
     "start_time": "2024-05-09T12:04:05.819570Z"
    }
   },
   "source": [
    "with open(os.path.join(DATA_DIR, 'union_dataset.jsonl')) as file:\n",
    "    lines: list[dict[str, Union[int, list[Union[int, str]], str]]] = [json.loads(line) for line in file]\n",
    "len(lines)"
   ],
   "id": "d202594e161d56d6",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13928"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 35
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "0beadc67",
    "outputId": "78f6b4c6-12e5-406c-e1bf-d529f2069cc5",
    "ExecuteTime": {
     "end_time": "2024-05-09T12:04:05.850669Z",
     "start_time": "2024-05-09T12:04:05.848498Z"
    }
   },
   "source": [
    "line: dict[str, Union[int, list[Union[int, str]], str]] = lines[42]\n",
    "line"
   ],
   "id": "0beadc67",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 43,\n",
       " 'text': 'der gemeine Mann z. B., wie seine Lage bioher beschaffen war, noch itzt beschaffen ist, und wahrscheinlicher Weise noch mehrere Jahrhunderte hindurch beschaffen seyn wird, fühlt gewiß mehr Ruhe dabey, wenn er gewisse Wahrheiten auf das Ansehen seiner vernünftigern, oder vernünftiger seyn sollenden Lehrer bloß glaubt, als wenn es ihm einfiele, sie ohne alle Hülfskenntnisse selbst zu prüfen.)',\n",
       " 'label': []}"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 36
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7add258b",
    "outputId": "30ed4969-750b-46ad-dd0b-d467ea114d04",
    "ExecuteTime": {
     "end_time": "2024-05-09T12:04:06.381145Z",
     "start_time": "2024-05-09T12:04:05.851892Z"
    }
   },
   "source": "predicted: list[dict[str, Union[float, int, str]]] = ner.ner(line['text'], print_table_to_console=True)",
   "id": "7add258b",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(no entities found)\n"
     ]
    }
   ],
   "execution_count": 37
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "465fb28c"
   },
   "source": [
    "# Apply kNERs to entire file"
   ],
   "id": "465fb28c"
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LwpP_yAiGGJy"
   },
   "source": [
    "## Generate Predictions"
   ],
   "id": "LwpP_yAiGGJy"
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "a3dfc25b",
    "outputId": "50807ddc-4b4c-41f2-806d-f90545788dde",
    "ExecuteTime": {
     "end_time": "2024-05-09T12:04:06.384123Z",
     "start_time": "2024-05-09T12:04:06.381968Z"
    }
   },
   "source": [
    "{\n",
    "    'id': line['id'],\n",
    "    'text': line['text'],\n",
    "    'ground_truth': line['label'],\n",
    "    'predicted': predicted\n",
    "}"
   ],
   "id": "a3dfc25b",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 43,\n",
       " 'text': 'der gemeine Mann z. B., wie seine Lage bioher beschaffen war, noch itzt beschaffen ist, und wahrscheinlicher Weise noch mehrere Jahrhunderte hindurch beschaffen seyn wird, fühlt gewiß mehr Ruhe dabey, wenn er gewisse Wahrheiten auf das Ansehen seiner vernünftigern, oder vernünftiger seyn sollenden Lehrer bloß glaubt, als wenn es ihm einfiele, sie ohne alle Hülfskenntnisse selbst zu prüfen.)',\n",
       " 'ground_truth': [],\n",
       " 'predicted': []}"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 38
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "fe7962db",
    "ExecuteTime": {
     "end_time": "2024-05-09T12:04:06.386503Z",
     "start_time": "2024-05-09T12:04:06.384927Z"
    }
   },
   "source": [
    "# todo remove this later\n",
    "# lines = lines[:50]"
   ],
   "id": "fe7962db",
   "outputs": [],
   "execution_count": 39
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "background_save": true,
     "base_uri": "https://localhost:8080/"
    },
    "id": "0b16d92a",
    "outputId": "97c1e866-eee7-40de-dc26-4f045192a10b",
    "jupyter": {
     "is_executing": true
    },
    "ExecuteTime": {
     "start_time": "2024-05-09T12:04:06.387224Z"
    }
   },
   "source": [
    "for line in tqdm(lines):\n",
    "    line['predicted'] = ner(line['text'])"
   ],
   "id": "0b16d92a",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 9/13928 [00:01<48:02,  4.83it/s]"
     ]
    }
   ],
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "d1Bux6xjGJMh"
   },
   "source": [
    "## Materialise as JSONL file"
   ],
   "id": "d1Bux6xjGJMh"
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "GxiN11enGvFw"
   },
   "source": [
    "The `json` module cannot materialise the `numpy.float32` data types which is used for the predictions' `score` information. Thus, convert it to the base data type `float`."
   ],
   "id": "GxiN11enGvFw"
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "ZFxA60jADwzt",
    "outputId": "5e70fefc-8ec9-405c-ec41-2d9b6a0dc3fc",
    "jupyter": {
     "is_executing": true
    }
   },
   "source": [
    "lines[2]"
   ],
   "id": "ZFxA60jADwzt",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "rUqGitPkESOb",
    "outputId": "db51e616-e59f-4a1e-a30e-255dfff0303a",
    "jupyter": {
     "is_executing": true
    }
   },
   "source": [
    "for key in lines[2]['predicted'][0].keys():\n",
    "    print(type(lines[2]['predicted'][0][key]))"
   ],
   "id": "rUqGitPkESOb",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "vZNeEKcnExVn",
    "outputId": "ea192bb3-580c-4d90-c7a6-3f3ea49b52f2",
    "jupyter": {
     "is_executing": true
    }
   },
   "source": [
    "lines[2]['predicted'][0]['score'].astype(float)"
   ],
   "id": "vZNeEKcnExVn",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "6yYRmusbFOnM",
    "outputId": "2002eee7-4d11-41aa-a8ac-0c4482b68fbf",
    "jupyter": {
     "is_executing": true
    }
   },
   "source": [
    "for line in tqdm(lines):\n",
    "    for prediction in line['predicted']:\n",
    "        prediction['score'] = prediction['score'].astype(float)"
   ],
   "id": "6yYRmusbFOnM",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "bn8fdCMcEB75",
    "outputId": "9265cd29-4d4d-4c80-9d80-08216921e98b",
    "jupyter": {
     "is_executing": true
    }
   },
   "source": [
    "json.dumps(lines[2])"
   ],
   "id": "bn8fdCMcEB75",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "8870fede",
    "outputId": "a11d9c2b-3baf-4868-aa7b-7cecbb8df90d",
    "jupyter": {
     "is_executing": true
    }
   },
   "source": [
    "with open(os.path.join(DATA_DIR, 'predictions.jsonl'), \"w\") as file:\n",
    "    for line in tqdm(lines):\n",
    "        file.write(json.dumps(line) + \"\\n\")"
   ],
   "id": "8870fede",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "1mBnPgg3Fifr"
   },
   "source": [],
   "id": "1mBnPgg3Fifr",
   "outputs": [],
   "execution_count": null
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [
    "cc71f8550aacb4a4",
    "242ed9de",
    "a1a91c9859dc7bd9",
    "225969368eba1a48"
   ],
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}