Hidden-Markov-Model-NER-tagging.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import conllu as cl\n",
    "from os.path import join\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Quad:\n",
    "    def __init__(self, entity, indexSentence, indexStart, indexEnd):\n",
    "        self.entity = entity\n",
    "        self.indexSentence = indexSentence\n",
    "        self.indexStart = indexStart\n",
    "        self.indexEnd = indexEnd\n",
    "    \n",
    "    def __eq__(self, other):\n",
    "        return (self.entity == other.entity and\n",
    "                self.indexSentence == other.indexSentence and\n",
    "                self.indexStart == other.indexStart and\n",
    "                self.indexEnd == other.indexEnd)\n",
    "    \n",
    "    def __hash__(self):\n",
    "        return hash((self.entity, self.indexSentence, self.indexStart, self.indexEnd))\n",
    "\n",
    "    def toString(self):\n",
    "        return f\"Quad(Entity={self.entity}, indexSentence={self.indexSentence}, indexStart={self.indexStart}, indexEnd={self.indexEnd})\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reconstructSentences(sentences):\n",
    "    modified_sentences = []\n",
    "    for sentence in sentences:\n",
    "        modified_sentence = []\n",
    "        # create a start token\n",
    "        new_token_start = {\n",
    "            \"id\": len(sentence),\n",
    "            \"form\": \"<s>\",\n",
    "            \"lemma\": \"START\",\n",
    "        }\n",
    "        # create a end token\n",
    "        new_token_end = {\n",
    "            \"id\": len(sentence) + 1,\n",
    "            \"form\": \"</s>\",\n",
    "            \"lemma\": \"END\",\n",
    "        }\n",
    "        # append the start token\n",
    "        modified_sentence.append(new_token_start)\n",
    "        # append the rest of the sentence\n",
    "        for word in sentence:\n",
    "            modified_sentence.append(word)\n",
    "        # append the end token\n",
    "        modified_sentence.append(new_token_end)\n",
    "\n",
    "        # append the new sentence to the list of sentences\n",
    "        modified_sentences.append(modified_sentence)\n",
    "\n",
    "    return modified_sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load(language):\n",
    "    filePath = \".\\Datasets\\\\\" + language + \"\\\\\"\n",
    "    \n",
    "    with open(join(filePath, \"train.conllu\"), 'r', encoding='utf-8') as file:\n",
    "        data = file.read()\n",
    "    train_sentences = reconstructSentences(cl.parse(data))\n",
    "\n",
    "    with open(join(filePath, \"test.conllu\"), 'r', encoding='utf-8') as file:\n",
    "        data = file.read()\n",
    "    test_sentences = cl.parse(data)\n",
    "\n",
    "    with open(join(filePath, \"val.conllu\"), 'r', encoding='utf-8') as file:\n",
    "        data = file.read()\n",
    "    val_sentences = cl.parse(data)\n",
    "\n",
    "    return train_sentences, test_sentences, val_sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fit the HMM model with the train sentences, it generates 2 outputps containing the probabilities that will be used \n",
    "# in the decoding phase.\n",
    "def HMMFit(train_sentences):\n",
    "    # dictionary of dictionaries\n",
    "    countTag = {}   # Count(tag(i))\n",
    "    transitionCount = {}    # Count(tag(i-1), tag(i))\n",
    "    wordTagCount = {}   # Count(tag(i), word(i))\n",
    "    tagDistribution = {}    # Count(tag(i-1), tag(i)) / Count(tag(i-1))\n",
    "    wordDistribution = {}   # Count(tag(i), word(i))\n",
    "    uniqueTagDistribution = {}\n",
    "\n",
    "    # complete creation of counTag dictorary\n",
    "    # partial initialization of transitionCount dictionary\n",
    "    # partial creation of wordTagCount dictionary\n",
    "    for sentence in train_sentences:\n",
    "        for word in sentence:\n",
    "            if word['lemma'] in countTag:\n",
    "                countTag[word['lemma']] += 1\n",
    "            else:\n",
    "                countTag[word['lemma']] = 1\n",
    "                uniqueTagDistribution[word['lemma']] = 0\n",
    "                transitionCount[word['lemma']] = {}\n",
    "            \n",
    "            if word['form'] not in wordTagCount:\n",
    "                wordTagCount[word['form']] = {}\n",
    "            if word['lemma'] in wordTagCount[word['form']]:\n",
    "                wordTagCount[word['form']][word['lemma']] += 1\n",
    "            else:\n",
    "                wordTagCount[word['form']][word['lemma']] = 1\n",
    "    \n",
    "    # complete inizialization of transitionCount dictionary\n",
    "    for key in transitionCount.keys():\n",
    "        for tag in countTag.keys():\n",
    "            transitionCount[key][tag] = 0\n",
    "\n",
    "    # complete creation of wordTagCount dictionary\n",
    "    for word in wordTagCount.keys():\n",
    "        for tag in countTag.keys():\n",
    "            if tag not in wordTagCount[word]:\n",
    "                wordTagCount[word][tag] = 0\n",
    "\n",
    "    # complete creation of transitionCount dictionary\n",
    "    # previousTag --> Markov's hypothesis of first grade\n",
    "    for tag in transitionCount.keys():\n",
    "        for sentence in train_sentences:\n",
    "            previousTag = None\n",
    "            for word in sentence:\n",
    "                if previousTag == None:\n",
    "                    previousTag = word['lemma']\n",
    "                else:\n",
    "                    if word['lemma'] == tag:\n",
    "                        if previousTag in transitionCount[tag]:\n",
    "                            transitionCount[tag][previousTag] += 1\n",
    "                    previousTag = word['lemma']\n",
    "    \n",
    "    # complete creation of tagDistribution dictionary\n",
    "    # we use the log-probability for better accuracy (since the probability are very small)\n",
    "    # if the probability is 0, then the log-probability will be log(10**-10) \n",
    "    epsilon = 10**-10\n",
    "    for eventTag in countTag.keys():\n",
    "        tagDistribution[eventTag] = {}\n",
    "        for condTag in countTag.keys():\n",
    "            if transitionCount[eventTag][condTag]/countTag[condTag] == 0:\n",
    "                tagDistribution[eventTag][condTag] = math.log(epsilon)\n",
    "            else:\n",
    "                tagDistribution[eventTag][condTag] = math.log(transitionCount[eventTag][condTag]/countTag[condTag])\n",
    "            \n",
    "    # complete creation of wordDistribution dictionary\n",
    "    # we use the log-probability for better accuracy (since the probability are very small)\n",
    "    # if the probability is 0, then the log-probability will be log(10**-10)\n",
    "    for word in wordTagCount.keys():\n",
    "        wordDistribution[word] = {}\n",
    "\n",
    "        uniqueTagDistribution.pop(\"START\", None)\n",
    "        uniqueTagDistribution.pop(\"END\", None)\n",
    "        if sum(wordTagCount[word][tagLoop] for tagLoop in countTag.keys()) == 1:\n",
    "            tag = [tagLoop for tagLoop in countTag.keys() if wordTagCount[word][tagLoop] == 1]\n",
    "            uniqueTagDistribution[tag[0]] += 1\n",
    "\n",
    "        for tag in countTag.keys():\n",
    "            if wordTagCount[word][tag]/countTag[tag] == 0:\n",
    "                wordDistribution[word][tag] = math.log(epsilon)\n",
    "            else:\n",
    "                wordDistribution[word][tag] = math.log(wordTagCount[word][tag]/countTag[tag])\n",
    "        \n",
    "    return tagDistribution, wordDistribution, wordTagCount, uniqueTagDistribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [],
   "source": [
    "# strategy can be: \n",
    "# 0: always \"O\": P(unk|O) = 1 \n",
    "# 1: always \"O\" and \"MISC\": P(unk|O) = P(unk|B-MISC) = 0.5\n",
    "# 2: uniform: P(unk|tag(i)) = 1 / #(NER_TAGs)\n",
    "# 3: 1 / # of word's tags appearing only once\n",
    "def smoothing(strategy, currentTag, totNERtags, uniqueTagDistribution):\n",
    "    epsilon = 10**-10\n",
    "    match(strategy):\n",
    "        case 0:\n",
    "            if currentTag == \"O\":\n",
    "                return math.log(1)\n",
    "            else:\n",
    "                return math.log(epsilon)\n",
    "        case 1: \n",
    "            if currentTag == \"O\" or currentTag == \"B-MISC\":\n",
    "                return math.log(0.5)\n",
    "            else:\n",
    "                return math.log(epsilon)\n",
    "        case 2:\n",
    "            return math.log(1 / totNERtags)\n",
    "        case 3:\n",
    "            if uniqueTagDistribution[currentTag] != 0:\n",
    "                return math.log(1 / uniqueTagDistribution[currentTag])\n",
    "            else:\n",
    "                math.log(epsilon)\n",
    "        case _:\n",
    "            return math.log(epsilon)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [],
   "source": [
    "def viterbiAlgorithm(sentence, tagDistribution, wordDistribution, b_value, uniqueTagDistribution, strategySmoothing):\n",
    "    epsilon = 10**-10\n",
    "    columns = len(sentence) # words\n",
    "    rows = len(tagDistribution.keys())-2 #tags\n",
    "    viterbi = [[math.log(epsilon) for _ in range(columns)] for _ in range(rows)]\n",
    "    backpointer = [[\"\" for _ in range(columns)] for _ in range(rows)]\n",
    "\n",
    "    # initialization step\n",
    "    i = 0\n",
    "    for tag in tagDistribution.keys():\n",
    "        if tag != \"START\" and tag != \"END\": \n",
    "            viterbi[i][0] = tagDistribution[tag][\"START\"] + b_value\n",
    "            backpointer[i][0] = \"0\"\n",
    "            i += 1\n",
    "\n",
    "    tagList = list(tagDistribution)\n",
    "    tagList.remove(\"START\")\n",
    "    tagList.remove(\"END\")\n",
    "\n",
    "    # recursion step\n",
    "    for i in range(1, len(sentence)):\n",
    "        j = 0\n",
    "        for tag in tagDistribution.keys():\n",
    "            if tag != \"START\" and tag != \"END\":\n",
    "                if (sentence[i])['form'] not in wordDistribution:\n",
    "                    obsLikelihood = smoothing(strategySmoothing, tag, len(tagList), uniqueTagDistribution)\n",
    "                else:\n",
    "                    obsLikelihood = wordDistribution[(sentence[i])['form']][tag]\n",
    "                \n",
    "                viterbi[j][i] = max(viterbi[tagLoop][i-1] + obsLikelihood + tagDistribution[tag][tagList[tagLoop]] for tagLoop in range(len(tagList)))\n",
    "                _, maxTag = max((viterbi[tagLoop][i-1] + tagDistribution[tag][tagList[tagLoop]], tagLoop) for tagLoop in range(len(tagList)))\n",
    "                backpointer[j][i] = maxTag\n",
    "                j += 1\n",
    "\n",
    "    # termitation step\n",
    "    viterbi_end = max(viterbi[tagLoop][len(sentence)-1] + tagDistribution[\"END\"][tagList[tagLoop]] for tagLoop in range(len(tagList)))\n",
    "    _, maxTag = max((viterbi[tagLoop][len(sentence)-1] + tagDistribution[\"END\"][tagList[tagLoop]], tagLoop) for tagLoop in range(len(tagList)))\n",
    "    backpointer_end = maxTag\n",
    "    \n",
    "    # reconstructing the solution\n",
    "    path = [None] * len(sentence)\n",
    "    solution = [None] * len(sentence)\n",
    "    path[len(sentence)-1] = backpointer_end\n",
    "    solution[len(sentence)-1] = tagList[backpointer_end]\n",
    "    for i in range(len(sentence)-2, -1, -1): # decrementing loop, 0 inclusive\n",
    "        path[i] = backpointer[path[i+1]][i+1]\n",
    "        solution[i] = tagList[path[i]]\n",
    "\n",
    "    return solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convertToListEntites(predictions, test_sentences):\n",
    "    listTestSentences = []\n",
    "    currentTag = \"\"\n",
    "    indexSentence = 0\n",
    "    indexStart = 0\n",
    "    indexEnd = 0\n",
    "    for sentence in test_sentences:\n",
    "        for word in sentence:\n",
    "            if word['lemma'] != \"O\": \n",
    "                lemma = word['lemma'].split(\"-\")[1]\n",
    "                if currentTag != lemma:\n",
    "                    if currentTag == \"O\":\n",
    "                        currentTag = lemma\n",
    "                        indexStart = word['id']\n",
    "                    elif currentTag != \"\":\n",
    "                        indexEnd = word['id']-1\n",
    "                        listTestSentences.append(Quad(currentTag, indexSentence, indexStart, indexEnd))\n",
    "                    currentTag = lemma\n",
    "                    indexStart = word['id']\n",
    "            elif word['lemma'] == \"O\" and currentTag != \"\" and currentTag != \"O\":\n",
    "                indexEnd = word['id']-1\n",
    "                listTestSentences.append(Quad(currentTag, indexSentence, indexStart, indexEnd))\n",
    "                currentTag = \"O\"\n",
    "\n",
    "        indexSentence += 1\n",
    "\n",
    "    listPredict = []\n",
    "    currentTag = \"\"\n",
    "    indexSentence = 0\n",
    "    indexStart = 0\n",
    "    indexEnd = 0\n",
    "    for i in range(len(predictions)):\n",
    "        for j in range(len(predictions[i])):\n",
    "            if predictions[i][j] != \"O\":\n",
    "                lemma = predictions[i][j].split(\"-\")[1]\n",
    "                if currentTag != lemma:\n",
    "                    if currentTag == \"O\":\n",
    "                        currentTag = lemma\n",
    "                        indexStart = j\n",
    "                    elif currentTag != \"\":\n",
    "                        indexEnd = j-1\n",
    "                        listPredict.append(Quad(currentTag, i, indexStart, indexEnd))\n",
    "                    currentTag = lemma\n",
    "                    indexStart = j\n",
    "            elif predictions[i][j] == \"O\" and currentTag != \"\" and currentTag != \"O\":\n",
    "                indexEnd = j-1\n",
    "                listPredict.append(Quad(currentTag, i, indexStart, indexEnd))\n",
    "                currentTag = \"O\"\n",
    "    \n",
    "    return listPredict, listTestSentences\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convertListToDict(list):\n",
    "    dict = {}\n",
    "    for quad in list:\n",
    "        key = (quad.entity, quad.indexSentence, quad.indexStart, quad.indexEnd)\n",
    "        dict[key] = quad\n",
    "    \n",
    "    return dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [],
   "source": [
    "def precisionAndRecall(predictions, test_sentences, trueClass):\n",
    "\n",
    "    dict_predictions = convertListToDict(predictions)\n",
    "    dict_testSentences = convertListToDict(test_sentences)    \n",
    "\n",
    "    truePositive = [pred for pred in predictions if pred.entity == trueClass and (pred.entity, pred.indexSentence, pred.indexStart, pred.indexEnd) in dict_testSentences]\n",
    "    falsePositive = [pred for pred in predictions if pred.entity == trueClass and (pred.entity, pred.indexSentence, pred.indexStart, pred.indexEnd) not in dict_testSentences]\n",
    "    falseNegative = [test for test in test_sentences if test.entity == trueClass and (test.entity, test.indexSentence, test.indexStart, test.indexEnd) not in dict_predictions]\n",
    "\n",
    "    precision = len(truePositive) / (len(truePositive) + len(falsePositive)) if (len(truePositive) + len(falsePositive)) > 0 else 0\n",
    "    recall = len(truePositive) / (len(truePositive) + len(falseNegative)) if (len(truePositive) + len(falseNegative)) > 0 else 0\n",
    "\n",
    "    return precision, recall        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluation(predictions, test_sentences, tagDistribution):\n",
    "    totPredictions = 0\n",
    "    correctPredictions = 0\n",
    "    \n",
    "    tagList = list(tagDistribution)\n",
    "    tagList.remove(\"START\")\n",
    "    tagList.remove(\"END\")\n",
    "    tagList.remove(\"O\")\n",
    "    for i in range(len(tagList)):\n",
    "        tagList[i] = tagList[i].split(\"-\")[1]\n",
    "    entity = list(set(tagList))\n",
    "\n",
    "    # accuracy\n",
    "    i = 0\n",
    "    for sentence in test_sentences:\n",
    "        j = 0\n",
    "        for word in sentence:\n",
    "            if word['lemma'] == predictions[i][j]:\n",
    "                correctPredictions += 1\n",
    "            totPredictions += 1\n",
    "            j += 1\n",
    "        i += 1\n",
    "    accuracy = correctPredictions / totPredictions\n",
    "\n",
    "    listPredictions, listTestSentences = convertToListEntites(predictions, test_sentences)\n",
    "    \n",
    "    dictPrecision = {}\n",
    "    dictRecall = {}\n",
    "    for i in range(len(entity)):\n",
    "        precision, recall = precisionAndRecall(listPredictions, listTestSentences, entity[i])\n",
    "        dictPrecision[entity[i]] = precision\n",
    "        dictRecall[entity[i]] = recall\n",
    "\n",
    "    return accuracy, dictPrecision, dictRecall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [],
   "source": [
    "def HMMPredict(test_senteces, tagDistribution, wordDistribution, uniqueTagDistribution, strategySmoothing):\n",
    "    b_value = 0 # in log probabilities 0 is the neutral element\n",
    "    solution = []\n",
    "\n",
    "    for sentence in test_senteces:\n",
    "        sentenceSolution = viterbiAlgorithm(sentence, tagDistribution, wordDistribution, b_value, uniqueTagDistribution, strategySmoothing)\n",
    "        # recontruct and add to the solution...\n",
    "        solution.append(sentenceSolution)\n",
    "\n",
    "    return solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [],
   "source": [
    "def printEvaluation(accuracy, precision, recall):\n",
    "    precisionTot = 0\n",
    "    recallTot = 0\n",
    "    print(\"The model's accuracy is: \" +  str(accuracy))\n",
    "    print()\n",
    "    print(\"Entities' precision and recall:\")\n",
    "    for key in precision.keys():\n",
    "        print(\"Precision \" + key + \": \" + str(precision[key]))\n",
    "        print(\"Recall \" + key + \": \" + str(recall[key]))\n",
    "        precisionTot = precisionTot + precision[key]\n",
    "        recallTot = recallTot + recall[key]\n",
    "        print()\n",
    "\n",
    "    precisionTot = precisionTot / len(precision.keys())\n",
    "    recallTot = recallTot / len(recall.keys())\n",
    "    print(\"Average precision: \" + str(precisionTot))\n",
    "    print(\"Average recall: \" + str(precisionTot))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [],
   "source": [
    "def baselineFrequentTag(wordTagCount, it_test_sentences):\n",
    "    predictions = []\n",
    "    \n",
    "    for sentence in it_test_sentences:\n",
    "        sentencePrediction = []\n",
    "        for word in sentence:\n",
    "            if word['form'] not in wordTagCount:\n",
    "                maxTag = \"B-MISC\"\n",
    "            else:\n",
    "                _, maxTag = max((wordTagCount[word['form']][tagLoop] , tagLoop) for tagLoop in wordTagCount[word['form']].keys())\n",
    "            sentencePrediction.append(maxTag)\n",
    "        predictions.append(sentencePrediction)\n",
    "\n",
    "    return predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [],
   "source": [
    "# it: italian\n",
    "# es: español \n",
    "# en: english\n",
    "language = \"en\"\n",
    "train_sentences, test_sentences, val_sentences =  load(language)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [],
   "source": [
    "tagDistribution, wordDistribution, wordTagCount, uniqueTagDistribution = HMMFit(train_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [],
   "source": [
    "# strategy can be: \n",
    "# 0: always \"O\": P(unk|O) = 1 \n",
    "# 1: always \"O\" and \"MISC\": P(unk|O) = P(unk|B-MISC) = 0.5\n",
    "# 2: uniform: P(unk|tag(i)) = 1 / #(NER_TAGs)\n",
    "# 3: 1 / # of word's tags appearing only once\n",
    "strategySmoothing = 3\n",
    "predictions = HMMPredict(test_sentences, tagDistribution, wordDistribution, uniqueTagDistribution, strategySmoothing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The model's accuracy is: 0.9610675450581123\n",
      "\n",
      "Entities' precision and recall:\n",
      "Precision MISC: 0.7411217242223855\n",
      "Recall MISC: 0.6098347440548166\n",
      "\n",
      "Precision ORG: 0.6668602962532675\n",
      "Recall ORG: 0.6676359406804303\n",
      "\n",
      "Precision LOC: 0.8233612804878049\n",
      "Recall LOC: 0.7282993426597\n",
      "\n",
      "Precision PER: 0.861746847267632\n",
      "Recall PER: 0.708253358925144\n",
      "\n",
      "Average precision: 0.7732725370577724\n",
      "Average recall: 0.7732725370577724\n"
     ]
    }
   ],
   "source": [
    "accuracy, precision, recall = evaluation(predictions, test_sentences, tagDistribution)\n",
    "printEvaluation(accuracy, precision, recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The model's accuracy is: 0.931242162789392\n",
      "\n",
      "Entities' precision and recall:\n",
      "Precision MISC: 0.23050379572118704\n",
      "Recall MISC: 0.6058041112454655\n",
      "\n",
      "Precision ORG: 0.43610755441741356\n",
      "Recall ORG: 0.49520209363186973\n",
      "\n",
      "Precision LOC: 0.625724404378622\n",
      "Recall LOC: 0.6551491656834654\n",
      "\n",
      "Precision PER: 0.5594050084729806\n",
      "Recall PER: 0.5702495201535509\n",
      "\n",
      "Average precision: 0.46293519074755074\n",
      "Average recall: 0.46293519074755074\n"
     ]
    }
   ],
   "source": [
    "predictionsBaseline = baselineFrequentTag(wordTagCount, test_sentences)\n",
    "accuracy, precision, recall = evaluation(predictionsBaseline, test_sentences, tagDistribution)\n",
    "printEvaluation(accuracy, precision, recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'import memm_tagger as memm\\n\\ndata = memm.initialize()\\npredictions = memm.train(\".\\\\Datasets\\\\it\\\\train.conllu\", data)\\nmemm.test(\".\\\\Datasets\\\\it\\\\test.conllu\", predictions, data)'"
      ]
     },
     "execution_count": 240,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"import memm_tagger as memm\n",
    "\n",
    "data = memm.initialize()\n",
    "predictions = memm.train(\".\\Datasets\\\\it\\\\train.conllu\", data)\n",
    "memm.test(\".\\Datasets\\\\it\\\\test.conllu\", predictions, data)\"\"\"\n",
    "\n",
    "# aggiunto file = open(filename, 'r', encoding='utf-8') alla funzione load_data (riga 175)\n",
    "# aggiunto un padding ai token (righe 200-202), altrimenti andava in eccezione (riga 208-210)\n",
    "# MemoryError: Unable to allocate 32.5 GiB for an array with shape (88400, 266) and data type <U371"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}