00 - Get counter examples.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "c535042c",
   "metadata": {},
   "outputs": [],
   "source": [
    "CORPUS_DIR = \"/home/thibault/dev/latin-lemmatized-texts/lemmatized/xml/\"\n",
    "\n",
    "import glob\n",
    "import lxml.etree as ET\n",
    "from typing import List\n",
    "import random\n",
    "import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "4e1380d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2516/2516 [00:00<00:00, 13337.13it/s]\n"
     ]
    }
   ],
   "source": [
    "def get_forbidden_passages(directory: str = \"/home/thibault/dev/these-corpus/data/*.xml\"):\n",
    "    refs = []\n",
    "    sentences = []\n",
    "    for file in tqdm.tqdm(glob.glob(directory)):\n",
    "        xml = ET.parse(file)\n",
    "        urn = xml.xpath(\"//idno[@type='CTS_URN']\")[0].text\n",
    "        ref = xml.xpath(\"//biblScope[@unit='ref']\")[0].text\n",
    "        refs.append(f\"{urn}:{ref}\")\n",
    "        sentences.append(\" \".join(xml.xpath(\"//w[@pos!='PUNC']/text()\")))\n",
    "    return set(refs), \" \".join(list(set(sentences)))\n",
    "known_refs, known_sentences = get_forbidden_passages()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "5e325c3d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 853/853 [01:00<00:00, 14.05it/s]\n"
     ]
    }
   ],
   "source": [
    "def has_other_than_punc(sentence: List[ET.Element]) -> bool:\n",
    "    return len([tok for tok in sentence if tok.attrib[\"pos\"] != \"PUNC\"]) > 0\n",
    "\n",
    "def debug_print(sentence: List[ET.Element]):\n",
    "    print(\" \".join([\n",
    "        w.text.strip()\n",
    "        for w in sentence\n",
    "    ]))\n",
    "\n",
    "def get_urn(sent: List[ET.Element], file_id: str):\n",
    "    ref1, ref2 = sent[0].attrib[\"n\"], sent[-1].attrib[\"n\"]\n",
    "    if ref1 == ref2:\n",
    "        return f\"{file_id}:{ref1}\"\n",
    "    else:\n",
    "        return f\"{file_id}:{ref1}-{ref2}\"\n",
    "    \n",
    "def get_simplified_sentence(sent: List[ET.Element]):\n",
    "    return \" \".join([et.text for et in sent if et.attrib.get(\"pos\") != \"PUNC\"])\n",
    "    \n",
    "def write_sample(sent: List[ET.Element], file_id: str, idx: int):\n",
    "    prefix = \"_\".join(file_id.split(\":\")[2:])\n",
    "    with open(f\"dataset/negative-examples/{prefix}--{idx}.xml\", \"w\") as out:\n",
    "        joined_w = \"\"\"\"\"\".join([ET.tostring(w, encoding=str).replace('xmlns=\"http://www.tei-c.org/ns/1.0\"', \"\") for w in sent])\n",
    "        \n",
    "        out.write(f\"\"\"<div type=\"fragment\" ana=\"#negative-example\">\n",
    "    <bibl type=\"source\">\n",
    "        <idno type=\"CTS_URN\">{file_id}</idno>\n",
    "    </bibl>\n",
    "    <quote xml:lang=\"lat\" source=\"{get_urn(sent, file_id)}\">\n",
    "        {joined_w}\n",
    "    </quote>\n",
    " </div>\"\"\")\n",
    "\n",
    "sample = 0\n",
    "KEEP = 30\n",
    "for file in tqdm.tqdm(glob.glob(CORPUS_DIR+\"*.xml\")):\n",
    "    x = ET.parse(file)\n",
    "    # xmlns=\"http://www.tei-c.org/ns/1.0\"\n",
    "    sentences = [\n",
    "        []\n",
    "    ]\n",
    "    file_id = x.xpath(\"//tei:teiHeader/@n\", namespaces={\"tei\": \"http://www.tei-c.org/ns/1.0\"})[0]\n",
    "    \n",
    "    for w in x.xpath(\"//tei:w\", namespaces={\"tei\": \"http://www.tei-c.org/ns/1.0\"}):\n",
    "            \n",
    "        # Work Around for Vulgate\n",
    "        if \"greekLit\" in file_id:\n",
    "            if len(sentences[-1]) and sentences[-1][-1].attrib[\"n\"] != w.attrib[\"n\"]:\n",
    "                sentences.append([])\n",
    "            sentences[-1].append(w)\n",
    "        else:\n",
    "            sentences[-1].append(w)\n",
    "\n",
    "            if w.attrib[\"pos\"] == \"PUNC\" \\\n",
    "                and w.attrib[\"lemma\"] in {\"?\", \"!\", \"...\", \".\", \";\", \")\"} \\\n",
    "                and len(sentences[-1]) and has_other_than_punc(sentences[-1]):\n",
    "                \n",
    "                sentences.append([])\n",
    "\n",
    "    # Keep sentences larger than 5 words\n",
    "    sentences = [s for s in sentences if len(s) >= 5]\n",
    "    \n",
    "    # Shuffle !\n",
    "    random.shuffle(sentences)\n",
    "    \n",
    "    # Keep 5 ?\n",
    "    \n",
    "    kept_samples = []\n",
    "    while len(kept_samples) < KEEP and len(sentences) > 0:\n",
    "        sent = sentences.pop()\n",
    "        urn = get_urn(sent, file_id)\n",
    "        full_text = get_simplified_sentence(sent)\n",
    "        if urn in known_refs or full_text in known_sentences:\n",
    "            continue\n",
    "        kept_samples.append(sent)\n",
    "    \n",
    "    for idx, sent in enumerate(kept_samples):\n",
    "        write_sample(sent, file_id, idx)\n",
    "    sample += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d8303f7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}