_CLF_BERT_SPARSE_MLP.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "view-in-github"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/zzunebye/Capstone-code-data/blob/main/_CLF_BERT.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Colab"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dQSdijCsmtPH",
        "outputId": "194e4d67-6b6f-4247-f51c-93f2e1fefc07"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.3.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.43)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (3.7.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.10.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (1.8.0+cu101)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torch) (1.19.5)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch) (3.7.4.3)\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers\n",
        "!pip install torch"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 57,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 364
        },
        "id": "4FaLWxNckcCp",
        "outputId": "631efc0a-9468-478e-9910-c8986bc8d05b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/MyDrive/My Drive/Capstone/code_data\n"
          ]
        },
        {
          "ename": "ValueError",
          "evalue": "ignored",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-57-1836b0b99c66>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m# 실행시 등장하는 URL을 클릭하여 허용해주면 인증KEY가 나타난다. 복사하여 URL아래 빈칸에 붙여넣으면 마운트에 성공하게된다.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mgoogle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolab\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdrive\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mdrive\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmount\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'./MyDrive'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/google/colab/drive.py\u001b[0m in \u001b[0;36mmount\u001b[0;34m(mountpoint, force_remount, timeout_ms, use_metadata_server)\u001b[0m\n\u001b[1;32m    254\u001b[0m             \u001b[0;34m': timeout during initial read of root folder; for more info: '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    255\u001b[0m             'https://research.google.com/colaboratory/faq.html#drive-timeout')\n\u001b[0;32m--> 256\u001b[0;31m       \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'mount failed'\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mextra_reason\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    257\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mcase\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    258\u001b[0m       \u001b[0;31m# Not already authorized, so do the authorization dance.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mValueError\u001b[0m: mount failed"
          ]
        }
      ],
      "source": [
        "import os\n",
        "print(os.getcwd())\n",
        "\n",
        "# 실행시 등장하는 URL을 클릭하여 허용해주면 인증KEY가 나타난다. 복사하여 URL아래 빈칸에 붙여넣으면 마운트에 성공하게된다.\n",
        "from google.colab import drive\n",
        "drive.mount('./MyDrive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NTiePp9YlA8R",
        "outputId": "8afeef62-c6b8-4425-f66d-d5c78e5ef155"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[Errno 107] Transport endpoint is not connected: 'MyDrive/MyDrive/Capstone/code_data'\n",
            "/content/MyDrive/MyDrive/Capstone/code_data\n"
          ]
        }
      ],
      "source": [
        "cd MyDrive/MyDrive/Capstone/code_data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ml-UXy1nkI_x"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import re\n",
        "from tqdm import tqdm\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import time\n",
        "\n",
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zNAzuZdm-5eY"
      },
      "source": [
        "# BERT + SPARSE\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9RkOFN0Tpp5s"
      },
      "source": [
        "## Functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BKhXQU5BE2YM"
      },
      "outputs": [],
      "source": [
        "import re\n",
        "from tqdm import tqdm\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import time\n",
        "\n",
        "%matplotlib inline\n",
        "from transformers import BertTokenizer\n",
        "from sklearn.metrics import accuracy_score, roc_curve, auc, classification_report, f1_score\n",
        "\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "import torch.nn.functional as F\n",
        "\n",
        "import time\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from transformers import BertModel\n",
        "from transformers import AdamW, get_linear_schedule_with_warmup\n",
        "\n",
        "import random\n",
        "import time\n",
        "\n",
        "from transformers import BertTokenizer\n",
        "from sklearn.metrics import accuracy_score, roc_curve, auc\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Class"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "class BertClassifier(nn.Module): # Create the BertClassfier class\n",
        "    \"\"\"Bert Model for Classification Tasks.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, freeze_bert=False):\n",
        "        \"\"\"\n",
        "        @param    bert: a BertModel object\n",
        "        @param    classifier: a torch.nn.Module classifier\n",
        "        @param    freeze_bert (bool): Set `False` to fine-tune the BERT model\n",
        "        \"\"\"\n",
        "        super(BertClassifier, self).__init__()\n",
        "        # Specify hidden size of BERT, hidden size of our classifier, and number of labels\n",
        "        D_in, H, D_out = 768, 50, 2\n",
        "\n",
        "        # Instantiate BERT model\n",
        "        self.bert = BertModel.from_pretrained('bert-base-uncased')\n",
        "\n",
        "        # Instantiate an one-layer feed-forward classifier\n",
        "        self.classifier = nn.Sequential(\n",
        "            nn.Linear(D_in, H),\n",
        "            nn.ReLU(),\n",
        "            # nn.Dropout(0.5),\n",
        "            nn.Linear(H, D_out)\n",
        "        )\n",
        "\n",
        "        # Freeze the BERT model\n",
        "        if freeze_bert:\n",
        "            for param in self.bert.parameters():\n",
        "                param.requires_grad = False\n",
        "\n",
        "    def forward(self, input_ids, attention_mask, sparse_features):\n",
        "        \"\"\"\n",
        "        Feed input to BERT and the classifier to compute logits.\n",
        "        @param    input_ids (torch.Tensor): an input tensor with shape (batch_size,\n",
        "                      max_length)\n",
        "        @param    attention_mask (torch.Tensor): a tensor that hold attention mask\n",
        "                      information with shape (batch_size, max_length)\n",
        "        @return   logits (torch.Tensor): an output tensor with shape (batch_size,\n",
        "                      num_labels)\n",
        "        \"\"\"\n",
        "        # Feed input to BERT\n",
        "        outputs = self.bert(input_ids=input_ids,\n",
        "                            attention_mask=attention_mask)\n",
        "\n",
        "        # Extract the last hidden state of the token `[CLS]` for classification task\n",
        "        last_hidden_state_cls = outputs[0][:, 0, :]\n",
        "\n",
        "        # Feed input to classifier to compute logits\n",
        "        logits = self.classifier(last_hidden_state_cls)\n",
        "\n",
        "        return logits"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UgmOuPxRFhof"
      },
      "outputs": [],
      "source": [
        "def getDevice():\n",
        "  if torch.cuda.is_available():       \n",
        "      device = torch.device(\"cuda\")\n",
        "      print(f'There are {torch.cuda.device_count()} GPU(s) available.')\n",
        "      print('Device name:', torch.cuda.get_device_name(0))\n",
        "\n",
        "  else:\n",
        "      print('No GPU available, using the CPU instead.')\n",
        "      device = torch.device(\"cpu\")\n",
        "  return device\n",
        "\n",
        "def evaluate_roc(probs, y_true):\n",
        "    \"\"\"\n",
        "    - Print AUC and accuracy on the test set\n",
        "    - Plot ROC\n",
        "    @params    probs (np.array): an array of predicted probabilities with shape (len(y_true), 2)\n",
        "    @params    y_true (np.array): an array of the true values with shape (len(y_true),)\n",
        "    \"\"\"\n",
        "    preds = probs[:, 1]\n",
        "    fpr, tpr, threshold = roc_curve(y_true, preds)\n",
        "    roc_auc = auc(fpr, tpr)\n",
        "    print(f'AUC: {roc_auc:.4f}')\n",
        "\n",
        "    # Get accuracy over the test set\n",
        "    y_pred = np.where(preds >= 0.5, 1, 0)\n",
        "    accuracy = accuracy_score(y_true, y_pred)\n",
        "    print(f'Accuracy: {accuracy*100:.2f}%')\n",
        "\n",
        "    # Plot ROC AUC\n",
        "    plt.title('Receiver Operating Characteristic')\n",
        "    plt.plot(fpr, tpr, 'b', label='AUC = %0.2f' % roc_auc)\n",
        "    plt.legend(loc='lower right')\n",
        "    plt.plot([0, 1], [0, 1], 'r--')\n",
        "    plt.xlim([0, 1])\n",
        "    plt.ylim([0, 1])\n",
        "    plt.ylabel('True Positive Rate')\n",
        "    plt.xlabel('False Positive Rate')\n",
        "    plt.show()\n",
        "\n",
        "def text_preprocessing(text): # Create a function to tokenize a set of texts\n",
        "\n",
        "    \"\"\"\n",
        "    - Remove entity mentions (eg. '@united')\n",
        "    - Correct errors (eg. '&amp;' to '&')\n",
        "    @param    text (str): a string to be processed.\n",
        "    @return   text (Str): the processed string.\n",
        "    \"\"\"\n",
        "    # Remove '@name'\n",
        "    text = text.lower()\n",
        "    # text = re.sub(r'(@.*?)[\\s]', ' ', text)\n",
        "    # text = re.sub(r\"(#)(\\S+)\", r'\\1 \\2', text)\n",
        "\n",
        "    # text = re.sub(r\"http\\S+\", \"*\", text)  # http link -> '*'\n",
        "\n",
        "    # text = re.sub(r\"@\\S+\", \"@\", text)   # mention -> '@'\n",
        "    # text = re.sub(r\"@[^\\s]+\", \"@\", text)   # mention -> '@'\n",
        "\n",
        "    # sent = re.sub(r\"(#)(\\S+)\", r'\\1 \\2', sent)\n",
        "    # sent = re.sub(r'([^\\s\\w@#\\*]|_)+', '', sent) # Erasing Special Characters\n",
        "\n",
        "    # Replace '&amp;' with '&'\n",
        "    text = re.sub(r'&amp;', '&', text)\n",
        "\n",
        "    # Remove trailing whitespace\n",
        "    text = re.sub(r'\\s+', ' ', text).strip()\n",
        "\n",
        "    return text\n",
        "\n",
        "def preprocessing_for_bert(data): \n",
        "\n",
        "    \"\"\"Perform required preprocessing steps for pretrained BERT.\n",
        "    @param    data (np.array): Array of texts to be processed.\n",
        "    @return   input_ids (torch.Tensor): Tensor of token ids to be fed to a model.\n",
        "    @return   attention_masks (torch.Tensor): Tensor of indices specifying which\n",
        "                  tokens should be attended to by the model.\n",
        "    \"\"\"\n",
        "    # Create empty lists to store outputs (빈 리스트 2개 생성)\n",
        "    input_ids = []\n",
        "    attention_masks = []\n",
        "\n",
        "    # For every sentence...\n",
        "    for sent in data:\n",
        "        # `encode_plus` will:\n",
        "        #    (1) Tokenize the sentence\n",
        "        #    (2) Add the `[CLS]` and `[SEP]` token to the start and end\n",
        "        #    (3) Truncate/Pad sentence to max length\n",
        "        #    (4) Map tokens to their IDs\n",
        "        #    (5) Create attention mask\n",
        "        #    (6) Return a dictionary of outputs\n",
        "        encoded_sent = tokenizer.encode_plus(\n",
        "            text=text_preprocessing(sent),  # Preprocess sentence\n",
        "            add_special_tokens=True,        # Add `[CLS]` and `[SEP]`\n",
        "            # max_length=MAX_LEN,                  # Max length to truncate/pad\n",
        "            pad_to_max_length=True,         # Pad sentence to max length\n",
        "            # return_tensors='pt',           # Return PyTorch tensor\n",
        "            return_attention_mask=True,      # Return attention mask\n",
        "\n",
        "            # max_length=True,                  # Max length to truncate/pad\n",
        "            padding='max_length'\n",
        "        )\n",
        "\n",
        "        # Add the outputs to the lists (위의 빈 리스트에 상응하는 값 추가)\n",
        "        input_ids.append(encoded_sent.get('input_ids'))\n",
        "        attention_masks.append(encoded_sent.get('attention_mask'))\n",
        "\n",
        "    # Convert lists to tensors (리스트들을 텐서화)\n",
        "    input_ids = torch.tensor(input_ids)\n",
        "    attention_masks = torch.tensor(attention_masks)\n",
        "\n",
        "    return input_ids, attention_masks\n",
        "\n",
        "\n",
        "def initialize_model(epochs=4):\n",
        "    \"\"\"Initialize the Bert Classifier, the optimizer and the learning rate scheduler.\n",
        "    \"\"\"\n",
        "    # Instantiate Bert Classifier\n",
        "    bert_classifier = BertClassifier(freeze_bert=False)\n",
        "\n",
        "    # Tell PyTorch to run the model on GPU\n",
        "    bert_classifier.to(device)\n",
        "\n",
        "    # Create the optimizer\n",
        "    optimizer = AdamW(bert_classifier.parameters(),\n",
        "                      lr=5e-5,    # Default learning rate\n",
        "                      eps=1e-8    # Default epsilon value\n",
        "                      )\n",
        "\n",
        "    # Total number of training steps\n",
        "    total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "    criterion = nn.CrossEntropyLoss()\n",
        "\n",
        "    # Set up the learning rate scheduler\n",
        "    scheduler = get_linear_schedule_with_warmup(optimizer,\n",
        "                                                num_warmup_steps=0,  # Default value\n",
        "                                                num_training_steps=total_steps)\n",
        "    return bert_classifier, optimizer, scheduler, criterion\n",
        "\n",
        "def set_seed(seed_value=42):\n",
        "    \"\"\"Set seed for reproducibility.\n",
        "    \"\"\"\n",
        "    random.seed(seed_value)\n",
        "    np.random.seed(seed_value)\n",
        "    torch.manual_seed(seed_value)\n",
        "    torch.cuda.manual_seed_all(seed_value)\n",
        "\n",
        "def train(model, train_dataloader, val_dataloader=None, epochs=4, evaluation=False):\n",
        "    \"\"\"Train the BertClassifier model.\n",
        "    \"\"\"\n",
        "    # Start training loop\n",
        "    print(\"Start training...\\n\")\n",
        "    for epoch_i in range(epochs):\n",
        "        # =======================================\n",
        "        #               Training\n",
        "        # =======================================\n",
        "        # Print the header of the result table\n",
        "        print(f\"{'Epoch':^7} | {'Batch':^7} | {'Train Loss':^12} | {'Val Loss':^10} | {'Val Acc':^9} | {'Elapsed':^9}\")\n",
        "        print(\"-\"*70)\n",
        "\n",
        "        # Measure the elapsed time of each epoch\n",
        "        t0_epoch, t0_batch = time.time(), time.time()\n",
        "\n",
        "        # Reset tracking variables at the beginning of each epoch\n",
        "        total_loss, batch_loss, batch_counts = 0, 0, 0\n",
        "\n",
        "        # Put the model into the training mode\n",
        "        model.train()\n",
        "\n",
        "        # For each batch of training data...\n",
        "        for step, batch in enumerate(train_dataloader):\n",
        "            batch_counts += 1\n",
        "            # Load batch to GPU\n",
        "            b_input_ids, b_attn_mask, b_labels = tuple(\n",
        "                t.to(device) for t in batch)\n",
        "\n",
        "            # Zero out any previously calculated gradients\n",
        "            model.zero_grad()\n",
        "\n",
        "            # Perform a forward pass. This will return logits.\n",
        "            logits = model(b_input_ids, b_attn_mask)\n",
        "\n",
        "            # Compute loss and accumulate the loss values\n",
        "            loss = loss_fn(logits, b_labels)\n",
        "            batch_loss += loss.item()\n",
        "            total_loss += loss.item()\n",
        "\n",
        "            # Perform a backward pass to calculate gradients\n",
        "            loss.backward()\n",
        "\n",
        "            # Clip the norm of the gradients to 1.0 to prevent \"exploding gradients\"\n",
        "            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "            # Update parameters and the learning rate\n",
        "            optimizer.step()\n",
        "            scheduler.step()\n",
        "\n",
        "            # Print the loss values and time elapsed for every 20 batches\n",
        "            if (step % 20 == 0 and step != 0) or (step == len(train_dataloader) - 1):\n",
        "                # Calculate time elapsed for 20 batches\n",
        "                time_elapsed = time.time() - t0_batch\n",
        "\n",
        "                # Print training results\n",
        "                print(\n",
        "                    f\"{epoch_i + 1:^7} | {step:^7} | {batch_loss / batch_counts:^12.6f} | {'-':^10} | {'-':^9} | {time_elapsed:^9.2f}\")\n",
        "\n",
        "                # Reset batch tracking variables\n",
        "                batch_loss, batch_counts = 0, 0\n",
        "                t0_batch = time.time()\n",
        "\n",
        "        # Calculate the average loss over the entire training data\n",
        "        avg_train_loss = total_loss / len(train_dataloader)\n",
        "\n",
        "        print(\"-\"*70)\n",
        "        # =======================================\n",
        "        #               Evaluation\n",
        "        # =======================================\n",
        "        if evaluation == True:\n",
        "            # After the completion of each training epoch, measure the model's performance\n",
        "            # on our validation set.\n",
        "            val_loss, val_accuracy = evaluate(model, val_dataloader)\n",
        "\n",
        "            # Print performance over the entire training data\n",
        "            time_elapsed = time.time() - t0_epoch\n",
        "\n",
        "            print(f\"{epoch_i + 1:^7} | {'-':^7} | {avg_train_loss:^12.6f} | {val_loss:^10.6f} | {val_accuracy:^9.2f} | {time_elapsed:^9.2f}\")\n",
        "            print(\"-\"*70)\n",
        "        print(\"\\n\")\n",
        "\n",
        "    print(\"Training complete!\")\n",
        "\n",
        "def evaluate(model, val_dataloader):\n",
        "    \"\"\"After the completion of each training epoch, measure the model's performance\n",
        "    on our validation set.\n",
        "    \"\"\"\n",
        "    # Put the model into the evaluation mode. The dropout layers are disabled during\n",
        "    # the test time.\n",
        "    model.eval()\n",
        "\n",
        "    # Tracking variables\n",
        "    val_accuracy = []\n",
        "    val_loss = []\n",
        "\n",
        "    # For each batch in our validation set...\n",
        "    for batch in val_dataloader:\n",
        "        # Load batch to GPU\n",
        "        b_input_ids, b_attn_mask, b_labels = tuple(t.to(device) for t in batch)\n",
        "\n",
        "        # Compute logits\n",
        "        with torch.no_grad():\n",
        "            logits = model(b_input_ids, b_attn_mask)\n",
        "\n",
        "        # Compute loss\n",
        "        loss = loss_fn(logits, b_labels)\n",
        "        val_loss.append(loss.item())\n",
        "\n",
        "        # Get the predictions\n",
        "        preds = torch.argmax(logits, dim=1).flatten()\n",
        "\n",
        "        # Calculate the accuracy rate\n",
        "        accuracy = (preds == b_labels).cpu().numpy().mean() * 100\n",
        "        val_accuracy.append(accuracy)\n",
        "\n",
        "    # Compute the average accuracy and loss over the validation set.\n",
        "    val_loss = np.mean(val_loss)\n",
        "    val_accuracy = np.mean(val_accuracy)\n",
        "\n",
        "    return val_loss, val_accuracy\n",
        "\n",
        "def bert_predict(model, test_dataloader):\n",
        "    \"\"\"Perform a forward pass on the trained BERT model to predict probabilities\n",
        "    on the test set.\n",
        "    \"\"\"\n",
        "    # Put the model into the evaluation mode. The dropout layers are disabled during\n",
        "    # the test time.\n",
        "    model.eval()\n",
        "\n",
        "    all_logits = []\n",
        "\n",
        "    # For each batch in our test set...\n",
        "    for batch in test_dataloader:\n",
        "        # Load batch to GPU\n",
        "        b_input_ids, b_attn_mask = tuple(t.to(device) for t in batch)[:2]\n",
        "\n",
        "        # Compute logits\n",
        "        with torch.no_grad():\n",
        "            logits = model(b_input_ids, b_attn_mask)\n",
        "        all_logits.append(logits)\n",
        "    \n",
        "    # Concatenate logits from each batch\n",
        "    all_logits = torch.cat(all_logits, dim=0)\n",
        "\n",
        "    # Apply softmax to calculate probabilities\n",
        "    probs = F.softmax(all_logits, dim=1).cpu().numpy()\n",
        "\n",
        "    return probs\n",
        "\n",
        "def data_process(X_train, y_train, X_val, y_val, batch_size=16):\n",
        "  # Concatenate train data and test data\n",
        "  all_tweets = np.concatenate([X_train, X_val])\n",
        "\n",
        "  # Encode our concatenated data\n",
        "  encoded_tweets = [tokenizer.encode(sent, add_special_tokens=True) for sent in all_tweets]\n",
        "\n",
        "  # Find the maximum length\n",
        "  max_len = max([len(sent) for sent in encoded_tweets])\n",
        "  print('Max length: ', max_len)\n",
        "\n",
        "  # Specify `MAX_LEN`\n",
        "  MAX_LEN = max_len\n",
        "\n",
        "  # Run function `preprocessing_for_bert` on the train set and the validation set\n",
        "  print('\\nTokenizing data...')\n",
        "  train_inputs, train_masks = preprocessing_for_bert(X_train)\n",
        "  val_inputs, val_masks = preprocessing_for_bert(X_val)\n",
        "\n",
        "  # Convert other data types to torch.Tensor\n",
        "  train_labels = torch.tensor(y_train)\n",
        "  val_labels = torch.tensor(y_val)\n",
        "\n",
        "  # For fine-tuning BERT, the authors recommend a batch size of 16 or 32.\n",
        "  batch_size = 8\n",
        "\n",
        "  # Create the DataLoader for our training set\n",
        "  train_data = TensorDataset(train_inputs, train_masks, train_labels)\n",
        "  train_sampler = RandomSampler(train_data)\n",
        "  train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
        "\n",
        "  # Create the DataLoader for our validation set\n",
        "  val_data = TensorDataset(val_inputs, val_masks, val_labels)\n",
        "  val_sampler = SequentialSampler(val_data)\n",
        "  val_dataloader = DataLoader(val_data, sampler=val_sampler, batch_size=batch_size)\n",
        "\n",
        "  return train_dataloader, val_dataloader\n",
        "\n",
        "def train_process(train_dataloader, val_dataloader, epoch=4):\n",
        "  set_seed(42)    # Set seed for reproducibility\n",
        "  bert_classifier, optimizer, scheduler, loss_fn = initialize_model(epochs=epoch)\n",
        "\n",
        "  train(bert_classifier, train_dataloader, loss_fn, epochs=4, evaluation=True)\n",
        "\n",
        "  return bert_classifier\n",
        "\n",
        "def testing_process(bert_classifier, X_val, y_val):\n",
        "  #  Run `preprocessing_for_bert` on the test set\n",
        "  print('Tokenizing data...')\n",
        "  test_inputs, test_masks = preprocessing_for_bert(X_val)\n",
        "\n",
        "  # Create the DataLoader for our test set\n",
        "  test_dataset = TensorDataset(test_inputs, test_masks)\n",
        "  test_sampler = SequentialSampler(test_dataset)\n",
        "  test_dataloader = DataLoader(test_dataset, sampler=test_sampler, batch_size=32)\n",
        "\n",
        "  # Compute predicted probabilities on the test set\n",
        "  probs = bert_predict(bert_classifier, test_dataloader)\n",
        "\n",
        "  # Get predictions from the probabilities\n",
        "  threshold = 0.5\n",
        "  preds = np.where(probs[:, 1] > threshold, 1, 0)\n",
        "\n",
        "  # Number of tweets predicted non-negative\n",
        "  print(\"Number of tweets predicted as Rumor: \", preds.sum())\n",
        "\n",
        "  preds = np.argmax(probs, axis = 1)\n",
        "  print('Accuracy Score:\\t',accuracy_score(y_val, preds))\n",
        "  print('Precision Score:\\t', str(precision_score(y_val,preds)))\n",
        "  print('Recall Score:\\t\\t' + str(recall_score(y_val,preds)))\n",
        "  print('F1 Score:\\t',f1_score(y_val, preds, zero_division=1))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fk3m9D6kpsOP"
      },
      "source": [
        "## Executions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VAAH77uL_FiS"
      },
      "outputs": [],
      "source": [
        "raw_text = pd.read_csv('./data/_PHEME_text.csv')\n",
        "y = pd.read_csv('./data/_PHEME_target.csv')\n",
        "data = pd.concat([raw_text.text, y], axis=1).reset_index(drop=True)\n",
        "val = pd.read_csv('data/_PHEMEext_text.csv')\n",
        "\n",
        "X_train = data.text.values\n",
        "y_train = data.target.values\n",
        "\n",
        "X_val = val.drop(['Event'],axis=1).text.values\n",
        "y_val = val.target.values\n",
        "\n",
        "rhi_data = pd.read_csv('data/_RHI_text.csv').text.values\n",
        "rhi_y = pd.read_csv('data/_RHI_text.csv').isRumor"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K84HzvLMbP0X",
        "outputId": "94fc8e1a-8b64-4147-e898-009599c44ac1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "There are 1 GPU(s) available.\n",
            "Device name: Tesla T4\n",
            "Max length:  69\n",
            "\n",
            "Tokenizing data...\n"
          ]
        }
      ],
      "source": [
        "device = getDevice()\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)\n",
        "train_dataloader, val_dataloader = data_process(X_train, y_train, X_val, y_val, batch_size=6)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fFfTQEHZ_Hc6"
      },
      "outputs": [],
      "source": [
        "set_seed(42)    # Set seed for reproducibility\n",
        "bert_classifier, optimizer, scheduler, loss_fn = initialize_model(epochs=3)\n",
        "# train(bert_classifier, train_dataloader, val_dataloader, epochs=1, evaluation=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lyxq0fPUvbDA",
        "outputId": "dac36902-7194-473b-867d-c44d37555a7b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Start training...\n",
            "\n",
            " Epoch  |  Batch  |  Train Loss  |  Val Loss  |  Val Acc  |  Elapsed \n",
            "----------------------------------------------------------------------\n",
            "   1    |   20    |   0.629314   |     -      |     -     |   17.30  \n",
            "   1    |   40    |   0.553115   |     -      |     -     |   17.16  \n",
            "   1    |   60    |   0.588918   |     -      |     -     |   16.53  \n",
            "   1    |   80    |   0.500162   |     -      |     -     |   16.13  \n",
            "   1    |   100   |   0.512859   |     -      |     -     |   16.06  \n",
            "   1    |   120   |   0.545202   |     -      |     -     |   16.25  \n",
            "   1    |   140   |   0.431342   |     -      |     -     |   16.48  \n",
            "   1    |   160   |   0.406685   |     -      |     -     |   16.51  \n",
            "   1    |   180   |   0.529125   |     -      |     -     |   16.36  \n",
            "   1    |   200   |   0.490212   |     -      |     -     |   16.24  \n",
            "   1    |   220   |   0.484178   |     -      |     -     |   16.25  \n",
            "   1    |   240   |   0.468012   |     -      |     -     |   16.33  \n",
            "   1    |   260   |   0.473001   |     -      |     -     |   16.38  \n",
            "   1    |   280   |   0.413844   |     -      |     -     |   16.40  \n",
            "   1    |   300   |   0.547014   |     -      |     -     |   16.36  \n",
            "   1    |   320   |   0.397528   |     -      |     -     |   16.29  \n",
            "   1    |   340   |   0.429567   |     -      |     -     |   16.27  \n",
            "   1    |   360   |   0.355043   |     -      |     -     |   16.27  \n",
            "   1    |   380   |   0.368029   |     -      |     -     |   16.33  \n",
            "   1    |   400   |   0.375655   |     -      |     -     |   16.37  \n",
            "   1    |   420   |   0.414914   |     -      |     -     |   16.37  \n",
            "   1    |   440   |   0.343276   |     -      |     -     |   16.37  \n",
            "   1    |   460   |   0.338237   |     -      |     -     |   16.38  \n",
            "   1    |   480   |   0.387037   |     -      |     -     |   16.39  \n",
            "   1    |   500   |   0.353207   |     -      |     -     |   16.38  \n",
            "   1    |   520   |   0.319626   |     -      |     -     |   16.39  \n",
            "   1    |   540   |   0.534802   |     -      |     -     |   16.38  \n",
            "   1    |   560   |   0.333614   |     -      |     -     |   16.37  \n",
            "   1    |   580   |   0.337307   |     -      |     -     |   16.39  \n",
            "   1    |   600   |   0.386578   |     -      |     -     |   16.39  \n",
            "   1    |   620   |   0.318854   |     -      |     -     |   16.37  \n",
            "   1    |   640   |   0.430609   |     -      |     -     |   16.37  \n",
            "   1    |   660   |   0.301156   |     -      |     -     |   16.33  \n",
            "   1    |   680   |   0.324869   |     -      |     -     |   16.35  \n",
            "   1    |   700   |   0.447027   |     -      |     -     |   16.35  \n",
            "   1    |   720   |   0.303918   |     -      |     -     |   16.36  \n",
            "   1    |   725   |   0.455132   |     -      |     -     |   3.51   \n",
            "----------------------------------------------------------------------\n",
            "   1    |    -    |   0.427523   |  2.352970  |   28.07   |  612.67  \n",
            "----------------------------------------------------------------------\n",
            "\n",
            "\n",
            " Epoch  |  Batch  |  Train Loss  |  Val Loss  |  Val Acc  |  Elapsed \n",
            "----------------------------------------------------------------------\n",
            "   2    |   20    |   0.273147   |     -      |     -     |   17.13  \n",
            "   2    |   40    |   0.242201   |     -      |     -     |   16.33  \n",
            "   2    |   60    |   0.309965   |     -      |     -     |   16.33  \n",
            "   2    |   80    |   0.323439   |     -      |     -     |   16.35  \n",
            "   2    |   100   |   0.255190   |     -      |     -     |   16.31  \n",
            "   2    |   120   |   0.190140   |     -      |     -     |   16.34  \n",
            "   2    |   140   |   0.255846   |     -      |     -     |   16.33  \n",
            "   2    |   160   |   0.430966   |     -      |     -     |   16.33  \n",
            "   2    |   180   |   0.208292   |     -      |     -     |   16.34  \n",
            "   2    |   200   |   0.294808   |     -      |     -     |   16.31  \n",
            "   2    |   220   |   0.347223   |     -      |     -     |   16.30  \n",
            "   2    |   240   |   0.314548   |     -      |     -     |   16.25  \n",
            "   2    |   260   |   0.238003   |     -      |     -     |   16.26  \n",
            "   2    |   280   |   0.317204   |     -      |     -     |   16.26  \n",
            "   2    |   300   |   0.248775   |     -      |     -     |   16.27  \n",
            "   2    |   320   |   0.286750   |     -      |     -     |   16.27  \n",
            "   2    |   340   |   0.204693   |     -      |     -     |   16.31  \n",
            "   2    |   360   |   0.284120   |     -      |     -     |   16.31  \n",
            "   2    |   380   |   0.179818   |     -      |     -     |   16.32  \n",
            "   2    |   400   |   0.252450   |     -      |     -     |   16.33  \n",
            "   2    |   420   |   0.290128   |     -      |     -     |   16.27  \n",
            "   2    |   440   |   0.237999   |     -      |     -     |   16.28  \n",
            "   2    |   460   |   0.126027   |     -      |     -     |   16.24  \n",
            "   2    |   480   |   0.273371   |     -      |     -     |   16.23  \n",
            "   2    |   500   |   0.269467   |     -      |     -     |   16.25  \n",
            "   2    |   520   |   0.256358   |     -      |     -     |   16.27  \n",
            "   2    |   540   |   0.257195   |     -      |     -     |   16.24  \n",
            "   2    |   560   |   0.293283   |     -      |     -     |   16.25  \n",
            "   2    |   580   |   0.177041   |     -      |     -     |   16.26  \n",
            "   2    |   600   |   0.396848   |     -      |     -     |   16.32  \n",
            "   2    |   620   |   0.193316   |     -      |     -     |   16.30  \n",
            "   2    |   640   |   0.308464   |     -      |     -     |   16.31  \n",
            "   2    |   660   |   0.153723   |     -      |     -     |   16.30  \n",
            "   2    |   680   |   0.241623   |     -      |     -     |   16.32  \n",
            "   2    |   700   |   0.256835   |     -      |     -     |   16.32  \n",
            "   2    |   720   |   0.231722   |     -      |     -     |   16.33  \n",
            "   2    |   725   |   0.114021   |     -      |     -     |   3.52   \n",
            "----------------------------------------------------------------------\n",
            "   2    |    -    |   0.260693   |  2.892816  |   30.74   |  609.92  \n",
            "----------------------------------------------------------------------\n",
            "\n",
            "\n",
            " Epoch  |  Batch  |  Train Loss  |  Val Loss  |  Val Acc  |  Elapsed \n",
            "----------------------------------------------------------------------\n",
            "   3    |   20    |   0.184743   |     -      |     -     |   17.10  \n",
            "   3    |   40    |   0.071133   |     -      |     -     |   16.30  \n",
            "   3    |   60    |   0.080171   |     -      |     -     |   16.26  \n",
            "   3    |   80    |   0.165196   |     -      |     -     |   16.28  \n",
            "   3    |   100   |   0.183444   |     -      |     -     |   16.26  \n",
            "   3    |   120   |   0.108059   |     -      |     -     |   16.24  \n",
            "   3    |   140   |   0.054661   |     -      |     -     |   16.24  \n",
            "   3    |   160   |   0.196549   |     -      |     -     |   16.24  \n",
            "   3    |   180   |   0.140945   |     -      |     -     |   16.27  \n",
            "   3    |   200   |   0.108247   |     -      |     -     |   16.24  \n",
            "   3    |   220   |   0.237730   |     -      |     -     |   16.25  \n",
            "   3    |   240   |   0.111115   |     -      |     -     |   16.30  \n",
            "   3    |   260   |   0.119129   |     -      |     -     |   16.26  \n",
            "   3    |   280   |   0.180386   |     -      |     -     |   16.31  \n",
            "   3    |   300   |   0.122873   |     -      |     -     |   16.32  \n",
            "   3    |   320   |   0.151681   |     -      |     -     |   16.30  \n",
            "   3    |   340   |   0.132506   |     -      |     -     |   16.28  \n",
            "   3    |   360   |   0.223705   |     -      |     -     |   16.27  \n",
            "   3    |   380   |   0.089148   |     -      |     -     |   16.23  \n",
            "   3    |   400   |   0.217655   |     -      |     -     |   16.26  \n",
            "   3    |   420   |   0.186604   |     -      |     -     |   16.25  \n",
            "   3    |   440   |   0.074641   |     -      |     -     |   16.25  \n",
            "   3    |   460   |   0.008564   |     -      |     -     |   16.24  \n",
            "   3    |   480   |   0.257192   |     -      |     -     |   16.23  \n",
            "   3    |   500   |   0.131824   |     -      |     -     |   16.25  \n",
            "   3    |   520   |   0.106689   |     -      |     -     |   16.26  \n",
            "   3    |   540   |   0.098494   |     -      |     -     |   16.23  \n",
            "   3    |   560   |   0.162358   |     -      |     -     |   16.24  \n",
            "   3    |   580   |   0.111314   |     -      |     -     |   16.25  \n",
            "   3    |   600   |   0.040556   |     -      |     -     |   16.26  \n",
            "   3    |   620   |   0.244624   |     -      |     -     |   16.28  \n",
            "   3    |   640   |   0.115019   |     -      |     -     |   16.27  \n",
            "   3    |   660   |   0.105975   |     -      |     -     |   16.29  \n",
            "   3    |   680   |   0.135292   |     -      |     -     |   16.27  \n",
            "   3    |   700   |   0.058514   |     -      |     -     |   16.25  \n",
            "   3    |   720   |   0.109798   |     -      |     -     |   16.23  \n",
            "   3    |   725   |   0.009366   |     -      |     -     |   3.50   \n",
            "----------------------------------------------------------------------\n",
            "   3    |    -    |   0.133281   |  2.890345  |   45.70   |  608.56  \n",
            "----------------------------------------------------------------------\n",
            "\n",
            "\n",
            "Training complete!\n"
          ]
        }
      ],
      "source": [
        "train(bert_classifier, train_dataloader, val_dataloader, epochs=3, evaluation=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BwDvSUU-GrFW"
      },
      "outputs": [],
      "source": [
        "torch.save(bert_classifier.state_dict(), './Model/BERT_raw_to_fine_tune_ord4.pt')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 330
        },
        "id": "16ynZzt_A95T",
        "outputId": "4ca200ac-fb0b-4323-d0aa-31b79eaa888b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "AUC: 0.7044\n",
            "Accuracy: 45.36%\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3debxV8/rA8c9TmqRCXEODotAgDUeZGkgkEbekSCJllvHecO9F1zVcLpd7M1S6+RkKuRJKLk1Co9KoQVEnRVJIg4bn98d3nc7qOHufdc7Za689PO/Xa7/OHtZe+9nrnLOf/f1+1/f5iqpijDHGxFIm6gCMMcakNksUxhhj4rJEYYwxJi5LFMYYY+KyRGGMMSYuSxTGGGPiskRhikVEFolIu6jjSBUicreIDIvotUeIyANRvHaiichlIvJ+CZ9rf5Mhs0SRxkTkKxHZJiJbRGS998FxQJivqaqNVHVymK+RR0QqiMhDIrLae5/LReROEZFkvH4h8bQTkVz/far6oKpeHdLriYjcLCILReQXEckVkddF5IQwXq+kROQ+EXmpNPtQ1ZdV9ewAr/Wb5JjMv8lsZYki/Z2vqgcATYFmwF0Rx1NsIrJfjIdeB9oDnYAqwOVAf+DJEGIQEUm1/4cngQHAzcDBwLHAGOC8RL9QnN9B6KJ8bROQqtolTS/AV8BZvtt/B9713T4Z+ATYDHwOtPM9djDwH+AbYBMwxvdYZ2Ce97xPgCYFXxM4EtgGHOx7rBnwPVDOu30VsMTb/wTgKN+2CtwALAdWFfLe2gPbgVoF7m8F7AbqebcnAw8BM4GfgLcKxBTvGEwG/gZ87L2XesCVXsw/AyuBa7xtK3vb7AG2eJcjgfuAl7xt6njv6wpgtXcs7vG9XiXgBe94LAH+AOTG+N3W995nyzi//xHAYOBdL94ZwDG+x58E1njHZQ7Q2vfYfcBo4CXv8auBlsCn3rFaB/wbKO97TiPgf8APwLfA3UBH4Fdgp3dMPve2rQY87+1nLfAAUNZ7rI93zJ8ANnqP9QGmeY+L99h3XmwLgMa4Lwk7vdfbArxd8P8AKOvF9aV3TOZQ4G/ILiX4rIk6ALuU4pe37z9ITe8f6knvdg3vn7ATruXYwbt9qPf4u8CrwEFAOaCtd38z7x+0lfdPd4X3OhUKec2JQD9fPI8Cz3rXuwArgAbAfsCfgE9826r3oXMwUKmQ9/YwMCXG+/6a/A/wyd4HUWPch/kb5H9wF3UMJuM+0Bt5MZbDfVs/xvuwagtsBZp727ejwAc7hSeKobikcCKwA2jgf0/eMa8JzC+4P99+rwW+LuL3P8J7Py29+F8GRvke7wVU9x67HVgPVPTFvRO40Ds2lYAWuMS6n/delgC3eNtXwX3o3w5U9G63KngMfK/9JvCc9zv5HS6R5/3O+gC7gJu816rEvoniHNwH/IHe76EBcITvPT8Q5//gTtz/wXHec08Eqkf9v5rul8gDsEspfnnuH2QL7puTAh8CB3qP/RF4scD2E3Af/EfgvhkfVMg+nwH+WuC+peQnEv8/5dXARO+64L69tvFujwf6+vZRBvehe5R3W4Ez47y3Yf4PvQKPTcf7po77sH/Y91hD3DfOsvGOge+5g4o4xmOAAd71dgRLFDV9j88EenjXVwLn+B67uuD+fI/dA0wvIrYRwDDf7U7AF3G23wSc6It7ahH7vwV407veE5gbY7u9x8C7fRguQVby3dcTmORd7wOsLrCPPuQnijOBZbikVaaQ9xwvUSwFuoTx/5bNl1TrkzXFd6GqVsF9iB0PHOLdfxRwsYhszrsAp+OSRC3gB1XdVMj+jgJuL/C8WrhuloLeAE4RkSOANrjk85FvP0/69vEDLpnU8D1/TZz39b0Xa2GO8B4vbD9f41oGhxD/GBQag4icKyLTReQHb/tO5B/ToNb7rm8F8k4wOLLA68V7/xuJ/f6DvBYicoeILBGRH733Uo1930vB936siLzjnRjxE/Cgb/tauO6cII7C/Q7W+Y77c7iWRaGv7aeqE3HdXoOB70RkiIhUDfjaxYnTBGSJIkOo6hTct63HvLvW4L5NH+i7VFbVh73HDhaRAwvZ1RrgbwWet7+qjizkNTcB7wOXAJfiWgDq2881BfZTSVU/8e8izlv6AGglIrX8d4pIK9yHwUTf3f5tauO6VL4v4hj8JgYRqYBLfo8Bh6nqgcA4XIIrKt4g1uG6nAqLu6APgZoiklOSFxKR1rgxkO64luOBwI/kvxf47ft5BvgCqK+qVXF9/XnbrwGOjvFyBfezBteiOMR33KuqaqM4z9l3h6pPqWoLXAvxWFyXUpHP8177mCK2McVkiSKz/BPoICIn4gYpzxeRc0SkrIhU9E7vrKmq63BdQ0+LyEEiUk5E2nj7GApcKyKtvDOBKovIeSJSJcZrvgL0Brp51/M8C9wlIo0ARKSaiFwc9I2o6ge4D8s3RKSR9x5O9t7XM6q63Ld5LxFpKCL7A4OA0aq6O94xiPGy5YEKwAZgl4icC/hP2fwWqC4i1YK+jwJewx2Tg0SkBnBjrA299/c0MNKLubwXfw8RGRjgtargxgE2APuJyF+Aor6VV8ENHm8RkeOB63yPvQMcISK3eKctV/GSNrjjUifvrDHv7+t94B8iUlVEyojIMSLSNkDciMhJ3t9fOeAX3EkNe3yvFSthgeuy/KuI1Pf+fpuISPUgr2tis0SRQVR1A/B/wF9UdQ1uQPlu3IfFGty3srzf+eW4b95f4Aavb/H2MRvoh2v6b8INSPeJ87JjcWforFfVz32xvAk8AozyujEWAucW8y11BSYB7+HGYl7CnUlzU4HtXsS1ptbjBlpv9mIo6hjsQ1V/9p77Gu69X+q9v7zHvwBGAiu9LpXCuuPiGQTkAqtwLabRuG/esdxMfhfMZlyXykXA2wFeawLuuC3DdcdtJ35XF8AduPf8M+4Lw6t5D3jHpgNwPu44LwfO8B5+3fu5UUQ+8673xiXexbhjOZpgXWngEtpQ73lf47rhHvUeex5o6B3/MYU893Hc7+99XNJ7HjdYbkpB8nsKjEk/IjIZN5Aayezo0hCR63AD3YG+aRsTFWtRGJMkInKEiJzmdcUchzvV9M2o4zKmKKElChEZLiLficjCGI+LiDwlIitEZL6INA8rFmNSRHnc2T8/4wbj38KNQxiT0kLrevIGR7cA/6eqjQt5vBOur7kTbnLXk6raquB2xhhjohVai0JVp+LOnY+lCy6JqKpOBw70zsc3xhiTQqIsxlWDfc/CyPXuW1dwQxHpj6vzQuXKlVscf/zxSQnQGGOisGED/BDva7bPli3u5wEx6kYftuNrDti1mc911/eqemhJ4kmLqo2qOgQYApCTk6OzZ8+OOCJjjEmsIUPgFW8m0pw57mfbgOfDXXop9O/vuyNvSEEEnnkGvvsOue++r0saW5SJYi37zkyt6d1njDEZy58Q/KZMcT/btnWX33z4B7V2LVx3HVxyCVx2mbsOcN99JQ050kQxFrhRREbhBrN/9GZ0GmNMyoj1wV5S/oTgV6rkAK4VMWwY3HEH7NwJ5yVu2ZLQEoWIjMQVqjtE3Kpg9+IKhaGqz+Jq6HTCzfzdilsHwBhjUsorr8C8edC0aWL2V+qEUJgvv4R+/WDSJDjjDBg6FI5JXMmr0BKFqvYs4nHFLVxjjDEpxd+KyEsSkydHGlJ8Cxa4gY0hQ+Dqq93YRAKlxWC2McYkSpCuJH/3UNOmrgWQchYuhM8+g9694cILYeVKqB5O/UNLFMaYjOdPDrHGCPxC6R5KlF9/hQcfdJfDDoPu3aFixdCSBFiiMMZkqFjJIaWTQFFmzIC+fWHRIujVC554wiWJkFmiMMakvcK6kzImOeRZuxZat3atiHfeSehZTUWxRGGMSXuFnZmUEckBYNkyOPZYqFEDXn0V2reHqkFXhk0MSxTGmJQWZPA5Lc5MKq7Nm+EPf3BzIyZPhjZt4KKLIgnFEoUxJiUEmbEcS8qemVRSY8e6GdXr18Odd8JJJ0UajiUKY0xKiDWxLWO6kIK6+mp4/nk44QR46y3IyYk6IksUxpjUkXHdR0H5i/jl5MBRR8Ef/wjly0cbl8cShTEmMoXNgM46a9bAtddCjx5w+eXueoqxNbONMUk3ZAi0awfXXJM/BpFx4wxF2bPHlQBv1Mg1o3bsiDqimKxFYYxJurzxiKwbf8izfLkbi5g6Fc46y2XOunWjjiomSxTGmKRIu0J7YVq8GObPh+HDoU+fhBfxSzTrejLGJEVeKwKysJsJ4PPP4YUX3PUuXVwRvyuvTPkkAdaiMMaEyFoRuLGHBx6Ahx+GI45wK89VrAgHHRR1ZIFZi8IYE5qsb0V8+ik0a+YSxaWXwty5SSnil2jWojDGhCorWxHgivi1bQuHHw7jxsG550YdUYlZi8IYYxJpyRL3s0YNeO01VxI8jZMEWIvCGJMAseo0ZdUkuk2b4Pbb4T//cae9tm7tVp7LAJYojDFxFXfpUL+sGZd48024/nrYsAHuuivyIn6JZonCGLNXUQsAxZK1E+cArrrKtSKaNoV334XmzaOOKOEsURhj9sroBYASyV/E7+SToX59uOMOKFcu2rhCYonCmCxU1JhCVp6lFNTXX7siVZdeCr17Z0UGtbOejMlC/vkNflkzplASe/bA4MHQuDFMmwY7d0YdUdJYi8KYLGUth2JYutQV8Zs2Dc4+G557DurUiTqqpLFEYUyGKc4a0yagpUvdfIgRI1x3UxrUZ0ok63oyJsPE6lbysy6mAObOdWczAVxwgSvid8UVWZckwFoUxmQk61Yqhe3bYdAg+Pvf3ezqnj1dfaYDD4w6sshYi8IYY/J8/LHLsg895LqY5s1LyyJ+iWYtCmMygK09nQBr18IZZ7hWxIQJbtDaAJYojEkrsQaq/bOnbfyhmBYvhoYNXYJ44w2XLA44IOqoUoolCmPSQF6CiFVOw2ZPl8APP8Btt7lV56ZMgTZt4Pzzo44qJVmiMCZF+VsP/gRhCSEB3ngDbrgBNm6Ee+6Bli2jjiilWaIwJmJBupMsQSRQnz6uFdG8Obz3ng3oBGCJwpiIFVaIDyw5JJS/iN+pp0KDBm7tiP3sIzCIUI+SiHQEngTKAsNU9eECj9cGXgAO9LYZqKrjwozJmFRk8x5CtGqVy7a9erkJc5Z5iy20eRQiUhYYDJwLNAR6ikjDApv9CXhNVZsBPYCnw4rHmFQyZAi0a+cuRc2iNiW0ezc89ZQr4jd9en6rwhRbmBPuWgIrVHWlqv4KjAK6FNhGgare9WrANyHGY0zK8JfZsNNZQ7BkiVuKdMAA14e3aJEbmzAlEmbXUw1gje92LtCqwDb3Ae+LyE1AZeCswnYkIv2B/gC1a9dOeKDGJENhk+KsuykkK1a4Qn4vvgiXXZaV9ZkSKeoSHj2BEapaE+gEvCgiv4lJVYeoao6q5hx66KFJD9KYRLBWRMjmzIHhw9318893YxO9elmSSIAwWxRrgVq+2zW9+/z6Ah0BVPVTEakIHAJ8F2JcxkTGWhEh2LYN7r8fHnsMatVyGbhiRahatejnmkDCTBSzgPoiUheXIHoABb9DrQbaAyNEpAFQEdgQYkzGlEiQNR6KYjWYQjB1qltQaPly6NvXJQsr4pdwoSUKVd0lIjcCE3Cnvg5X1UUiMgiYrapjgduBoSJyK25gu4+qnZpgUkOsmdElZd1NCbZ2LbRv71oRH3zgrptQSLp9Lufk5Ojs2bOjDsNkgbxTV/NaATb5LUUsWAAnnOCuv/OOK+JXuXK0MaUBEZmjqjklea5NSzQmDhtTSCHffw+33govvZRfxK9z56ijygpRn/VkTEqxiXApSBVee82VAh81Cu69F1oVPNPehMlaFCbrxRqLsDGFFHHFFW4+RE4OfPhhfreTSRpLFCbr+YvyWSG+FOEv4te2LTRpArfcYkX8ImJH3RhsLCKlrFwJ/fq5yXJXXulOezWRsjEKk5VsLCIF7d4N//yn61qaNQvK2MdTqrDfhMlKVk4jxSxeDKed5s5qOuMMd/uKK6KOynis68lkLetuSiGrVsGXX7oM3qOH1WdKMZYojDHRmDXLNev69YPzznNjE1WqRB2VKYR1PZmsYeMSKWLrVrjjDjj5ZHjoIdi+3d1vSSJlWaIwWcPGJVLA5MnuVNd//MO1JObOtSJ+acC6nkzGKKrCqy0WFLHcXOjQAY46CiZOdIPWJi1Yi8JkDH+LoTDWiojI55+7nzVrwltvwfz5liTSjLUoTEaxFkMK2bDBrVk9cqT7pbRtC506RR2VKQFrUZi0ZgPUKUjVJYeGDWH0aLf63CmnRB2VKQVrUZi0Y0X8Utzll8PLL7sKr88/D40aRR2RKaXAiUJE9lfVrWEGY0wQVsQvBe3Z4ybJibjxhxYt4OaboWzZqCMzCVBkohCRU4FhwAFAbRE5EbhGVa8POzhjYrGxiBSyYoU71fXyy+Gqq6yIXwYKMkbxBHAOsBFAVT8H2oQZlDEF2VhECtq1Cx57zBXxmzsXypePOiITkkBdT6q6RvatvbI7nHCMyWdjESls4UJXAnz2bOjSBZ5+Go48MuqoTEiCJIo1XveTikg5YACwJNywjLGxiJS2ejV8/bVbmrR7dyvil+GCJIprgSeBGsBa4H3AxidMUthYRAqZMcNNnuvf382HWLkSDjgg6qhMEgRJFMep6mX+O0TkNODjcEIy2SZW6Y281oSJ2C+/wJ//7BYVOvpot05EhQqWJLJIkMHsfwW8z5gSiVV6w8YiUsDEia6I3xNPwLXXwmefuSRhskrMFoWInAKcChwqIrf5HqoK2MnRJqGsiykF5ebCOedA3brubII2drJjtorX9VQeN3diP8BfKP4noFuYQRljIjR3LjRr5or4vf22O5OgUqWoozIRipkoVHUKMEVERqjq10mMyRgThW+/dbOpX3stv4hfx45RR2VSQJDB7K0i8ijQCNi7woiqnhlaVMaY5FF1tZkGDIAtW+CBB+DUU6OOyqSQIIPZLwNfAHWB+4GvgFkhxmSMSaZLL3XlN447zp1VcM89UK5c1FGZFBKkRVFdVZ8XkQG+7ihLFKZECjsV1k6DjYC/iN/ZZ7sy4DfcYEX8TKGCtCh2ej/Xich5ItIMODjEmEwGK+xUWDsNNsmWLXMVXocPd7evvNIqvZq4grQoHhCRasDtuPkTVYFbQo3KZBR/K8LWrY7Qrl3w+ONw771QsaKdyWQCKzJRqOo73tUfgTNg78xsYwLx12yy1kNE5s93JcDnzIGLLoLBg+GII6KOyqSJeBPuygLdcTWe3lPVhSLSGbgbqAQ0S06IJhNYKyJiubmwZg28/jp07WpF/EyxxBujeB64GqgOPCUiLwGPAX9X1UBJQkQ6ishSEVkhIgNjbNNdRBaLyCIRKaTijzGmRD75BJ591l3PK+LXrZslCVNs8bqecoAmqrpHRCoC64FjVHVjkB17LZLBQAcgF5glImNVdbFvm/rAXcBpqrpJRH5X0jdijPFs2eJOcf3Xv+CYY9xgdYUKULly1JGZNBWvRfGrqu4BUNXtwMqgScLTElihqitV9VdgFNClwDb9gMGqusl7ne+KsX9jTEHvvw+NG7skccMNVsTPJES8FsXxIjLfuy7AMd5tAVRVmxSx7xrAGt/tXKBVgW2OBRCRj3GFBu9T1fcK7khE+gP9AWrXrl3EyxqTpdasgfPOc62IqVPh9NOjjshkiHiJokGSXr8+0A6oCUwVkRNUdbN/I1UdAgwByMnJ0STEZUz6mDMHWrSAWrVg3Dho3dqd/mpMgsTselLVr+NdAux7LVDLd7umd59fLjBWVXeq6ipgGS5xGGOKsn49XHwx5OTkLyreoYMlCZNwQWZml9QsoL6I1BWR8kAPYGyBbcbgWhOIyCG4rqiVIcZkTPpThRdegIYNXRnwBx+0In4mVEFmZpeIqu4SkRuBCbjxh+GqukhEBgGzVXWs99jZIrIY2A3cWcwBc2OyT48erhT4aafBsGFw/PFRR2QyXKBEISKVgNqqurQ4O1fVccC4Avf9xXddgdu8izEmFn8Rv06d3DjE9ddDmTA7BYxxivwrE5HzgXnAe97tpiJSsAvJGBOWL75wy5A+/7y7fcUVcOONliRM0gT5S7sPNydiM4CqzsOtTWGMCdPOnW784cQTYfFiOOCAqCMyWSpI19NOVf1R9p32b6eoGhOmefPcjOp581zZjX/9Cw4/POqoTJYKkigWicilQFmv5MbNwCfhhmVMllu/3l3eeAN+//uoozFZLkjX00249bJ3AK/gyo3behTGJNq0afD00+56x47w5ZeWJExKCNKiOF5V7wHuCTsYk94KW+YUbKnTIv38M9x1l1sjon596NvX1Wfaf/+oIzMGCNai+IeILBGRv4pI49AjMmmrsGVOwRYrimvCBFfE7+mnYcAAK+JnUlKQFe7OEJHDcYsYPSciVYFXVfWB0KMzKc+WOS2FNWugc2eoV891O9nsapOiAp2IrarrVfUp4FrcnIq/FPEUkyX8rQhrOQSgCjNnuuu1asH48TB3riUJk9KKbFGISAPgEqArsBF4Fbg95LhMGrFWREDr1rk1It580x2wtm3hrLOijsqYIgUZzB6OSw7nqOo3IcdjTOZRhREj4LbbYPt2eOQRV6fJmDQRZIzilGQEYkzG6t4dRo929ZmGDYNjj406ImOKJWaiEJHXVLW7iCxg35nYQVe4MxmqsAFsU8Du3a6AX5kycP75cOaZcM01Vp/JpKV4LYoB3s/OyQjEpI+8AeymTW0Au1BLlri5EFdeCf36Qe/eUUdkTKnETBSqus67er2q/tH/mIg8Avzxt88ymcpOgw1g5043/vDXv7oCftWqRR2RMQkRpB3coZD7zk10ICb1DBkC7dq5yzXX5K+2aa2IQsyd65Yk/fOf4aKLXKuie/eoozImIeKNUVwHXA8cLSLzfQ9VAT4OOzATPX8XU9u2Ljn07x91VCnq22/h++9hzBjo0iXqaIxJqHhjFK8A44GHgIG++39W1R9CjcpEKq+bybqYijB1KixY4OZGdOwIK1ZApUpRR2VMwsVLFKqqX4nIDQUfEJGDLVlkFv8YRF4XU14rwhTw008wcCA884w71fXqq119JksSJkMV1aLoDMzBnR7rX7lIgaNDjMskmXUzBTRunBuw+eYbN4Fu0CAr4mcyXryznjp7P23Z0wxlZzIV05o1bvzhuOPcBLpWraKOyJikKPKsJxE5TUQqe9d7icjjIlI7/NBM2KygXwCqMH26u16rFrz/visFbknCZJEgp8c+A2wVkRNxxQC/BF4MNSqTNHmtiMmTravpN775Bi68EE45JX/g5owzoHz5aOMyJsmCJIpdqqpAF+DfqjoYd4qsMZlJ1dVkatjQtSAee8yK+JmsFqR67M8ichdwOdBaRMoA5cINy5RWrGVJ/axOUwzdusF//+tG9YcNcwsLGZPFgrQoLgF2AFep6nqgJvBoqFGZUou1LKmfjUv47N4Ne/a46xdeCM8+CxMnWpIwhmBlxteLyMvASSLSGZipqv8XfmimtOwspoAWLnRzIfr2dUX8Lr886oiMSSlBznrqDswELsatmz1DRLqFHZgpPn9tpqJaEwb49Ve4/35o3hy+/BIOOijqiIxJSUHGKO4BTlLV7wBE5FDgA2B0mIGZ4rPy38UwZw706eNaE5deCv/8Jxx6aNRRGZOSgiSKMnlJwrORYGMbJgLW3RTQxo2weTO8/TZ0tiVXjIknSKJ4T0QmACO925cA48ILyRQl1hlNdhZTESZNckX8br4Zzj4bli+HihWjjsqYlFdky0BV7wSeA5p4lyEFFzIyyRXrjCbrborhxx9dfaYzz3SF/HbscPdbkjAmkHjrUdQHHgOOARYAd6jq2mQFZuKzLqaA3n4brr0W1q+HO+5wg9dWxM+YYonXohgOvAN0xVWQ/VdSIjImUdasga5doXp1V6/p0Udh//2jjsqYtBNvjKKKqg71ri8Vkc+SEZApela1jUXEoQqffgqnnppfxO/UU60+kzGlEK9FUVFEmolIcxFpDlQqcLtIItJRRJaKyAoRGRhnu64ioiKSU9w3kClirU9dGBuLiCE3Fy64wNVlyjuA7dpZkjCmlOK1KNYBj/tur/fdVuDMeDsWkbLAYKADkAvMEpGxqrq4wHZVgAHAjOKFnlls4aBS2LMHhg6FO++EXbvg8cfh9NOjjsqYjBFv4aIzSrnvlsAKVV0JICKjcBVoFxfY7q/AI8CdpXy9tGcD1CXUtSuMGePOaho6FI62xReNSaQwJ87VANb4bud69+3ldWHVUtV34+1IRPqLyGwRmb1hw4bERxoRK7lRCrt25Rfx69rVJYgPPrAkYUwIIpth7ZUrfxy3GFJcqjpEVXNUNefQDCqzYCvMldD8+W4xoaHeuRa9ermifiLxn2eMKZEgM7NLai1Qy3e7pndfnipAY2CyuH/ww4GxInKBqs4OMa6kK2omtXU3BbRjBzz4oLscdJDVZjImSYJUjxVvrey/eLdri0jLAPueBdQXkboiUh7oAYzNe1BVf1TVQ1S1jqrWAaYDGZckwGZSJ8SsWa7K66BB0LMnLFkCv/991FEZkxWCtCieBvbgznIaBPwMvAGcFO9JqrpLRG4EJgBlgeGqukhEBgGzVXVsvOenI2s5hGjTJtiyBcaNg3PPjToaY7JKkETRSlWbi8hcAFXd5LUQiqSq4yhQQFBV/xJj23ZB9pnK/Ke4+lnLoYQmTnRF/AYMcEX8li2z8hvGRCBIotjpzYlQ2LsexZ5Qo0ozeS0JazkkyObNbk7EsGHQoIGr1VShgiUJYyISJFE8BbwJ/E5E/gZ0A/4UalQpKlbXUt4k4LyJcqYU3noLrrsOvv0W/vAHuO8+SxDGRCzImtkvi8gcoD0gwIWquiT0yFJQrK4lm0mdIKtXw8UXu1bE2LGQk7UVXYxJKUUmChGpDWwF3vbfp6qrwwwsVVnXUoKpwrRp0Lo11K7tJs2dfLLVZzImhQTpenoXNz4hQEWgLrAUaBRiXCYbrF7txh/Gj3fZt21baNMm6qiMMQUE6Xo6wX/bK7txfWgRpRj/uISV906QPXvg2Wfhj390LYqnnsPkorEAABUBSURBVLIifsaksGLPzFbVz0SkVRjBpAp/cvAPVNtprgny+9+7QesOHdzBrlMn6oiMMXEEGaO4zXezDNAc+Ca0iFKAlfwOwa5dUKaMu1xyCXTpAn36WH0mY9JAkBZFFd/1XbgxizfCCSd12KB1An3+OVx1FfTr58YkevaMOiJjTDHETRTeRLsqqnpHkuIxmWT7dnjgAXjkETj4YDj88KgjMsaUQMxEISL7efWaTktmQCZDzJwJV1wBX3zhfj7+uEsWxpi0E69FMRM3HjFPRMYCrwO/5D2oqv8NOTaTzn76CbZtg/feg3POiToaY0wpBBmjqAhsxFWPzZtPoYAlCrOv99+HRYvg1lvhrLNg6VIrv2FMBoiXKH7nnfG0kPwEkUdDjcqkl02b4LbbYMQIaNQIrr/eivgZk0HiLVxUFjjAu1TxXc+7ZBRbv7qE/vtfaNgQXnwR7roLZs+2BGFMhonXolinqoOSFknE/HMnbGJdQKtXQ48e0LixW1CoWbOoIzLGhCBeosi6mVA2dyIAVZg61c1ErF3bLS7UqhWUKxd1ZMaYkMTremqftCgiYt1NxfT1124Z0nbt8mubnH66JQljMlzMRKGqPyQzkCjkdTeBdTfFtWcP/PvfbqB62jT4179cWXBjTFYodlHATGPdTQFceCG8/babD/Hcc3DUUVFHZIxJoqxPFCaGnTuhbFlXxK9nT+jWDS6/3Ir4GZOF4o1RZCQblwjgs8+gZUu3ZgS4RNG7tyUJY7JU1iUKG5eIY9s2NxeiZUtYvx5q1Yo6ImNMCsjKricblyjE9OmueN+yZa4k+GOPwUEHRR2VMSYFZGWiMIX45Rc3LvG//7k6TcYY47FEkc3ee88V8bv9dmjf3pUEL18+6qiMMSkm68YoDLBxo+tmOvdceOEF+PVXd78lCWNMISxRZBNVGD3aFfF75RX4059g1ixLEMaYuKzrKZusXu1O82rSxK0dceKJUUdkjEkDWdOiyJs/kXVzJ1Rd4T5wM6onT3ZnOFmSMMYElDWJwl9GPGvmTqxaBWef7Qaq84r4nXoq7GcNSWNMcFn1iZE18yd273ZF/O6+25XheOYZK+JnjCmxrEoUWaNLF3j3XejUyZXhsBnWxphSyOhEMWSI63KC/G6njOUv4nf55a4+06WXWn0mY0yphTpGISIdRWSpiKwQkYGFPH6biCwWkfki8qGIJLR+ddbUdZo9G3JyXBcTwCWXwGWXWZIwxiREaC0KESkLDAY6ALnALBEZq6qLfZvNBXJUdauIXAf8HbgkkXFk9LjEtm1w332uLtNhh9k6EcaYUITZomgJrFDVlar6KzAK6OLfQFUnqepW7+Z0oGaI8WSWTz91p7j+/e+uiN/ixdC5c9RRGWMyUJhjFDWANb7buUCrONv3BcYX9oCI9Af6A9SuXTtR8aW3bdvcEqUffOBOfzXGmJCkxGC2iPQCcoC2hT2uqkOAIQA5OTmaxNBSy7hxrojfnXfCmWfCkiVQrlzUURljMlyYXU9rAf95mTW9+/YhImcB9wAXqOqOEONJX99/D716wXnnwcsv5xfxsyRhjEmCMBPFLKC+iNQVkfJAD2CsfwMRaQY8h0sS34UYS3pShVGjoEEDeO01uPdemDnTivgZY5IqtK4nVd0lIjcCE4CywHBVXSQig4DZqjoWeBQ4AHhd3Kmcq1X1grBiSjurV7ty4CeeCM8/DyecEHVExpgsFOoYhaqOA8YVuO8vvuu2lFpBqvDhh26VuaOOcjWaTjrJTaYzxpgIZE1RwLTw5ZfuDKYOHfKL+J18siUJY0ykLFGkgt274fHHXdfSnDnw3HNWxM8YkzJS4vTYrHf++TB+vJsw98wzUNPmHRpjUocliqj8+qtbF6JMGejTxxXy69HD6jMZY1KOdT1FYeZMaNECnn7a3e7e3VV7tSRhjElBliiSaetWuP12OOUU2LQJjjkm6oiMMaZI1vWULNOmuTkRK1fCNdfAI49AtWpRR2WMMUWyRJEseQsLTZoE7dpFHY0xxgRmiSJMb7/tCvf94Q9wxhmuFPh+dsiNMekl48YohgxxX9jbtctf3S7pNmxwy+ldcAGMHJlfxM+ShDEmDWVcooh0+VNVF0CDBjB6NAwaBDNmWBE/Y0xay8ivuJEtf7p6NVx5JTRr5or4NWoUQRDGGJNYGdeiSLo9e2DCBHf9qKPgo4/g448tSRhjMoYlitJYvtytNNexI0yd6u5r2dKK+BljMoolipLYtQsefRSaNHEDIs8/b0X8jDEZKyPHKELXubPrburSxZXhOPLIqCMyJiXt3LmT3Nxctm/fHnUoWaNixYrUrFmTcglcKtkSRVA7drg1qsuUgauvhquugosvtvpMxsSRm5tLlSpVqFOnDmL/K6FTVTZu3Ehubi5169ZN2H7TNlEMGeLORC1o3jx31lNCTZ8OffvCtdfCTTdBt24JfgFjMtP27dstSSSRiFC9enU2bNiQ0P2m7RiFf76EX0LnTvzyC9x6K5x6Kvz8M9Svn6AdG5M9LEkkVxjHO21bFBDyfImPPnJF/Fatguuvh4cegqpVQ3oxY4xJXWnbogjdrl1uTGLKFBg82JKEMWlszJgxiAhffPHF3vsmT55M586d99muT58+jB49GnAD8QMHDqR+/fo0b96cU045hfHjx5c6loceeoh69epx3HHHMSFvDlYBrVu3pmnTpjRt2pQjjzySCy+8EHBjEDfffDP16tWjSZMmfPbZZ6WOJ4i0blEk3JgxrojfXXe5In6LFll9JmMywMiRIzn99NMZOXIk999/f6Dn/PnPf2bdunUsXLiQChUq8O233zJlypRSxbF48WJGjRrFokWL+OabbzjrrLNYtmwZZQvMvfroo4/2Xu/atStdunQBYPz48Sxfvpzly5czY8YMrrvuOmbMmFGqmIKwT0GAb791g9Svvw7Nm7vFhcqXtyRhTALdckviC3U2bQr//Gf8bbZs2cK0adOYNGkS559/fqBEsXXrVoYOHcqqVauoUKECAIcddhjdu3cvVbxvvfUWPXr0oEKFCtStW5d69eoxc+ZMTjnllEK3/+mnn5g4cSL/+c9/9j6/d+/eiAgnn3wymzdvZt26dRxxxBGliqso2d31pAovvggNG8Jbb8Hf/ubOcLIifsZkjLfeeouOHTty7LHHUr16debMmVPkc1asWEHt2rWpGqDL+dZbb93bTeS/PPzww7/Zdu3atdSqVWvv7Zo1a7J27dqY+x4zZgzt27ffG0dxn58o2f2VefVqNyciJ8fNrj7++KgjMiZjFfXNPywjR45kwIABAPTo0YORI0fSokWLmGcHFfesoSeeeKLUMcYycuRIrr766tD2H1T2JYq8In7nnuuK+H38sav2avWZjMk4P/zwAxMnTmTBggWICLt370ZEePTRR6levTqbNm36zfaHHHII9erVY/Xq1fz0009FtipuvfVWJk2a9Jv7e/TowcCBA/e5r0aNGqxZs2bv7dzcXGrUqFHofr///ntmzpzJm2++WaLnJ5SqptWlRYsWqqratq27FMvSpaqtW6uC6uTJxXyyMaa4Fi9eHOnrP/fcc9q/f/997mvTpo1OmTJFt2/frnXq1Nkb41dffaW1a9fWzZs3q6rqnXfeqX369NEdO3aoqup3332nr732WqniWbhwoTZp0kS3b9+uK1eu1Lp16+quXbsK3faZZ57R3r1773PfO++8ox07dtQ9e/bop59+qieddFKhzy3suAOztYSfu9kxRrFrFzzyiCvit2AB/Oc/0KZN1FEZY0I2cuRILrroon3u69q1KyNHjqRChQq89NJLXHnllTRt2pRu3boxbNgwqlWrBsADDzzAoYceSsOGDWncuDGdO3cONGYRT6NGjejevTsNGzakY8eODB48eO8ZT506deKbb77Zu+2oUaPo2bPnPs/v1KkTRx99NPXq1aNfv348/fTTpYonKHGJJn3k5OTo7NmzadfO3Q404e6cc+D99+H3v3dzIg4/PMQIjTF5lixZQoMGDaIOI+sUdtxFZI6q5pRkf5k7RrF9u5swV7Ys9O/vLl27Rh2VMcaknczsevr4Y3eC9eDB7nbXrpYkjDGmhNIuUSxdCu3axZi4s2UL3HyzW0Ro+3awJq8xkUu37u10F8bxTrtEsW2b+/mbKrFTpkDjxvDvf8ONN8LChdChQyQxGmOcihUrsnHjRksWSaLeehQVK1ZM6H7TboyiUqU4A9j77++qvp52WjJDMsbEULNmTXJzcxO+PoKJLW+Fu0RKu7OeqlTJ0Z9/nu1u/Pe/8MUXcPfd7vbu3TZxzhhjClGas55C7XoSkY4islREVojIwEIeryAir3qPzxCROoF2vH69W2Wua1d480349Vd3vyUJY4xJuNAShYiUBQYD5wINgZ4i0rDAZn2BTapaD3gCeKSo/VbbudENUr/zjltM6JNPrIifMcaEKMwWRUtghaquVNVfgVFAlwLbdAFe8K6PBtpLERW5DtvxtRu0/vxzGDjQzZUwxhgTmjAHs2sAa3y3c4FWsbZR1V0i8iNQHfjev5GI9Af6ezd3yLRpC63SKwCHUOBYZTE7FvnsWOSzY5HvuJI+MS3OelLVIcAQABGZXdIBmUxjxyKfHYt8dizy2bHIJyKzS/rcMLue1gK1fLdrevcVuo2I7AdUAzaGGJMxxphiCjNRzALqi0hdESkP9ADGFthmLHCFd70bMFHT7XxdY4zJcKF1PXljDjcCE4CywHBVXSQig3B10ccCzwMvisgK4AdcMinKkLBiTkN2LPLZschnxyKfHYt8JT4WaTfhzhhjTHKlXa0nY4wxyWWJwhhjTFwpmyhCK/+RhgIci9tEZLGIzBeRD0XkqCjiTIaijoVvu64ioiKSsadGBjkWItLd+9tYJCKvJDvGZAnwP1JbRCaJyFzv/6RTFHGGTUSGi8h3IrIwxuMiIk95x2m+iDQPtOOSLrYd5gU3+P0lcDRQHvgcaFhgm+uBZ73rPYBXo447wmNxBrC/d/26bD4W3nZVgKnAdCAn6rgj/LuoD8wFDvJu/y7quCM8FkOA67zrDYGvoo47pGPRBmgOLIzxeCdgPCDAycCMIPtN1RZFKOU/0lSRx0JVJ6nqVu/mdNyclUwU5O8C4K+4umHbkxlckgU5Fv2Awaq6CUBVv0tyjMkS5FgoUNW7Xg34JonxJY2qTsWdQRpLF+D/1JkOHCgiRxS131RNFIWV/6gRaxtV3QXklf/INEGOhV9f3DeGTFTksfCa0rVU9d1kBhaBIH8XxwLHisjHIjJdRDomLbrkCnIs7gN6iUguMA64KTmhpZzifp4AaVLCwwQjIr2AHKBt1LFEQUTKAI8DfSIOJVXsh+t+aodrZU4VkRNUdXOkUUWjJzBCVf8hIqfg5m81VtU9UQeWDlK1RWHlP/IFORaIyFnAPcAFqrojSbElW1HHogrQGJgsIl/h+mDHZuiAdpC/i1xgrKruVNVVwDJc4sg0QY5FX+A1AFX9FKiIKxiYbQJ9nhSUqonCyn/kK/JYiEgz4DlcksjUfmgo4lio6o+qeoiq1lHVOrjxmgtUtcTF0FJYkP+RMbjWBCJyCK4ramUyg0ySIMdiNdAeQEQa4BJFNq7POhbo7Z39dDLwo6quK+pJKdn1pOGV/0g7AY/Fo8ABwOveeP5qVb0gsqBDEvBYZIWAx2ICcLaILAZ2A3eqasa1ugMei9uBoSJyK25gu08mfrEUkZG4LweHeOMx9wLlAFT1Wdz4TCdgBbAVuDLQfjPwWBljjEmgVO16MsYYkyIsURhjjInLEoUxxpi4LFEYY4yJyxKFMcaYuCxRmJQkIrtFZJ7vUifOtlsS8HojRGSV91qfebN3i7uPYSLS0Lt+d4HHPiltjN5+8o7LQhF5W0QOLGL7pplaKdUkj50ea1KSiGxR1QMSvW2cfYwA3lHV0SJyNvCYqjYpxf5KHVNR+xWRF4Blqvq3ONv3wVXQvTHRsZjsYS0KkxZE5ABvrY3PRGSBiPymaqyIHCEiU33fuFt7958tIp96z31dRIr6AJ8K1POee5u3r4Uicot3X2UReVdEPvfuv8S7f7KI5IjIw0AlL46Xvce2eD9Hich5vphHiEg3ESkrIo+KyCxvnYBrAhyWT/EKuolIS+89zhWRT0TkOG+W8iDgEi+WS7zYh4vITG/bwqrvGrOvqOun28UuhV1wM4nneZc3cVUEqnqPHYKbWZrXIt7i/bwduMe7XhZX++kQ3Ad/Ze/+PwJ/KeT1RgDdvOsXAzOAFsACoDJu5vsioBnQFRjqe2417+dkvPUv8mLybZMX40XAC9718rhKnpWA/sCfvPsrALOBuoXEucX3/l4HOnq3qwL7edfPAt7wrvcB/u17/oNAL+/6gbj6T5Wj/n3bJbUvKVnCwxhgm6o2zbshIuWAB0WkDbAH9036MGC97zmzgOHetmNUdZ6ItMUtVPOxV96kPO6beGEeFZE/4WoA9cXVBnpTVX/xYvgv0Bp4D/iHiDyC6676qBjvazzwpIhUADoCU1V1m9fd1UREunnbVcMV8FtV4PmVRGSe9/6XAP/zbf+CiNTHlagoF+P1zwYuEJE7vNsVgdrevowplCUKky4uAw4FWqjqTnHVYSv6N1DVqV4iOQ8YISKPA5uA/6lqzwCvcaeqjs67ISLtC9tIVZeJW/eiE/CAiHyoqoOCvAlV3S4ik4FzgEtwi+yAW3HsJlWdUMQutqlqUxHZH1fb6AbgKdxiTZNU9SJv4H9yjOcL0FVVlwaJ1xiwMQqTPqoB33lJ4gzgN+uCi1sr/FtVHQoMwy0JOR04TUTyxhwqi8ixAV/zI+BCEdlfRCrjuo0+EpEjga2q+hKuIGNh6w7v9Fo2hXkVV4wtr3UC7kP/urzniMix3msWSt2KhjcDt0t+mf28ctF9fJv+jOuCyzMBuEm85pW4ysPGxGWJwqSLl4EcEVkA9Aa+KGSbdsDnIjIX9239SVXdgPvgHCki83HdTscHeUFV/Qw3djETN2YxTFXnAicAM70uoHuBBwp5+hBgft5gdgHv4xaX+kDd0p3gEtti4DMRWYgrGx+3xe/FMh+3KM/fgYe89+5/3iSgYd5gNq7lUc6LbZF325i47PRYY4wxcVmLwhhjTFyWKIwxxsRlicIYY0xcliiMMcbEZYnCGGNMXJYojDHGxGWJwhhjTFz/D1SzEREX+2mUAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light",
            "tags": []
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "# Compute predicted probabilities on the test set\n",
        "probs = bert_predict(bert_classifier, val_dataloader)\n",
        "\n",
        "# Evaluate the Bert classifier\n",
        "evaluate_roc(probs, y_val)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UNR9pBOAGz8p",
        "outputId": "97c24e9c-27c5-4917-b4fc-1204c36845b4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Tokenizing data...\n",
            "Number of tweets predicted as Rumor:  122\n",
            "\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "           0       0.29      0.92      0.45       116\n",
            "           1       0.93      0.31      0.46       369\n",
            "\n",
            "    accuracy                           0.45       485\n",
            "   macro avg       0.61      0.61      0.45       485\n",
            "weighted avg       0.78      0.45      0.46       485\n",
            "\n",
            "0.4536082474226804\n",
            "0.4602851323828921\n"
          ]
        }
      ],
      "source": [
        "PATH = './Model/BERT_raw_to_fine_tune_ord4.pt'\n",
        "bert_classifier.load_state_dict(torch.load(PATH))\n",
        "testing_process(bert_classifier, X_val, y_val)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7eKaw3r2bAwg",
        "outputId": "e84554cf-46e2-4590-a4fe-84043acc2cc0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Tokenizing data...\n",
            "Number of tweets predicted as Rumor:  795\n",
            "\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "           0       0.10      0.94      0.18       475\n",
            "           1       0.96      0.16      0.28      4752\n",
            "\n",
            "    accuracy                           0.23      5227\n",
            "   macro avg       0.53      0.55      0.23      5227\n",
            "weighted avg       0.89      0.23      0.27      5227\n",
            "\n",
            "0.23225559594413622\n",
            "0.27654588065621055\n"
          ]
        }
      ],
      "source": [
        "rhi_data = pd.read_csv('data/_RHI_text.csv')\n",
        "rhi_y = pd.read_csv('data/_RHI_text.csv')\n",
        "X_test = rhi_data.text.values\n",
        "y_test = rhi_y.isRumor.values\n",
        "\n",
        "PATH = './Model/BERT_raw_to_fine_tune_ord4.pt'\n",
        "bert_classifier.load_state_dict(torch.load(PATH))\n",
        "testing_process(bert_classifier, X_test, y_test)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CSx6mKV2bgB_"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y2_qMkhDfb-6"
      },
      "source": [
        "# BERTweet"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yMD4JSaapG-T"
      },
      "source": [
        "## Functions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "VoalIiSJfhWq"
      },
      "outputs": [],
      "source": [
        "import re\n",
        "from tqdm import tqdm\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import time\n",
        "\n",
        "%matplotlib inline\n",
        "from sklearn.metrics import accuracy_score, roc_curve, auc, classification_report, f1_score\n",
        "\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "import torch.nn.functional as F\n",
        "\n",
        "import time\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from transformers import BertModel\n",
        "from transformers import AdamW, get_linear_schedule_with_warmup\n",
        "\n",
        "import random\n",
        "import time\n",
        "\n",
        "from transformers import AutoModel, AutoTokenizer\n",
        "from sklearn.metrics import accuracy_score, roc_curve, auc\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EbsgwY9ZfhWq"
      },
      "outputs": [],
      "source": [
        "def getDevice():\n",
        "  if torch.cuda.is_available():       \n",
        "      device = torch.device(\"cuda\")\n",
        "      print(f'There are {torch.cuda.device_count()} GPU(s) available.')\n",
        "      print('Device name:', torch.cuda.get_device_name(0))\n",
        "\n",
        "  else:\n",
        "      print('No GPU available, using the CPU instead.')\n",
        "      device = torch.device(\"cpu\")\n",
        "  return device\n",
        "\n",
        "def evaluate_roc(probs, y_true):\n",
        "    \"\"\"\n",
        "    - Print AUC and accuracy on the test set\n",
        "    - Plot ROC\n",
        "    @params    probs (np.array): an array of predicted probabilities with shape (len(y_true), 2)\n",
        "    @params    y_true (np.array): an array of the true values with shape (len(y_true),)\n",
        "    \"\"\"\n",
        "    preds = probs[:, 1]\n",
        "    fpr, tpr, threshold = roc_curve(y_true, preds)\n",
        "    roc_auc = auc(fpr, tpr)\n",
        "    print(f'AUC: {roc_auc:.4f}')\n",
        "\n",
        "    # Get accuracy over the test set\n",
        "    y_pred = np.where(preds >= 0.5, 1, 0)\n",
        "    accuracy = accuracy_score(y_true, y_pred)\n",
        "    print(f'Accuracy: {accuracy*100:.2f}%')\n",
        "\n",
        "    # Plot ROC AUC\n",
        "    plt.title('Receiver Operating Characteristic')\n",
        "    plt.plot(fpr, tpr, 'b', label='AUC = %0.2f' % roc_auc)\n",
        "    plt.legend(loc='lower right')\n",
        "    plt.plot([0, 1], [0, 1], 'r--')\n",
        "    plt.xlim([0, 1])\n",
        "    plt.ylim([0, 1])\n",
        "    plt.ylabel('True Positive Rate')\n",
        "    plt.xlabel('False Positive Rate')\n",
        "    plt.show()\n",
        "\n",
        "def text_preprocessing(text): # Create a function to tokenize a set of texts\n",
        "\n",
        "    \"\"\"\n",
        "    - Remove entity mentions (eg. '@united')\n",
        "    - Correct errors (eg. '&amp;' to '&')\n",
        "    @param    text (str): a string to be processed.\n",
        "    @return   text (Str): the processed string.\n",
        "    \"\"\"\n",
        "    # Remove '@name'\n",
        "    text = text.lower()\n",
        "    # text = re.sub(r'(@.*?)[\\s]', ' ', text)\n",
        "    # text = re.sub(r\"(#)(\\S+)\", r'\\1 \\2', text)\n",
        "\n",
        "    # text = re.sub(r\"http\\S+\", \"*\", text)  # http link -> '*'\n",
        "\n",
        "    # text = re.sub(r\"@\\S+\", \"@\", text)   # mention -> '@'\n",
        "    # text = re.sub(r\"@[^\\s]+\", \"@\", text)   # mention -> '@'\n",
        "\n",
        "    # sent = re.sub(r\"(#)(\\S+)\", r'\\1 \\2', sent)\n",
        "    # sent = re.sub(r'([^\\s\\w@#\\*]|_)+', '', sent) # Erasing Special Characters\n",
        "\n",
        "    # Replace '&amp;' with '&'\n",
        "    text = re.sub(r'&amp;', '&', text)\n",
        "\n",
        "    # Remove trailing whitespace\n",
        "    text = re.sub(r'\\s+', ' ', text).strip()\n",
        "\n",
        "    return text\n",
        "\n",
        "def preprocessing_for_bert(data): \n",
        "\n",
        "    \"\"\"Perform required preprocessing steps for pretrained BERT.\n",
        "    @param    data (np.array): Array of texts to be processed.\n",
        "    @return   input_ids (torch.Tensor): Tensor of token ids to be fed to a model.\n",
        "    @return   attention_masks (torch.Tensor): Tensor of indices specifying which\n",
        "                  tokens should be attended to by the model.\n",
        "    \"\"\"\n",
        "    # Create empty lists to store outputs (빈 리스트 2개 생성)\n",
        "    input_ids = []\n",
        "    attention_masks = []\n",
        "\n",
        "    # For every sentence...\n",
        "    for sent in data:\n",
        "        # `encode_plus` will:\n",
        "        #    (1) Tokenize the sentence\n",
        "        #    (2) Add the `[CLS]` and `[SEP]` token to the start and end\n",
        "        #    (3) Truncate/Pad sentence to max length\n",
        "        #    (4) Map tokens to their IDs\n",
        "        #    (5) Create attention mask\n",
        "        #    (6) Return a dictionary of outputs\n",
        "        encoded_sent = tokenizer.encode_plus(\n",
        "            text=text_preprocessing(sent),  # Preprocess sentence\n",
        "            add_special_tokens=True,        # Add `[CLS]` and `[SEP]`\n",
        "            # max_length=MAX_LEN,                  # Max length to truncate/pad\n",
        "            pad_to_max_length=True,         # Pad sentence to max length\n",
        "            # return_tensors='pt',           # Return PyTorch tensor\n",
        "            return_attention_mask=True,      # Return attention mask\n",
        "\n",
        "            # max_length=True,                  # Max length to truncate/pad\n",
        "            padding='max_length'\n",
        "        )\n",
        "\n",
        "        # Add the outputs to the lists (위의 빈 리스트에 상응하는 값 추가)\n",
        "        input_ids.append(encoded_sent.get('input_ids'))\n",
        "        attention_masks.append(encoded_sent.get('attention_mask'))\n",
        "\n",
        "    # Convert lists to tensors (리스트들을 텐서화)\n",
        "    input_ids = torch.tensor(input_ids)\n",
        "    attention_masks = torch.tensor(attention_masks)\n",
        "\n",
        "    return input_ids, attention_masks\n",
        "\n",
        "\n",
        "def initialize_model(epochs=4):\n",
        "    \"\"\"Initialize the Bert Classifier, the optimizer and the learning rate scheduler.\n",
        "    \"\"\"\n",
        "    # Instantiate Bert Classifier\n",
        "    bert_classifier = BertTweetClassifier(freeze_bert=False)\n",
        "\n",
        "    # Tell PyTorch to run the model on GPU\n",
        "    bert_classifier.to(device)\n",
        "\n",
        "    # Create the optimizer\n",
        "    optimizer = AdamW(bert_classifier.parameters(),\n",
        "                      lr=5e-5,    # Default learning rate\n",
        "                      eps=1e-8    # Default epsilon value\n",
        "                      )\n",
        "\n",
        "    # Total number of training steps\n",
        "    total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "    criterion = nn.CrossEntropyLoss()\n",
        "\n",
        "    # Set up the learning rate scheduler\n",
        "    scheduler = get_linear_schedule_with_warmup(optimizer,\n",
        "                                                num_warmup_steps=0,  # Default value\n",
        "                                                num_training_steps=total_steps)\n",
        "    return bert_classifier, optimizer, scheduler, criterion\n",
        "\n",
        "def set_seed(seed_value=42):\n",
        "    \"\"\"Set seed for reproducibility.\n",
        "    \"\"\"\n",
        "    random.seed(seed_value)\n",
        "    np.random.seed(seed_value)\n",
        "    torch.manual_seed(seed_value)\n",
        "    torch.cuda.manual_seed_all(seed_value)\n",
        "\n",
        "def train(model, train_dataloader, val_dataloader=None, epochs=4, evaluation=False):\n",
        "    \"\"\"Train the BertClassifier model.\n",
        "    \"\"\"\n",
        "    # Start training loop\n",
        "    print(\"Start training...\\n\")\n",
        "    for epoch_i in range(epochs):\n",
        "        # =======================================\n",
        "        #               Training\n",
        "        # =======================================\n",
        "        # Print the header of the result table\n",
        "        print(f\"{'Epoch':^7} | {'Batch':^7} | {'Train Loss':^12} | {'Val Loss':^10} | {'Val Acc':^9} | {'Elapsed':^9}\")\n",
        "        print(\"-\"*70)\n",
        "\n",
        "        # Measure the elapsed time of each epoch\n",
        "        t0_epoch, t0_batch = time.time(), time.time()\n",
        "\n",
        "        # Reset tracking variables at the beginning of each epoch\n",
        "        total_loss, batch_loss, batch_counts = 0, 0, 0\n",
        "\n",
        "        # Put the model into the training mode\n",
        "        model.train()\n",
        "\n",
        "        # For each batch of training data...\n",
        "        for step, batch in enumerate(train_dataloader):\n",
        "            batch_counts += 1\n",
        "            # Load batch to GPU\n",
        "            b_input_ids, b_attn_mask, b_labels = tuple(\n",
        "                t.to(device) for t in batch)\n",
        "\n",
        "            # Zero out any previously calculated gradients\n",
        "            model.zero_grad()\n",
        "\n",
        "            # Perform a forward pass. This will return logits.\n",
        "            logits = model(b_input_ids, b_attn_mask)\n",
        "\n",
        "            # Compute loss and accumulate the loss values\n",
        "            loss = loss_fn(logits, b_labels)\n",
        "            batch_loss += loss.item()\n",
        "            total_loss += loss.item()\n",
        "\n",
        "            # Perform a backward pass to calculate gradients\n",
        "            loss.backward()\n",
        "\n",
        "            # Clip the norm of the gradients to 1.0 to prevent \"exploding gradients\"\n",
        "            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "            # Update parameters and the learning rate\n",
        "            optimizer.step()\n",
        "            scheduler.step()\n",
        "\n",
        "            # Print the loss values and time elapsed for every 20 batches\n",
        "            if (step % 20 == 0 and step != 0) or (step == len(train_dataloader) - 1):\n",
        "                # Calculate time elapsed for 20 batches\n",
        "                time_elapsed = time.time() - t0_batch\n",
        "\n",
        "                # Print training results\n",
        "                print(\n",
        "                    f\"{epoch_i + 1:^7} | {step:^7} | {batch_loss / batch_counts:^12.6f} | {'-':^10} | {'-':^9} | {time_elapsed:^9.2f}\")\n",
        "\n",
        "                # Reset batch tracking variables\n",
        "                batch_loss, batch_counts = 0, 0\n",
        "                t0_batch = time.time()\n",
        "\n",
        "        # Calculate the average loss over the entire training data\n",
        "        avg_train_loss = total_loss / len(train_dataloader)\n",
        "\n",
        "        print(\"-\"*70)\n",
        "        # =======================================\n",
        "        #               Evaluation\n",
        "        # =======================================\n",
        "        if evaluation == True:\n",
        "            # After the completion of each training epoch, measure the model's performance\n",
        "            # on our validation set.\n",
        "            val_loss, val_accuracy = evaluate(model, val_dataloader)\n",
        "\n",
        "            # Print performance over the entire training data\n",
        "            time_elapsed = time.time() - t0_epoch\n",
        "\n",
        "            print(f\"{epoch_i + 1:^7} | {'-':^7} | {avg_train_loss:^12.6f} | {val_loss:^10.6f} | {val_accuracy:^9.2f} | {time_elapsed:^9.2f}\")\n",
        "            print(\"-\"*70)\n",
        "        print(\"\\n\")\n",
        "\n",
        "    print(\"Training complete!\")\n",
        "\n",
        "def evaluate(model, val_dataloader):\n",
        "    \"\"\"After the completion of each training epoch, measure the model's performance\n",
        "    on our validation set.\n",
        "    \"\"\"\n",
        "    # Put the model into the evaluation mode. The dropout layers are disabled during\n",
        "    # the test time.\n",
        "    model.eval()\n",
        "\n",
        "    # Tracking variables\n",
        "    val_accuracy = []\n",
        "    val_loss = []\n",
        "\n",
        "    # For each batch in our validation set...\n",
        "    for batch in val_dataloader:\n",
        "        # Load batch to GPU\n",
        "        b_input_ids, b_attn_mask, b_labels = tuple(t.to(device) for t in batch)\n",
        "\n",
        "        # Compute logits\n",
        "        with torch.no_grad():\n",
        "            logits = model(b_input_ids, b_attn_mask)\n",
        "\n",
        "        # Compute loss\n",
        "        loss = loss_fn(logits, b_labels)\n",
        "        val_loss.append(loss.item())\n",
        "\n",
        "        # Get the predictions\n",
        "        preds = torch.argmax(logits, dim=1).flatten()\n",
        "\n",
        "        # Calculate the accuracy rate\n",
        "        accuracy = (preds == b_labels).cpu().numpy().mean() * 100\n",
        "        val_accuracy.append(accuracy)\n",
        "\n",
        "    # Compute the average accuracy and loss over the validation set.\n",
        "    val_loss = np.mean(val_loss)\n",
        "    val_accuracy = np.mean(val_accuracy)\n",
        "\n",
        "    return val_loss, val_accuracy\n",
        "\n",
        "def bert_predict(model, test_dataloader):\n",
        "    \"\"\"Perform a forward pass on the trained BERT model to predict probabilities\n",
        "    on the test set.\n",
        "    \"\"\"\n",
        "    # Put the model into the evaluation mode. The dropout layers are disabled during\n",
        "    # the test time.\n",
        "    model.eval()\n",
        "\n",
        "    all_logits = []\n",
        "\n",
        "    # For each batch in our test set...\n",
        "    for batch in test_dataloader:\n",
        "        # Load batch to GPU\n",
        "        b_input_ids, b_attn_mask = tuple(t.to(device) for t in batch)[:2]\n",
        "\n",
        "        # Compute logits\n",
        "        with torch.no_grad():\n",
        "            logits = model(b_input_ids, b_attn_mask)\n",
        "        all_logits.append(logits)\n",
        "    \n",
        "    # Concatenate logits from each batch\n",
        "    all_logits = torch.cat(all_logits, dim=0)\n",
        "\n",
        "    # Apply softmax to calculate probabilities\n",
        "    probs = F.softmax(all_logits, dim=1).cpu().numpy()\n",
        "\n",
        "    return probs\n",
        "\n",
        "def data_process(X_train, y_train, X_val, y_val, batch_size=16):\n",
        "  # Concatenate train data and test data\n",
        "  all_tweets = np.concatenate([X_train, X_val])\n",
        "\n",
        "  # Encode our concatenated data\n",
        "  encoded_tweets = [tokenizer.encode(sent, add_special_tokens=True) for sent in all_tweets]\n",
        "\n",
        "  # Find the maximum length\n",
        "  max_len = max([len(sent) for sent in encoded_tweets])\n",
        "  print('Max length: ', max_len)\n",
        "\n",
        "  # Specify `MAX_LEN`\n",
        "  MAX_LEN = max_len\n",
        "\n",
        "  # Run function `preprocessing_for_bert` on the train set and the validation set\n",
        "  print('\\nTokenizing data...')\n",
        "  train_inputs, train_masks = preprocessing_for_bert(X_train)\n",
        "  val_inputs, val_masks = preprocessing_for_bert(X_val)\n",
        "\n",
        "  # Convert other data types to torch.Tensor\n",
        "  train_labels = torch.tensor(y_train)\n",
        "  val_labels = torch.tensor(y_val)\n",
        "\n",
        "  # For fine-tuning BERT, the authors recommend a batch size of 16 or 32.\n",
        "  batch_size = 8\n",
        "\n",
        "  # Create the DataLoader for our training set\n",
        "  train_data = TensorDataset(train_inputs, train_masks, train_labels)\n",
        "  train_sampler = RandomSampler(train_data)\n",
        "  train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
        "\n",
        "  # Create the DataLoader for our validation set\n",
        "  val_data = TensorDataset(val_inputs, val_masks, val_labels)\n",
        "  val_sampler = SequentialSampler(val_data)\n",
        "  val_dataloader = DataLoader(val_data, sampler=val_sampler, batch_size=batch_size)\n",
        "\n",
        "  return train_dataloader, val_dataloader\n",
        "\n",
        "def train_process(train_dataloader, val_dataloader, epoch=4):\n",
        "  set_seed(42)    # Set seed for reproducibility\n",
        "  bert_classifier, optimizer, scheduler, loss_fn = initialize_model(epochs=epoch)\n",
        "\n",
        "  train(bert_classifier, train_dataloader, loss_fn, epochs=4, evaluation=True)\n",
        "\n",
        "  return bert_classifier\n",
        "\n",
        "def testing_process(bert_classifier, X_val, y_val):\n",
        "  #  Run `preprocessing_for_bert` on the test set\n",
        "  print('Tokenizing data...')\n",
        "  test_inputs, test_masks = preprocessing_for_bert(X_val)\n",
        "\n",
        "  # Create the DataLoader for our test set\n",
        "  test_dataset = TensorDataset(test_inputs, test_masks)\n",
        "  test_sampler = SequentialSampler(test_dataset)\n",
        "  test_dataloader = DataLoader(test_dataset, sampler=test_sampler, batch_size=32)\n",
        "\n",
        "  # Compute predicted probabilities on the test set\n",
        "  probs = bert_predict(bert_classifier, test_dataloader)\n",
        "\n",
        "  # Get predictions from the probabilities\n",
        "  threshold = 0.5\n",
        "  preds = np.where(probs[:, 1] > threshold, 1, 0)\n",
        "\n",
        "  # Number of tweets predicted non-negative\n",
        "  print(\"Number of tweets predicted as Rumor: \", preds.sum())\n",
        "\n",
        "  preds = np.argmax(probs, axis = 1)\n",
        "  print(\"\\n\",classification_report(y_val, preds))\n",
        "  print('Accuracy Score:\\t',accuracy_score(y_val, preds))\n",
        "  print('Precision Score:\\t', str(precision_score(y_val,preds)))\n",
        "  print('Recall Score:\\t\\t' + str(recall_score(y_val,preds)))\n",
        "  print('F1 Score:\\t',f1_score(y_val, preds, zero_division=1))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Class"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {},
      "outputs": [],
      "source": [
        "class BertTweetClassifier(nn.Module): # Create the BertClassfier class\n",
        "    \"\"\"Bert Model for Classification Tasks.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, freeze_bert=False):\n",
        "        \"\"\"\n",
        "        @param    bert: a BertModel object\n",
        "        @param    classifier: a torch.nn.Module classifier\n",
        "        @param    freeze_bert (bool): Set `False` to fine-tune the BERT model\n",
        "        \"\"\"\n",
        "        super(BertTweetClassifier, self).__init__()\n",
        "        # Specify hidden size of BERT, hidden size of our classifier, and number of labels\n",
        "        D_in, H, D_out = 768, 50, 2\n",
        "\n",
        "        # Instantiate BERT model\n",
        "        self.bert = AutoModel.from_pretrained(\"vinai/bertweet-base\")\n",
        "\n",
        "\n",
        "        # Instantiate an one-layer feed-forward classifier\n",
        "        self.classifier = nn.Sequential(\n",
        "            nn.Linear(D_in, H),\n",
        "            nn.ReLU(),\n",
        "            # nn.Dropout(0.5),\n",
        "            nn.Linear(H, D_out)\n",
        "        )\n",
        "\n",
        "        # Freeze the BERT model\n",
        "        if freeze_bert:\n",
        "            for param in self.bert.parameters():\n",
        "                param.requires_grad = False\n",
        "\n",
        "    def forward(self, input_ids, attention_mask):\n",
        "        \"\"\"\n",
        "        Feed input to BERT and the classifier to compute logits.\n",
        "        @param    input_ids (torch.Tensor): an input tensor with shape (batch_size,\n",
        "                      max_length)\n",
        "        @param    attention_mask (torch.Tensor): a tensor that hold attention mask\n",
        "                      information with shape (batch_size, max_length)\n",
        "        @return   logits (torch.Tensor): an output tensor with shape (batch_size,\n",
        "                      num_labels)\n",
        "        \"\"\"\n",
        "        # Feed input to BERT\n",
        "        outputs = self.bert(input_ids=input_ids,\n",
        "                            attention_mask=attention_mask)\n",
        "\n",
        "        # Extract the last hidden state of the token `[CLS]` for classification task\n",
        "        last_hidden_state_cls = outputs[0][:, 0, :]\n",
        "\n",
        "        # Feed input to classifier to compute logits\n",
        "        logits = self.classifier(last_hidden_state_cls)\n",
        "\n",
        "        return logits\n",
        "\n",
        "    def embedding(self, input_ids):\n",
        "        outputs = self.bert(input_ids=input_ids)\n",
        "\n",
        "        # outputs = (sequence_output, pooled_output,) + encoder_outputs[1:]  # add hidden_states and attentions if they are here\n",
        "        return outputs  # sequence_output, pooled_output, (hidden_states), (attentions)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M181HsetpgSU"
      },
      "source": [
        "## Execution"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 460
        },
        "id": "v334DoYDfhWs",
        "outputId": "2a3379fd-0900-4d9e-d198-b4a6333d585b"
      },
      "outputs": [
        {
          "ename": "OSError",
          "evalue": "ignored",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-53-faca395afa4d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mraw_text\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'./data/_PHEME_text.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'./data/_PHEME_target.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mraw_text\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'data/_PHEMEext_text.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m    686\u001b[0m     )\n\u001b[1;32m    687\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 688\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    689\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    690\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    452\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    453\u001b[0m     \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 454\u001b[0;31m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfp_or_buf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    455\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    456\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m    946\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    947\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 948\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    949\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    950\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m   1178\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"c\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1179\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"c\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1180\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1181\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1182\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"python\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m   2008\u001b[0m         \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"usecols\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2009\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2010\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2011\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2012\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;31mOSError\u001b[0m: [Errno 107] Transport endpoint is not connected: './data/_PHEME_text.csv'"
          ]
        }
      ],
      "source": [
        "raw_text = pd.read_csv('./data/_PHEME_text.csv')\n",
        "y = pd.read_csv('./data/_PHEME_target.csv')\n",
        "data = pd.concat([raw_text.text, y], axis=1).reset_index(drop=True)\n",
        "val = pd.read_csv('data/_PHEMEext_text.csv')\n",
        "\n",
        "X_train = data.text.values\n",
        "y_train = data.target.values\n",
        "\n",
        "X_val = val.drop(['Event'],axis=1).text.values\n",
        "y_val = val.target.values\n",
        "\n",
        "rhi_data = pd.read_csv('data/_RHI_text.csv').text.values\n",
        "rhi_y = pd.read_csv('data/_RHI_text.csv').isRumor"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OwRCb2rGsFQ8",
        "outputId": "1cbd2912-896a-4471-e4ef-3404c95a3919"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(5802,)\n",
            "(5802,)\n",
            "(485,)\n",
            "(485,)\n"
          ]
        }
      ],
      "source": [
        "print(X_train.shape)\n",
        "print(y_train.shape)\n",
        "print(X_val.shape)\n",
        "print(y_val.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4ZUhevayfhWu",
        "outputId": "4294347a-ac32-4e50-9641-1c9fb768233c"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "emoji is not installed, thus not converting emoticons or emojis into text. Please install emoji: pip3 install emoji\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "There are 1 GPU(s) available.\n",
            "Device name: Tesla T4\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embedding are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Max length:  58\n",
            "\n",
            "Tokenizing data...\n"
          ]
        }
      ],
      "source": [
        "device = getDevice()\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"vinai/bertweet-base\", use_fast=False)\n",
        "train_dataloader, val_dataloader = data_process(X_train, y_train, X_val, y_val, batch_size=8)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x-HVi7I2uVey",
        "outputId": "b0f83d49-358d-420f-d664-d4b40a576f6d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "There are 1 GPU(s) available.\n",
            "Device name: Tesla T4\n"
          ]
        }
      ],
      "source": [
        "if torch.cuda.is_available():       \n",
        "      device = torch.device(\"cuda\")\n",
        "      print(f'There are {torch.cuda.device_count()} GPU(s) available.')\n",
        "      print('Device name:', torch.cuda.get_device_name(0))\n",
        "\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vtfc0vdYfhWu",
        "outputId": "e3c09e56-86a9-4cc1-808e-b1fd61430657"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Start training...\n",
            "\n",
            " Epoch  |  Batch  |  Train Loss  |  Val Loss  |  Val Acc  |  Elapsed \n",
            "----------------------------------------------------------------------\n",
            "   1    |   20    |   0.657699   |     -      |     -     |   4.73   \n",
            "   1    |   40    |   0.572159   |     -      |     -     |   4.54   \n",
            "   1    |   60    |   0.599299   |     -      |     -     |   4.65   \n",
            "   1    |   80    |   0.502215   |     -      |     -     |   4.70   \n",
            "   1    |   100   |   0.424222   |     -      |     -     |   4.74   \n",
            "   1    |   120   |   0.471874   |     -      |     -     |   4.66   \n",
            "   1    |   140   |   0.555176   |     -      |     -     |   4.64   \n",
            "   1    |   160   |   0.485429   |     -      |     -     |   4.55   \n",
            "   1    |   180   |   0.442740   |     -      |     -     |   4.51   \n",
            "   1    |   200   |   0.574039   |     -      |     -     |   4.47   \n",
            "   1    |   220   |   0.487704   |     -      |     -     |   4.47   \n",
            "   1    |   240   |   0.527812   |     -      |     -     |   4.45   \n",
            "   1    |   260   |   0.427125   |     -      |     -     |   4.44   \n",
            "   1    |   280   |   0.511334   |     -      |     -     |   4.43   \n",
            "   1    |   300   |   0.385671   |     -      |     -     |   4.40   \n",
            "   1    |   320   |   0.463129   |     -      |     -     |   4.42   \n",
            "   1    |   340   |   0.411448   |     -      |     -     |   4.40   \n",
            "   1    |   360   |   0.424775   |     -      |     -     |   4.43   \n",
            "   1    |   380   |   0.455251   |     -      |     -     |   4.43   \n",
            "   1    |   400   |   0.397658   |     -      |     -     |   4.45   \n",
            "   1    |   420   |   0.486652   |     -      |     -     |   4.47   \n",
            "   1    |   440   |   0.442327   |     -      |     -     |   4.49   \n",
            "   1    |   460   |   0.430557   |     -      |     -     |   4.50   \n",
            "   1    |   480   |   0.434291   |     -      |     -     |   4.51   \n",
            "   1    |   500   |   0.459816   |     -      |     -     |   4.51   \n",
            "   1    |   520   |   0.394905   |     -      |     -     |   4.52   \n",
            "   1    |   540   |   0.542678   |     -      |     -     |   4.60   \n",
            "   1    |   560   |   0.493274   |     -      |     -     |   4.54   \n",
            "   1    |   580   |   0.518410   |     -      |     -     |   4.49   \n",
            "   1    |   600   |   0.466646   |     -      |     -     |   4.48   \n",
            "   1    |   620   |   0.421744   |     -      |     -     |   4.46   \n",
            "   1    |   640   |   0.436818   |     -      |     -     |   4.47   \n",
            "   1    |   660   |   0.485056   |     -      |     -     |   4.46   \n",
            "   1    |   680   |   0.715749   |     -      |     -     |   4.45   \n",
            "   1    |   700   |   0.601952   |     -      |     -     |   4.43   \n",
            "   1    |   720   |   0.416572   |     -      |     -     |   4.43   \n",
            "   1    |   725   |   0.777217   |     -      |     -     |   1.00   \n",
            "----------------------------------------------------------------------\n",
            "   1    |    -    |   0.489019   |  0.858612  |   62.62   |  167.24  \n",
            "----------------------------------------------------------------------\n",
            "\n",
            "\n",
            "Training complete!\n"
          ]
        }
      ],
      "source": [
        "set_seed(42)    # Set seed for reproducibility\n",
        "bert_classifier, optimizer, scheduler, loss_fn = initialize_model(epochs=4)\n",
        "train(bert_classifier, train_dataloader, val_dataloader, epochs=1, evaluation=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xqI9LRVbfhWu"
      },
      "outputs": [],
      "source": [
        "torch.save(bert_classifier.state_dict(), './Model/BERTweet_raw_to_fine_tune_ord4.pt')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 330
        },
        "id": "1E0J9rlrfhWu",
        "outputId": "94c6ed92-673a-473e-a3c8-1cd4973459bf"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "AUC: 0.4855\n",
            "Accuracy: 62.89%\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5gUVdbH8e8hIyIquAYQZQUUUIKMYCSIARFBF0RkRVEEs5jYNQdeVtfVNa2uAuJiBBUTRtSVYBYQJIoiLjAgiogKIkg47x+3hmnGmZ5mmO6e7vl9nqef6Qpddbpmpk/fulXnmrsjIiJSlArpDkBERMo2JQoREYlLiUJEROJSohARkbiUKEREJC4lChERiUuJQraJmc0xsw7pjqOsMLNrzezhNO17lJkNTce+S5uZ/dnM3izha/U3mWRKFBnMzP5nZr+a2RozWx59cOyYzH26ezN3n5jMfeQxs6pmdpuZLY7e55dmNtjMLBX7LySeDmaWGzvP3W9193OTtD8zs0vNbLaZ/WJmuWb2rJkdlIz9lZSZ3WxmT2zPNtz9SXc/LoF9/S45pvJvsrxSosh8J7n7jkBLoBVwTZrj2WZmVqmIRc8CnYAuQE2gLzAQuDcJMZiZlbX/h3uBQcClwK5AY+BF4MTS3lGc30HSpXPfkiB31yNDH8D/gGNipv8BvBozfSjwAfAj8BnQIWbZrsB/gGXAKuDFmGVdgRnR6z4AmhfcJ7AX8Cuwa8yyVsD3QOVo+hxgXrT98cA+Mes6cBHwJfB1Ie+tE7AO2LvA/LbAJqBhND0RuA34BPgZeKlATPGOwUTgb8D70XtpCJwdxbwaWAicF61bI1pnM7AmeuwF3Aw8Ea2zb/S+zgIWR8fiupj9VQcejY7HPOAvQG4Rv9tG0ftsE+f3Pwp4AHg1ivdjYL+Y5fcCS6LjMg04KmbZzcBY4Ilo+blAG+DD6Fh9A9wPVIl5TTPgLeAH4FvgWqAz8BuwITomn0Xr1gJGRttZCgwFKkbL+kXH/G5gZbSsH/BetNyiZd9Fsc0CDiR8SdgQ7W8N8HLB/wOgYhTXV9ExmUaBvyE9SvBZk+4A9NiOX97W/yD1on+oe6PputE/YRdCy/HYaHq3aPmrwNPALkBloH00v1X0D9o2+qc7K9pP1UL2+Q4wICaeO4CHoufdgQVAE6AScD3wQcy6Hn3o7ApUL+S9/R2YVMT7XkT+B/jE6IPoQMKH+XPkf3AXdwwmEj7Qm0UxViZ8W98v+rBqD6wFDo7W70CBD3YKTxQjCEmhBbAeaBL7nqJjXg+YWXB7Mds9H1hUzO9/VPR+2kTxPwmMiVl+BlA7WnYlsByoFhP3BuDk6NhUB1oTEmul6L3MAy6L1q9J+NC/EqgWTbcteAxi9v0CMCz6nfyBkMjzfmf9gI3AJdG+qrN1ojie8AG/c/R7aALsGfOeh8b5PxhM+D/YP3ptC6B2uv9XM/2R9gD02I5fXvgHWUP45uTAf4Gdo2V/BR4vsP54wgf/noRvxrsUss0Hgf8rMG8++Ykk9p/yXOCd6LkRvr22i6ZfB/rHbKMC4UN3n2jagaPjvLeHYz/0Ciz7iOibOuHD/u8xy5oSvnFWjHcMYl47pJhj/CIwKHregcQSRb2Y5Z8AvaPnC4HjY5adW3B7McuuAz4qJrZRwMMx012Az+OsvwpoERP35GK2fxnwQvT8dGB6EettOQbR9O6EBFk9Zt7pwIToeT9gcYFt9CM/URwNfEFIWhUKec/xEsV8oHsy/t/K86OsnZOVbXeyu9ckfIgdANSJ5u8DnGpmP+Y9gCMJSWJv4Ad3X1XI9vYBrizwur0Jp1kKeg44zMz2BNoRks+7Mdu5N2YbPxCSSd2Y1y+J876+j2ItzJ7R8sK2s4jQMqhD/GNQaAxmdoKZfWRmP0TrdyH/mCZqeczztUDeBQZ7FdhfvPe/kqLffyL7wsyuMrN5ZvZT9F5qsfV7KfjeG5vZK9GFET8Dt8asvzfhdE4i9iH8Dr6JOe7DCC2LQvcdy93fIZz2egD4zsyGm9lOCe57W+KUBClRZAl3n0T4tnVnNGsJ4dv0zjGPGu7+92jZrma2cyGbWgL8rcDrdnD30YXscxXwJnAa0IfQAvCY7ZxXYDvV3f2D2E3EeUtvA23NbO/YmWbWlvBh8E7M7Nh16hNOqXxfzDH4XQxmVpWQ/O4Ednf3nYHXCAmuuHgT8Q3hlFNhcRf0X6CemeWUZEdmdhShD6QXoeW4M/AT+e8Ffv9+HgQ+Bxq5+06Ec/156y8B/ljE7gpuZwmhRVEn5rjv5O7N4rxm6w263+furQktxMaEU0rFvi7a937FrCPbSIkiu9wDHGtmLQidlCeZ2fFmVtHMqkWXd9Zz928Ip4b+bWa7mFllM2sXbWMEcL6ZtY2uBKphZieaWc0i9vkUcCbQM3qe5yHgGjNrBmBmtczs1ETfiLu/TfiwfM7MmkXv4dDofT3o7l/GrH6GmTU1sx2AIcBYd98U7xgUsdsqQFVgBbDRzE4AYi/Z/BaobWa1En0fBTxDOCa7mFld4OKiVoze37+B0VHMVaL4e5vZ1QnsqyahH2AFUMnMbgSK+1Zek9B5vMbMDgAuiFn2CrCnmV0WXbZcM0raEI7LvnlXjUV/X28C/zSzncysgpntZ2btE4gbMzsk+vurDPxCuKhhc8y+ikpYEE5Z/p+ZNYr+fpubWe1E9itFU6LIIu6+AngMuNHdlxA6lK8lfFgsIXwry/ud9yV88/6c0Hl9WbSNqcAAQtN/FaFDul+c3Y4jXKGz3N0/i4nlBeB2YEx0GmM2cMI2vqUewATgDUJfzBOEK2kuKbDe44TW1HJCR+ulUQzFHYOtuPvq6LXPEN57n+j95S3/HBgNLIxOqRR2Oi6eIUAu8DWhxTSW8M27KJeSfwrmR8IplVOAlxPY13jCcfuCcDpuHfFPdQFcRXjPqwlfGJ7OWxAdm2OBkwjH+UugY7T42ejnSjP7NHp+JiHxziUcy7EkdioNQkIbEb1uEeE03B3RspFA0+j4v1jIa+8i/P7eJCS9kYTOctkOln+mQCTzmNlEQkdqWu6O3h5mdgGhozuhb9oi6aIWhUiKmNmeZnZEdCpmf8Klpi+kOy6R4iQtUZjZI2b2nZnNLmK5mdl9ZrbAzGaa2cHJikWkjKhCuPpnNaEz/iVCP4RImZa0U09R5+ga4DF3P7CQ5V0I55q7EG7uutfd2xZcT0RE0itpLQp3n0y4dr4o3QlJxN39I2Dn6Hp8EREpQ9JZjKsuW1+FkRvN+6bgimY2kFDnhRo1arQ+4IADUhKgiEjGW7SIjSt/5DPf+L2771aSTWRE1UZ3Hw4MB8jJyfGpU6emOSIRkbJh+HB46qkCM/O6FMzotsuDrF/9HZ+tv3lRSfeRzkSxlK3vTK0XzRMRkThik8OkSeFn++gi6zrrl3L5lxcwYbfTeHv3PzNurwtC8ZhJN5d4f+lMFOOAi81sDKEz+6fojk4REYkU1mKITQ7t20OfPjBwgMPDD8NVV8GGDRxx2Ylcf17+a7ZnuK+kJQozG00oVFfHwqhgNxEKheHuDxFq6HQh3Pm7ljAOgIhIuRevxZD3vE8fGDgwmvHVV9BpAEyYAB07wogRsF/plbxKWqJw99OLWe6EgWtERCTGU0/BjBnQsmUhSaEws2bBtGkhw5x77vY1HwqREZ3ZIiLZLrYVkZckJk6M84LZs+HTT+HMM+Hkk2HhQqidnPqHKuEhIlIG5LUiICSJPn2KWPG33+Dmm+Hgg+G662DdujA/SUkC1KIQEUmpQi9nJcFWxMcfQ//+MGcOnHEG3H03VKuWrFC3UKIQEUmy4jqnoZhWBMDSpXDUUbD77vDKK3DiiUmJtTBKFCIiSVBUckioczrWF19A48ZQty48/TR06gQ7JToybOlQohARKSWllhwAfvwR/vKXcG/ExInQrh2cckpph5wQJQoRkVKyzZe1FmXcOLjgAli+HAYPhkMOKfVYt4UShYhIKSq2Q7o4554LI0fCQQfBSy9BTk5phVZiShQiIukWU8SPnBzYZx/461+hSpX0xhVRohAR2Q6F3Si3TZYsgfPPh969oW/f8LyM0Q13IiLbIeEb5QravBkefBCaNQvnqtavT1aI200tChGR7bTN/RJffhn6IiZPhmOOCc2SBg2SFd52U6IQEUm1uXNh5kx45BHo16/Ui/iVNiUKEZFU+OyzcI7qrLOge/dQxG+XXdIdVULURyEikkzr18MNN4SrmW64Ib+IX4YkCVCiEBFJng8/hFatYOjQ0Ms9fXpKiviVNp16EhFJhqVLw+3Ze+wBr70GJ5yQ7ohKTC0KEZHSNG9e+Fm3LjzzTCgJnsFJApQoRERKx6pVcM450LQpvPtumHfyyVCzZnrjKgVKFCIiJTB8OHToEC5kOvL7F0KCeOwxuOaatBfxK23qoxARSVBhZcRf3eMcusz5T7jr7tVXwxClWUaJQkQkQVvKiLdw2reDPn82unAorGwEV10FlSunO8SkUKIQEYmjYNG/4w9YxNPVzwuXu555JlCSAScyixKFiEgBhZ1i6tBuMzfv9iAXzrgaKjmcemr6AkwxdWaLiBQQWxG2fXsYc8t8Jmxuz2ULLqZK+8Nh9mzo3z+9QaaQWhQiIoXYqiLsuPlwzxwYNSqcbirjRfxKmxKFiJRbsaeYYs2YAT33mw7/mQFnnw3duoUifjvvnPogywCdehKRciv2FFOeKpvXcf9O1zJ8xiFw8835RfzKaZIAtShEpJzb6hTT+++Hvocl80NL4p//zMgifqVNiUJEBEIRv44dQ42m8ePhuOPSHVGZoUQhIuVKwfsiujeaCzQNCeK550Ky2HHHtMZY1qiPQkTKlbx+iZobfmBMtX48OrVZGLsa4KSTlCQKoRaFiJQ7l9Z9jiFfXQQrV8J110GbNukOqUxTohCRrBd7umngB/3os+HRULzvjTdCb7bEpVNPIpL1nnrSmTHdAVi27+F8fMrf4eOPlSQSlNQWhZl1Bu4FKgIPu/vfCyyvDzwK7Bytc7W7v5bMmESknPn6a+6cOZC39jyDayaeRXko4lfaktaiMLOKwAPACUBT4HQza1pgteuBZ9y9FdAb+Hey4hGRcmbTJrjvPjjwQJqu/gjD0x1Rxkrmqac2wAJ3X+juvwFjgO4F1nFgp+h5LWBZEuMRkfJi3jw46igYNAjat6dfzhze2KNfuqPKWMlMFHWBJTHTudG8WDcDZ5hZLvAacElhGzKzgWY21cymrlixIhmxikg2WbCAdTPn87cDHqfDL6/y5uf10x1RRkt3Z/bpwCh3rwd0AR43s9/F5O7D3T3H3XN22223lAcpIhlg2jR45JHw/KST+FPLr7njmzPAjJYtwzhDUjLJ7MxeCuwdM10vmherP9AZwN0/NLNqQB3guyTGJSLZ5Ndf4ZZb4M47Ye+9Q0aoVo21lXbauo6TlFgyWxRTgEZm1sDMqhA6q8cVWGcx0AnAzJoA1QCdWxKRxEyeDC1awO23Q79+MH26ivglQdJaFO6+0cwuBsYTLn19xN3nmNkQYKq7jwOuBEaY2eWEju1+7q5LE0SkeEuXQqdOoRXx9tvhuSRFUu+jiO6JeK3AvBtjns8FjkhmDCKSZWbNgoMOCkX8XnghFPGrUSPdUWW1dHdmi4gk5vvvoW9faN48v4hf165KEimgRCEiZZs7PPMMNG0KY8bATTdB27ZFrj58OHTo8PuR66TkVBRQRMq2s86Cxx+HnBz473/DaacCYov+TZoUfrZvr0tiS4sShYiUPXnXtJiFT/zmzeGyy6BS/kdWUckhL0EMVEmnUqNEISJly8KFMGAAnHEGnH02wzf156lXgFe2Xk3JIXWUKESkTBjx0CbW3/kv+n99HZutIvctPZPxj26dEGIpOaSOEoWIpN/cuRzxl3NouvpjPtz1RO5q/BArqtYDlBDKAiUKEUmbvH6GQ1d+zVVrvmJIk6e4cU5vnjVLd2gSQ4lCRNJjyhR++ucMZnw7AFqeSN/DF3LKmTVBOaLMUaIQkdRauxZuvBHuvps/V9mHtw7py5sTqwE10x2ZFEE33IlISgwfDpe1nMjS2s3hn/9k3O4DOLTKdH6roCJ+ZZ1aFCKSEm/9J5enPjuW76rtw2XN32HGLh35I7opLhMoUYhIcn32GbRowYqq9bj+wJe4/eMO3LPDDumOSraBTj2JSHKsWBGaCy1bbrkZ4uPaXUBJIuOoRSEipcs9FO+79FL46acw+txhh6U7KtkOShQiUrr69oUnnwwVXkeOhGbN0h2RbKeEE4WZ7eDua5MZjIhkqM2bQwE/szCQUOvWoUVRsWK6I5NSUGwfhZkdbmZzgc+j6RZm9u+kRyYimWHBgjAM6X/+E6b794fLL4eKFbeMDaHxITJbIp3ZdwPHAysB3P0zoF0ygxKRDLBxI9x5ZxgfYvp0qFIFYKvkcN55+UX9WrbUpbCZKqFTT+6+xLauvbIpOeGISEaYPRvOPhumToXu3eHf/4a99gJC7aYZM0JiUEG/7JBIolhiZocDbmaVgUHAvOSGJSJl2uLFsGhRuLqpV6/QNxGjZUuYODE9oUnpSyRRnA/cC9QFlgJvAhcmMygRKYM+/jjcPDdwIHTpEgYY2nHHdEclKZBIH8X+7v5nd9/d3f/g7mcATZIdmIiUEb/8AldcEe6F+Mc/YP36MD8mSajTOrslkij+leA8Eck277wTxqu++244/3z49FOoWvV3q+X1S4A6rbNRkaeezOww4HBgNzO7ImbRToAujhbJdrm5cPzx0KBBuHSp3dYXO+YNOgT5ndfql8hO8fooqgA7RuvEFor/GeiZzKBEJI2mT4dWraBePXj55XDpUvXqwNbJIXYsa7Uispu5e/wVzPZx90UpiqdYOTk5PnXq1HSHIZJ9vv023E39zDOhadC+PVB0cgBd+ppJzGyau+eU5LWJXPW01szuAJoBW0YYcfejS7JDESlj3ENtpkGDYM0aGDoUDj98y2LdFyGJJIongaeBroRLZc8CViQzKBFJnQVt+9Bwyhhm73QY/2g+ksVvNYG38per/0ESSRS13X2kmQ1y90nAJDObkuzARCSJYor4jf35OFZVO4wpLS9is/3+OhX1P0giiWJD9PMbMzsRWAbsmryQRCSpvvgCBgyAM8+E/v15Y4+zYQ+1GKRoiSSKoWZWC7iScP/ETsBlSY1KRErfxo1w111w001QrdqWK5lEilNsonD3V6KnPwEdAczsiGQGJSKlbOZMOOccmDYNTjkFHngA9twz3VFJhoh3w11FoBehxtMb7j7bzLoC1wLVgVapCVFEtltuLixZAs8+Cz16/K6In0g88Up4jATOBWoD95nZE8CdwD/cPaEkYWadzWy+mS0ws6uLWKeXmc01szlm9tS2vgERKcIHH8BDD4XneUX8evZUkpBtFu/UUw7Q3N03m1k1YDmwn7uvTGTDUYvkAeBYIBeYYmbj3H1uzDqNgGuAI9x9lZn9oaRvREQia9bAddfBv/4F++0Xxo2oWhVq1NiySmHlN0SKEq9F8Zu7bwZw93XAwkSTRKQNsMDdF7r7b8AYoHuBdQYAD7j7qmg/323D9kWkoDffhAMPDEniootUxE9KRbwWxQFmNjN6bsB+0bQB7u7Ni9l2XWBJzHQu0LbAOo0BzOx9QqHBm939jYIbMrOBwECA+vXrF7NbkXJqyRI48cTQipg8GY48Mu7quolOEhUvUaRizIlKQCOgA1APmGxmB7n7j7EruftwYDiEWk8piEskc0ybBq1bw957w2uvwVFHhctfRUpJkaee3H1RvEcC214K7B0zXS+aFysXGOfuG9z9a+ALQuIQkeIsXw6nngo5OfnV+o49tsgkocGFpKQSueGupKYAjcysASFB9AYKngl9ETgd+I+Z1SGcilqYxJhEMp87PPYY6y68nAq/rmVUg1t5+obD2VTMMGQqCy4llbRE4e4bzexiYDyh/+ERd59jZkOAqe4+Llp2nJnNBTYBg7exw1ykXBk+HBpe35ujVzzDVI7gXB5mj/oHJPRaVX6Vkip2PAoAM6sO1Hf3+ckPKT6NRyHlUlTEr0NHo8knj3JA3dW8uNeFnP7nCvrgl4Rsz3gUxY6ZbWYnATOAN6LplmY2riQ7E5ES+PzzMAzpyJEAzGtzFoO+vJgJk5QkJDWKTRTAzYR7In4EcPcZQIMkxiQiABs2wK23QosWrJs+lyF37ahOaEmLhMqMu/tPtvVt/7pEVSSZZszg+25nU2fJDCbW6Unv7//Ft/P22NLPIJJKiSSKOWbWB6gYldy4FPgguWGJlHPLl2PfLqfvDs+xpNmfOAAYoo5oSZNEEsUlwHXAeuApwpVKQ5MZlEi59N57oRz4hRdC5870afMV6yvuoLunJe0S6aM4wN2vc/dDosf1Ue0nESkNq1fDxReHO6rvuQfWrwdgfcUd0hyYSJBIi+KfZrYHMBZ42t1nJzkmkfJj/PhwPmnJEhg0iEf+OJTHjg9F/FTVVcqKYlsU7t6RMLLdCmCYmc0ys+uTHplItluyBLp2hR12CKed7rmHx57fUVVdpcxJ6M5sd19OGLxoAvAX4EbUTyGy7dxhyhRo0yYU8Xv99VDlNaY+k6q6SlmTyA13TczsZjObBfyLcMVTvaRHJpJtvvkmDEPatm1+4aVjjlGlVynzEmlRPAI8DRzv7suSHI9I9nGHUaPgiitg3Tq4/XY44oh0RyWSsGIThbsflopARLJWr14wdmy4qunhh6Fx43RHJLJNikwUZvaMu/eKTjnF3omd6Ah3IuXXpk1gBhUqwEknwdFHw3nnhWmRDBOvRTEo+tk1FYGIZI1586B/fzj7bBgwAM48M+7qw4eHMaxBl8RK2RRvhLtvoqcXFjK63YWpCU8kg2zYAEOHhk/6+fOhVq2EXvbUU+iSWCnTEunMPhb4a4F5JxQyT6T8mj4d+vULJThOOw3uuw/+8IeEX65LYqUsi9dHcQGh5fBHM5sZs6gm8H6yAxPJKN9+C99/Dy++CN27pzsakVIVr0XxFPA6cBtwdcz81e7+Q1KjEskEkyfDrFlw0UXQuTMsWADVqyf0UvVLSCaJlyjc3f9nZhcVXGBmuypZSLn1889w9dXw4IPhUtdzz4WqVYtNErHJIe9+u/bt1S8hZV9xLYquwDTC5bGxIxc58MckxiVSNr32WrjMddmycAPdkCEhSSQgr9O6ZUu2DECk8SUkExSZKNy9a/RTw56KQCji17077L9/uIGubdtt3oQ6rSUTJVLr6QgzqxE9P8PM7jKz+skPTaQMcIePPgrP994b3nwTPv004SQxfDh06BAeGu9aMlUit4k+CKw1sxbAlcBXwONJjUqkLFi2DE4+GQ47LL9ToWNHqFIl4U3oHgnJBoncR7HR3d3MugP3u/tIM+uf7MBE0sYdRo6Eq64Ko83deed2FfHT6SbJdIkkitVmdg3QFzjKzCoAlZMblkga9ewJzz8fepwffhgaNkx3RCJplcipp9OA9cA50QBG9YA7khqVSKpt2gSbN4fnJ58MDz0E77yjJCFCYkOhLgeeBGqZWVdgnbs/lvTIRFJl9uxwamnkyDDdt68qvYrESOSqp17AJ8CpQC/gYzPrmezARJLut9/gllvg4IPhq69gl13SHZFImZRIH8V1wCHu/h2Ame0GvA2MTWZgIkk1bVoo4jd7drgU6Z57YLfdSmXTKs8h2SaRtnWFvCQRWZng60TKrpUr4ccf4eWX4cknSy1JgC6JleyTSIviDTMbD4yOpk8DXkteSCJJMmFCKOJ36aVw3HHw5ZdQrVpSdqVLYiWbJNKZPRgYBjSPHsPdXWNRSOb46afQOX300aGQ3/r1YX6SkoRItok3HkUj4E5gP2AWcJW7L01VYCKl4uWX4fzzYfnycAPdLbckXMRvW6hfQrJZvBbFI8ArQA9CBdl/pSQikdKyZAn06AG1a4d6TXfcATvsUKq7yKvldN55+VU+1C8h2SZeH0VNdx8RPZ9vZp+mIiCR7eIOH34Ihx+eX8Tv8MO3qT5TcYoaV0JlwyVbxWtRVDOzVmZ2sJkdDFQvMF0sM+tsZvPNbIGZXR1nvR5m5maWs61vQGSL3Fzo1i3cPJf3Cd6hQ6kkidgqsLGth/btYdiw0HGtJCHZKl6L4hvgrpjp5THTDhwdb8NmVhF4ADgWyAWmmNk4d59bYL2awCDg420LXSSyeTOMGAGDB8PGjXDXXXDkkaW6Cw06JOVZvIGLOm7nttsAC9x9IYCZjQG6A3MLrPd/wO3A4O3cn5RXPXrAiy+Gq5pGjIA/JmfwRV3yKuVVMm+cqwssiZnOjeZtEZ3C2tvdX423ITMbaGZTzWzqihUrSj9SyTwbN+YX8evRIySIt99OWpIQKc/Sdod1VK78LsJgSHG5+3B3z3H3nN1K8Q5ayVAzZ4bBhEZE11qccQacey6YxX+diJRIMhPFUmDvmOl60bw8NYEDgYlm9j/gUGCcOrSlSOvXw003QevWsGhRqZbdEJGiJVI91qKxsm+MpuubWZsEtj0FaGRmDcysCtAbGJe30N1/cvc67r6vu+8LfAR0c/epJXonkt2mTAlVXocMgdNPh3nz4E9/SndUIuVCIi2KfwOHAadH06sJVzPF5e4bgYuB8cA84Bl3n2NmQ8ysWwnjlfJq1SpYswZeew0eeyzcRCciKZFIUcC27n6wmU0HcPdVUQuhWO7+GgUKCLr7jUWs2yGRbUo58s47oYjfoEGhiN8XXySl/IaIxJdIi2JDdE+Ew5bxKDYnNSop3378EQYMgE6dwt1seUX8lCRE0iKRRHEf8ALwBzP7G/AecGtSo5Ly66WXoGlTeOQR+MtfwgBDShAiaVXsqSd3f9LMpgGdAANOdvd5SY9Myp/Fi+HUU6FJExg3DnLSewGcKsKKBIlc9VQfWAu8TLhq6Zdonsj2c4d33w3P69cPN81NmZL2JAEaqU4kTyKd2a8S+icMqAY0AOYDzZIYl5QHixeHsSJefz3UxmjfHtq1K9VdxLYKtlVeK0JlO6S8S+TU00Gx01HZjQuTFpFkv82b4aGH4K9/DS2K++4r9SJ+eQkitsrrtlIrQiRIpEWxFXf/1MzaJiMYKSf+9KfQaTHF0psAABX/SURBVH3sseETfd99S2WzGidCJDmKTRRmdkXMZAXgYGBZ0iKS7LRxI1SoEB6nnQbdu0O/fqVan0mlwEWSI5EWRc2Y5xsJfRbPJSccyUqffQbnnBPujTj//FCCI0nUpyBS+uImiuhGu5ruflWK4pFssm4dDB0Kt98Ou+4Ke+xR6rvQJawiyVdkojCzSu6+0cyOSGVAkiU++QTOOgs+/zz8vOuukCxKQVF9Eep8FkmOeC2KTwj9ETPMbBzwLPBL3kJ3fz7JsUkm+/ln+PVXeOMNOP74Em2iqEtbY5OD+iJEki+RPopqwErCGNl591M4oEQhW3vzTZgzBy6/HI45BubP367yG7Gd07GUHERSK16i+EN0xdNs8hNEHk9qVJJZVq2CK66AUaOgWTO48MKQIEqhRpM6p0XSL16iqAjsyNYJIo8ShQTPPw8XXQQrVsA118CNNyacIIq7a1qd0yJlQ7xE8Y27D0lZJJJ5Fi+G3r3hwAPDgEKtWm3Ty4s6tZRHndMiZUO8RKGR6uX33GHy5NBRUL9+GFyobVuoXLlEm9OpJZGyL1712E4pi0Iyw6JFcMIJ0KFD/qVHRx5Z4iQhIpmhyETh7j+kMhApwzZvhvvvDx3V770H//oXHHVUuqMSkRTZ5qKAUg6dfDK8/HK4H2LYMNhnnxJvSndSi2QeJQop3IYNULFiKOJ3+unQsyf07ZtwEb9EbpZTZ7VIZlCikN/79FPo3z8U8bvwwhIV8dPNciLZQ4lC8v36KwwZAnfcAbvtBnvvvU0vL+y0kq5oEsl8ShQSfPRRKN73xRehJPidd8IuuxT7MhXoE8l+ShQS/PJL6Jd4661QpymOopKDTiuJZCclivLsjTdCEb8rr4ROnUJJ8CpVin2ZRpITKV+UKMqjlStDEb/HHoODDoJLLgkJIoEkkUf9DyLlR7w7syXbuMPYsdC0aWgWXH89TJmyTQlCRMoftSjKk8WLw3mi5s3D2BEtWqQ7IhHJAEoU2c4dJkyAo48Od1RPnAht2kClSsWW+S6K7qgWKV906imbff01HHdc6KiOLk8aPvtwOhxTiQ4d4Lzz8q9a2ha69FWkfFGLIhtt2sQHfe6n1dhr2WwVGdboQV6+8SjcdDmriGw7JYps1L07h7/6KuMrdeHh1g+xolr+HdZKDiKyrZQoMlBhfQsVN29gs1XErQIdv+tLbvXT+fKQPkycpPGnRGT7JLWPwsw6m9l8M1tgZlcXsvwKM5trZjPN7L9mVvL61eVI3g1vefZfPZVhn+bQfdmDAEz4w2l82ebP9PmzkoSIbL+ktSjMrCLwAHAskAtMMbNx7j43ZrXpQI67rzWzC4B/AKclK6Zs0rIlTHz9V7j55lCXaffduezufbisa7ojE5Fsk8wWRRtggbsvdPffgDFA99gV3H2Cu6+NJj8C6iUxnqzS9KcPw30Q//hHKOI3dy50VZYQkdKXzD6KusCSmOlcoG2c9fsDrxe2wMwGAgMB6tevX1rxZYTC+iNmzIBW+/wahih9++1w+auISJKUic5sMzsDyAHaF7bc3YcDwwFycnI8haGlXWwBvrYrX2PftXOg5WCa9Dkazp4HlSunO0QRyXLJTBRLgdiRb+pF87ZiZscA1wHt3X19EuPJGAUHAGrf7HteqncZTHoSWrTggjcHRfWZlCREJPmS2UcxBWhkZg3MrArQGxgXu4KZtQKGAd3c/bskxpJRtlzV5M4Ve41hzMwm8MwzcNNN8MknKuInIimVtBaFu280s4uB8UBF4BF3n2NmQ4Cp7j4OuAPYEXjWzAAWu3u3ZMVU1hRVa2nLMKKPLobGZ4VO65EjQ0lwEZEUS2ofhbu/BrxWYN6NMc/jD6WW5WL7H7Zwp/8+/2X/PseEIn6TJsEhh0DFimmLU0TKtzLRmV2eFOx/2GoAoK++ggEDYOYE2H8i0B4OPTQ9gYqIRJQoUqCoMaa3VGHdtAnuvTcMJFS5MgwbBkcdlbZ4RURiKVGkQLFjTHc5CV5/Pdww9+CDUE/3HYpI2aFEUYqK7ZyeGDPzt9+gUiWoUAH69YO+faF3bzDVZxKRskUDF5WigsX68vxuoJ9PPoHWreHf/w7TvXrB6acrSYhImaQWRSn7Xcsh1tq1cMMNcM89sOeesN9+qQxNRKRElChS5b334KyzYOHCMAbp7bdDrVrpjkpEpFg69VQKhg+HDh0KP+20xYYN4V6ICRPgoYeUJEQkY6hFUQpir2raqi/i5Zdh3jz4y1+gY8dQCrySDrmIZBa1KEpJXt/EwIHAihUhY3TrBqNHhyucQElCRDKSEkVpcg/NiyZNYOxYGDIEPv5YRfxEJKPpK+42KO4+CRYvhrPPhlatQhG/Zs1SHqOISGlToihGUeU38phvZkD9t2jU5/hQxO/dd8M9EiriJyJZQomiEEUlh9+V3/jyy1DEb9YkOGAS0A7atElHyCIiSaNEUYhiazNt3Ah33w033ghVq4bTTCriJyJZSokiErf8d0Fdu8L48dC9eyjDsddeqQpTJKNs2LCB3Nxc1q1bl+5Qyo1q1apRr149KlcuvaGSy3WiKLb8d6z160MJ8AoV4Nxz4Zxz4NRTVZ9JJI7c3Fxq1qzJvvvui+l/JencnZUrV5Kbm0uDBg1KbbvlLlEk3P8Q66OPoH9/OP98uOQS6NkzZfGKZLJ169YpSaSQmVG7dm1WrFhRqtstN4kiL0EknBwAfvklDCZ0771hjIhGjVIWr0i2UJJIrWQc73KTKPI6qItNDnnefTcU8fv6a7jwQrjtNthpp5TEKiJSlmT1ndl5xfryCvZtVWajOBs3hj6JSZPggQeUJEQy2IsvvoiZ8fnnn2+ZN3HiRLp27brVev369WPs2LFA6Ii/+uqradSoEQcffDCHHXYYr7/++nbHctttt9GwYUP2339/xo8fH3fdSy+9lB133HHL9KJFi+jUqRPNmzenQ4cO5Obmbnc8iciKFkVRd0wX20Fd0IsvhiJ+11wTivjNmaP6TCJZYPTo0Rx55JGMHj2aW265JaHX3HDDDXzzzTfMnj2bqlWr8u233zIp70OlhObOncuYMWOYM2cOy5Yt45hjjuGLL76gYiE36E6dOpVVq1ZtNe+qq67izDPP5KyzzuKdd97hmmuu4fHHH9+umBKRFZ+Csfc9xEr4NNO334ZO6mefhYMPhiuvDPWZlCRESs1llxVTir8EWrYM44DFs2bNGt577z0mTJjASSedlFCiWLt2LSNGjODrr7+matWqAOy+++706tVru+J96aWX6N27N1WrVqVBgwY0bNiQTz75hMMOO2yr9TZt2sTgwYN56qmneOGFF7bMnzt3LnfddRcAHTt25OSTT96ueBKVNZ+Ece97KIo7PPFE+Ateswb+9jcYPDicchKRrPDSSy/RuXNnGjduTO3atZk2bRqtW7eO+5oFCxZQv359dkrglPPll1/OhAkTfje/d+/eXH311VvNW7p0KYceeuiW6Xr16rF06dLfvfb++++nW7du7LnnnlvNb9GiBc8//zyDBg3ihRdeYPXq1axcuZLatWsXG+f2yJpEUSKLF4d7InJywt3VBxyQ7ohEslZx3/yTZfTo0QwaNAgIH96jR4+mdevWRV4dtK1XDd19993bHWOsZcuW8eyzzzKxkG++d955JxdffDGjRo2iXbt21K1bt9DTVqWt/CWKzZvDXdUnnBCK+L3/fqj2qiJ+Ilnnhx9+4J133mHWrFmYGZs2bcLMuOOOO6hdu/bv+gB++OEH6tSpQ8OGDVm8eDE///xzsa2KbWlR1K1blyVLlmyZzs3NpW7dulutM336dBYsWEDDhg2BcBqsYcOGLFiwgL322ovnn38eCKfUnnvuOXbeeefED0hJuXtGPVq3bu3u7sOGubdvHx61aoWfxZo/3/2oo9zBfeLEBF4gIttj7ty5ad3/sGHDfODAgVvNa9eunU+aNMnXrVvn++6775YY//e//3n9+vX9xx9/dHf3wYMHe79+/Xz9+vXu7v7dd9/5M888s13xzJ4925s3b+7r1q3zhQsXeoMGDXzjxo1xX1OjRo0tz1esWOGbNm1yd/drr73Wb7jhhkJfU9hxB6Z6CT93M/by2LwObEjgiqaNG+H226F5c5g1C/7zH2jXLiVxikj6jB49mlNOOWWreT169GD06NFUrVqVJ554grPPPpuWLVvSs2dPHn74YWpF49kPHTqU3XbbjaZNm3LggQfStWvXhPos4mnWrBm9evWiadOmdO7cmQceeGDLqaMuXbqwbNmyuK+fOHEi+++/P40bN+bbb7/luuuu2654EmUh0WSOnJwcnzp1Kh06hOmEOrCPPx7efBP+9KdwT8QeeyQxQhHJM2/ePJo0aZLuMMqdwo67mU1z95ySbC97+yjWrQtXL1WsGK6PHTgQevRId1QiIhkn4049zZ+ff6d1kd5/P5yPeuCBMN2jh5KEiEgJZVyi+PXX8LPQfok1a+DSS8MgQuvWgZq8ImmXaae3M10yjnfGnXqqXr2IfolJk0IRv8WL4eKL4dZbIaZGioikXrVq1bbcEKYqssnn0XgU1apVK9XtZlyiiGuHHULV1yOOSHckIkK48zg3N7fUx0eQouWNcFeaMu6qp5o1c3z16qlh4vnn4fPP4dprw/SmTbpxTkSkENtz1VNS+yjMrLOZzTezBWZ2dSHLq5rZ09Hyj81s34Q2vHx5GGWuRw944QX47bcwX0lCRKTUJS1RmFlF4AHgBKApcLqZNS2wWn9glbs3BO4Gbi9uu7U2rAyd1K+8EgYT+uCDUOlVRESSIpktijbAAndf6O6/AWOA7gXW6Q48Gj0fC3SyYnq8dl+/CA48ED77DK6+WpVeRUSSLJmd2XWBJTHTuUDbotZx941m9hNQG/g+diUzGwjkjSqx3t57b7YqvQJQhwLHqhzTscinY5FPxyLf/iV9YUZc9eTuw4HhAGY2taQdMtlGxyKfjkU+HYt8Ohb5zGxqSV+bzFNPS4G9Y6brRfMKXcfMKgG1gJVJjElERLZRMhPFFKCRmTUwsypAb2BcgXXGAWdFz3sC73imXa8rIpLlknbqKepzuBgYD1QEHnH3OWY2hFAXfRwwEnjczBYAPxCSSXGGJyvmDKRjkU/HIp+ORT4di3wlPhYZd8OdiIikVsYVBRQRkdRSohARkbjKbKJIWvmPDJTAsbjCzOaa2Uwz+6+Z7ZOOOFOhuGMRs14PM3Mzy9pLIxM5FmbWK/rbmGNmT6U6xlRJ4H+kvplNMLPp0f9Jl3TEmWxm9oiZfWdms4tYbmZ2X3ScZprZwQltuKSDbSfzQej8/gr4I1AF+AxoWmCdC4GHoue9gafTHXcaj0VHYIfo+QXl+VhE69UEJgMfATnpjjuNfxeNgOnALtH0H9IddxqPxXDgguh5U+B/6Y47SceiHXAwMLuI5V2A1wEDDgU+TmS7ZbVFkZTyHxmq2GPh7hPcfW00+RHhnpVslMjfBcD/EeqGrUtlcCmWyLEYADzg7qsA3P27FMeYKokcCwd2ip7XApalML6UcffJhCtIi9IdeMyDj4CdzWzP4rZbVhNFYeU/6ha1jrtvBPLKf2SbRI5FrP6EbwzZqNhjETWl93b3V1MZWBok8nfRGGhsZu+b2Udm1jll0aVWIsfiZuAMM8sFXgMuSU1oZc62fp4AGVLCQxJjZmcAOUD7dMeSDmZWAbgL6JfmUMqKSoTTTx0IrczJZnaQu/+Y1qjS43RglLv/08wOI9y/daC7b053YJmgrLYoVP4jXyLHAjM7BrgO6Obu61MUW6oVdyxqAgcCE83sf4RzsOOytEM7kb+LXGCcu29w96+BLwiJI9skciz6A88AuPuHQDVCwcDyJqHPk4LKaqJQ+Y98xR4LM2sFDCMkiWw9Dw3FHAt3/8nd67j7vu6+L6G/ppu7l7gYWhmWyP/Ii4TWBGZWh3AqamEqg0yRRI7FYqATgJk1ISSK8jg+6zjgzOjqp0OBn9z9m+JeVCZPPXnyyn9knASPxR3AjsCzUX/+YnfvlragkyTBY1EuJHgsxgPHmdlcYBMw2N2zrtWd4LG4EhhhZpcTOrb7ZeMXSzMbTfhyUCfqj7kJqAzg7g8R+me6AAuAtcDZCW03C4+ViIiUorJ66klERMoIJQoREYlLiUJEROJSohARkbiUKEREJC4lCimTzGyTmc2IeewbZ901pbC/UWb2dbSvT6O7d7d1Gw+bWdPo+bUFln2wvTFG28k7LrPN7GUz27mY9Vtma6VUSR1dHitlkpmtcfcdS3vdONsYBbzi7mPN7DjgTndvvh3b2+6YituumT0KfOHuf4uzfj9CBd2LSzsWKT/UopCMYGY7RmNtfGpms8zsd1VjzWxPM5sc8437qGj+cWb2YfTaZ82suA/wyUDD6LVXRNuabWaXRfNqmNmrZvZZNP+0aP5EM8sxs78D1aM4noyWrYl+jjGzE2NiHmVmPc2sopndYWZTonECzkvgsHxIVNDNzNpE73G6mX1gZvtHdykPAU6LYjktiv0RM/skWrew6rsiW0t3/XQ99CjsQbiTeEb0eIFQRWCnaFkdwp2leS3iNdHPK4HroucVCbWf6hA++GtE8/8K3FjI/kYBPaPnpwIfA62BWUANwp3vc4BWQA9gRMxra0U/JxKNf5EXU8w6eTGeAjwaPa9CqORZHRgIXB/NrwpMBRoUEueamPf3LNA5mt4JqBQ9PwZ4LnreD7g/5vW3AmdEz3cm1H+qke7ftx5l+1EmS3iIAL+6e8u8CTOrDNxqZu2AzYRv0rsDy2NeMwV4JFr3RXefYWbtCQPVvB+VN6lC+CZemDvM7HpCDaD+hNpAL7j7L1EMzwNHAW8A/zSz2wmnq97dhvf1OnCvmVUFOgOT3f3X6HRXczPrGa1Xi1DA7+sCr69uZjOi9z8PeCtm/UfNrBGhREXlIvZ/HNDNzK6KpqsB9aNtiRRKiUIyxZ+B3YDW7r7BQnXYarEruPvkKJGcCIwys7uAVcBb7n56AvsY7O5j8ybMrFNhK7n7FxbGvegCDDWz/7r7kETehLuvM7OJwPHAaYRBdiCMOHaJu48vZhO/untLM9uBUNvoIuA+wmBNE9z9lKjjf2IRrzegh7vPTyReEVAfhWSOWsB3UZLoCPxuXHALY4V/6+4jgIcJQ0J+BBxhZnl9DjXMrHGC+3wXONnMdjCzGoTTRu+a2V7AWnd/glCQsbBxhzdELZvCPE0oxpbXOoHwoX9B3mvMrHG0z0J5GNHwUuBKyy+zn1cuul/MqqsJp+DyjAcusah5ZaHysEhcShSSKZ4EcsxsFnAm8Hkh63QAPjOz6YRv6/e6+wrCB+doM5tJOO10QCI7dPdPCX0XnxD6LB529+nAQcAn0Smgm4Chhbx8ODAzrzO7gDcJg0u97WHoTgiJbS7wqZnNJpSNj9vij2KZSRiU5x/AbdF7j33dBKBpXmc2oeVROYptTjQtEpcujxURkbjUohARkbiUKEREJC4lChERiUuJQkRE4lKiEBGRuJQoREQkLiUKERGJ6/8BpkqKcmHKfaMAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light",
            "tags": []
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "# Compute predicted probabilities on the test set\n",
        "probs = bert_predict(bert_classifier, val_dataloader)\n",
        "\n",
        "# Evaluate the Bert classifier\n",
        "evaluate_roc(probs, y_val)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vxHGJj7BfhWv",
        "outputId": "cf1e94c2-8739-4508-8805-60781b2c3b7a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Tokenizing data...\n",
            "Number of tweets predicted as Rumor:  391\n",
            "\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "           0       0.22      0.18      0.20       116\n",
            "           1       0.76      0.80      0.78       369\n",
            "\n",
            "    accuracy                           0.65       485\n",
            "   macro avg       0.49      0.49      0.49       485\n",
            "weighted avg       0.63      0.65      0.64       485\n",
            "\n",
            "0.6536082474226804\n",
            "0.7789473684210527\n"
          ]
        }
      ],
      "source": [
        "PATH = './Model/BERTweet_raw_to_fine_tune_ord4.pt'\n",
        "bert_classifier.load_state_dict(torch.load(PATH))\n",
        "testing_process(bert_classifier, X_val, y_val)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yu1vQFK_fhWv",
        "outputId": "05e4aaa2-74fa-4254-a47b-6218057d8e92"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Tokenizing data...\n",
            "Number of tweets predicted as Rumor:  3823\n",
            "\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "           0       0.11      0.33      0.17       475\n",
            "           1       0.92      0.74      0.82      4752\n",
            "\n",
            "    accuracy                           0.70      5227\n",
            "   macro avg       0.52      0.54      0.49      5227\n",
            "weighted avg       0.84      0.70      0.76      5227\n",
            "\n",
            "0.7013583317390473\n",
            "0.8179591836734694\n"
          ]
        }
      ],
      "source": [
        "rhi_data = pd.read_csv('data/_RHI_text.csv')\n",
        "rhi_y = pd.read_csv('data/_RHI_text.csv')\n",
        "X_test = rhi_data.text.values\n",
        "y_test = rhi_y.isRumor.values\n",
        "\n",
        "PATH = './Model/BERTweet_raw_to_fine_tune_ord4.pt'\n",
        "bert_classifier.load_state_dict(torch.load(PATH))\n",
        "testing_process(bert_classifier, X_test, y_test)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F08yfWtRAzty"
      },
      "source": [
        "## Embedding Only"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(\"vinai/bertweet-base\", use_fast=False)\n",
        "bert_classifier, optimizer, scheduler, loss_fn = initialize_model(epochs=1)\n",
        "PATH = './Model/BERTweet_raw_to_fine_tune_ord4.pt'\n",
        "bert_classifier.load_state_dict(torch.load(PATH))\n",
        "# testing_process(bert_classifier, X_val, y_val)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "def preprocessing_for_bert(data):\n",
        "    \"\"\"Perform required preprocessing steps for pretrained BERT.\n",
        "    @param    data (np.array): Array of texts to be processed.\n",
        "    @return   input_ids (torch.Tensor): Tensor of token ids to be fed to a model.\n",
        "    @return   attention_masks (torch.Tensor): Tensor of indices specifying which\n",
        "                  tokens should be attended to by the model.\n",
        "    \"\"\"\n",
        "    # Create empty lists to store outputs (빈 리스트 2개 생성)\n",
        "    input_ids = []\n",
        "    attention_masks = []\n",
        "\n",
        "    # For every sentence...\n",
        "    for sent in data:\n",
        "        # `encode_plus` will:\n",
        "        #    (1) Tokenize the sentence\n",
        "        #    (2) Add the `[CLS]` and `[SEP]` token to the start and end\n",
        "        #    (3) Truncate/Pad sentence to max length\n",
        "        #    (4) Map tokens to their IDs\n",
        "        #    (5) Create attention mask\n",
        "        #    (6) Return a dictionary of outputs\n",
        "        encoded_sent = tokenizer.encode_plus(\n",
        "            text=text_preprocessing(sent),  # Preprocess sentence\n",
        "            add_special_tokens=True,        # Add `[CLS]` and `[SEP]`\n",
        "            max_length=MAX_LEN,                  # Max length to truncate/pad\n",
        "            pad_to_max_length=True,         # Pad sentence to max length\n",
        "            # return_tensors='pt',           # Return PyTorch tensor\n",
        "            return_attention_mask=True      # Return attention mask\n",
        "        )\n",
        "\n",
        "        # Add the outputs to the lists (위의 빈 리스트에 상응하는 값 추가)\n",
        "        input_ids.append(encoded_sent.get('input_ids'))\n",
        "        attention_masks.append(encoded_sent.get('attention_mask'))\n",
        "\n",
        "    # Convert lists to tensors (리스트들을 텐서화)\n",
        "    input_ids = torch.tensor(input_ids)\n",
        "    attention_masks = torch.tensor(attention_masks)\n",
        "\n",
        "    return input_ids, attention_masks"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {},
      "outputs": [
        {
          "ename": "NameError",
          "evalue": "name 'raw_text' is not defined",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-4-a3f8a7a2ed4c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mraw_text\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0minput_ids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattention_masks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpreprocessing_for_bert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m# text.dropna(inplace=True)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m# text = text.sample(frac=0.4, random_state=999)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'raw_text' is not defined"
          ]
        }
      ],
      "source": [
        "text = raw_text.text\n",
        "input_ids, attention_masks = preprocessing_for_bert(text)\n",
        "# text.dropna(inplace=True)\n",
        "# text = text.sample(frac=0.4, random_state=999)\n",
        "\n",
        "# tokenizer = AutoTokenizer.from_pretrained(\"vinai/bertweet-base\", use_fast=False)\n",
        "# bertweet = AutoModel.from_pretrained(\"vinai/bertweet-base\")\n",
        "\n",
        "# sents = [tokenizer.tokenize(tweet) for tweet in text]\n",
        "len(input_ids)\n",
        "# embeddings = []\n",
        "# for sent in input_ids:\n",
        "#     with torch.no_grad():\n",
        "#         features = bert_classifier.embedding(input_ids)\n",
        "#     embeddings.append(features)\n",
        "\n",
        "# result = [torch.mean(features.last_hidden_state[-1], dim=0).tolist() for features in embeddings]\n",
        "\n",
        "last_hidden_states = bert_classifier.embedding(input_ids)\n",
        "features = last_hidden_states[0][:,0,:].numpy() # considering o only the [CLS] for each sentences \n",
        "features.shape\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "bertweetEmbed = pd.DataFrame(features)\n",
        "print(bertweetEmbed.shape)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "model.eval()\n",
        "\n",
        "all_logits = []\n",
        "\n",
        "# For each batch in our test set...\n",
        "for batch in test_dataloader:\n",
        "    # Load batch to GPU\n",
        "    b_input_ids, b_attn_mask = tuple(t.to(device) for t in batch)[:2]\n",
        "\n",
        "    # Compute logits\n",
        "    with torch.no_grad():\n",
        "        logits = model(b_input_ids, b_attn_mask)\n",
        "    all_logits.append(logits)\n",
        "\n",
        "# Concatenate logits from each batch\n",
        "all_logits = torch.cat(all_logits, dim=0)\n",
        "\n",
        "# Apply softmax to calculate probabilities\n",
        "probs = F.softmax(all_logits, dim=1).cpu().numpy()\n",
        "\n",
        "return probs\n"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "vtIMlTiMUFzS",
        "fJjwums-kI2t",
        "lqDU7M-oP54i",
        "_HtUKy5HP8X8"
      ],
      "include_colab_link": true,
      "name": "_CLF_BERT.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    },
    "orig_nbformat": 2,
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "0660f1c2cada41d3bdfe2589d7184b56": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": "initial"
          }
        },
        "0c3cc67272e24e0c807af12b592c3a55": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1bd1e9fe81934cf3885154196e620c03": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f954f9c5e5954c2bb96b5993a9395109",
              "IPY_MODEL_ce478aa49888428c8b784e7389e191f4"
            ],
            "layout": "IPY_MODEL_bd458a99a85e4a13afd1cf36b0a57ef7"
          }
        },
        "930e5993b2fb4a07943e2d0006d54dba": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bd458a99a85e4a13afd1cf36b0a57ef7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ce478aa49888428c8b784e7389e191f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_930e5993b2fb4a07943e2d0006d54dba",
            "placeholder": "​",
            "style": "IPY_MODEL_f228d1ac81f6413a997d164c52f2ce02",
            "value": " 232k/232k [00:00&lt;00:00, 301kB/s]"
          }
        },
        "f228d1ac81f6413a997d164c52f2ce02": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f954f9c5e5954c2bb96b5993a9395109": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "Downloading: 100%",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0c3cc67272e24e0c807af12b592c3a55",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0660f1c2cada41d3bdfe2589d7184b56",
            "value": 231508
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}