pre_processing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pre Processing function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     /Users/hanxunhuang/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "from nltk.stem import WordNetLemmatizer \n",
    "import nltk\n",
    "nltk.download('wordnet')\n",
    "\n",
    "\n",
    "remove_words = [\"@handle\", \"RT\", \"http\"]\n",
    "lemmatizer = WordNetLemmatizer() \n",
    "\n",
    "def pre_process(sentence, max_length):\n",
    "    sentence = sentence.split()\n",
    "    target_remove = set()\n",
    "    for i, token in enumerate(sentence):\n",
    "        for target in remove_words:\n",
    "            if (target == \"http\") and (target in token.lower()):\n",
    "                target_remove.add(token)\n",
    "                break\n",
    "            if target in token:\n",
    "                target_remove.add(token)\n",
    "                break\n",
    "    for target in target_remove:\n",
    "        while target in sentence:\n",
    "            sentence.remove(target)\n",
    "            \n",
    "    for i, token in enumerate(sentence):\n",
    "        sentence[i] = lemmatizer.lemmatize(token)\n",
    "        \n",
    "    max_length = max(max_length, len(sentence))\n",
    "    sentence = ' '.join(sentence)\n",
    "    return sentence, max_length"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load Train data to train_dict. \n",
    "train_dict[id] = [[train_instace1], [train_instance2] ....]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total rows: 328931\n",
      "Total ids: 9295\n",
      "Longest Sentence: 37\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "import collections\n",
    "import numpy as np\n",
    "\n",
    "train_file_path = \"data/train_tweets.txt\"\n",
    "train_dict = collections.defaultdict(list)\n",
    "max_length = 0\n",
    "\n",
    "length_array = []\n",
    "with open(train_file_path, encoding='utf-8') as tsvfile:\n",
    "    reader = tsvfile.readlines()\n",
    "    for i, row in enumerate(reader):\n",
    "        row = row.strip().split(\"\\t\")\n",
    "        id = int(row[0])\n",
    "        instance, max_length = pre_process(row[1], max_length)\n",
    "        if not instance == \"\":\n",
    "            train_dict[id].append(instance)\n",
    "            length_array.append(len(instance.split()))\n",
    "            if len(instance) == 0:\n",
    "                print(\"Error\")\n",
    "    print(\"Total rows: %d\" % i)\n",
    "    \n",
    "print(\"Total ids: %d\" % len(train_dict))\n",
    "print(\"Longest Sentence: %d\" % (max_length))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(train_dict[1319])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split Dev Train\n",
    "!! There are Some ID have 0 train instance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9295 9295 9295\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "\n",
    "dev_split = 0.1\n",
    "train_split = 1 - dev_split\n",
    "\n",
    "dev_set_dict = {}\n",
    "train_set_dict = {}\n",
    "\n",
    "for id in train_dict:\n",
    "    target_list = train_dict[id]\n",
    "    length = len(target_list)\n",
    "    random.shuffle(target_list)\n",
    "    split = int(np.ceil(length*dev_split))\n",
    "    dev_set_dict[id] = target_list[:split]\n",
    "    train_set_dict[id] = target_list[split:length]\n",
    "    \n",
    "print(len(dev_set_dict), len(train_set_dict), len(train_dict))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Build ID idx and train/dev set save to file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36116\n",
      "291758\n",
      "9295\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "\n",
    "def save_to_file(target_dict, file_path):\n",
    "    id_list = []\n",
    "    sentence_list = []\n",
    "    for id in target_dict:\n",
    "        for sentence in target_dict[id]:\n",
    "            id_list.append(id)\n",
    "            sentence_list.append(sentence)\n",
    "      \n",
    "    id_list = np.array(id_list)\n",
    "    sentence_list = np.array(sentence_list)\n",
    "    random_index = np.array(range(len(sentence_list)))\n",
    "    random.shuffle(random_index)\n",
    "    id_list = id_list[random_index]\n",
    "    sentence_list = sentence_list[random_index]\n",
    "    \n",
    "    dataframe = pd.DataFrame({'id':id_list,'sentence':sentence_list})\n",
    "    dataframe.to_csv(file_path,index=False,sep='\\t',header=None)\n",
    "    print(len(id_list))\n",
    "    return\n",
    "\n",
    "dev_set_path = 'data/v4/dev_set.txt'\n",
    "train_set_path = 'data/v4/train_set.txt'\n",
    "idx_file_path = 'data/v4/idx.pickle'\n",
    "\n",
    "save_to_file(dev_set_dict, dev_set_path)\n",
    "save_to_file(train_set_dict, train_set_path)\n",
    "\n",
    "\n",
    "# Build IDX and Save\n",
    "idx = {}\n",
    "for i, id in enumerate(train_dict):\n",
    "    idx[id] = i\n",
    "print(len(idx))\n",
    "\n",
    "with open(idx_file_path, 'wb') as handle:\n",
    "    pickle.dump(idx, handle, protocol=pickle.HIGHEST_PROTOCOL)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from feature_extract_inference import FeatureExtract\n",
    "test_model = FeatureExtract(checkpoints_path=\"/Users/hanxunhuang/Desktop/checkpoints/coconut_extract_model_v2_best.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy import dot\n",
    "from numpy.linalg import norm\n",
    "\n",
    "feature = test_model.get_features([\"Shannon was right, Hamilton should step down #fb\", \"Listen live on as I debate @handle Friday Oct 2nd at 10:00 am #fb\"])\n",
    "feature1 = feature[0]\n",
    "feature2 = feature[1]\n",
    "\n",
    "cos_sim = dot(feature1, feature2)/(norm(feature1)*norm(feature2))\n",
    "print(cos_sim)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}