train_cleaned_word_level.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "########################################\n",
    "## import packages\n",
    "########################################\n",
    "import os\n",
    "import re\n",
    "import csv\n",
    "import codecs\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import operator\n",
    "import sys\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import SnowballStemmer\n",
    "from string import punctuation\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "import sotoxic.models.keras.model_zoo as model_zoo\n",
    "from sotoxic.train import trainer\n",
    "from sotoxic.data_helper.data_loader import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path = 'Dataset/'\n",
    "EMBEDDING_FILE='features/crawl-300d-2M.vec'\n",
    "#EMBEDDING_FILE='features/glove.840B.300d.txt'\n",
    "#EMBEDDING_FILE='features/glove.twitter.27B.200d.txt'\n",
    "TRAIN_DATA_FILE=path + 'train.csv'\n",
    "TEST_DATA_FILE=path + 'test.csv'\n",
    "\n",
    "MAX_SEQUENCE_LENGTH = 400\n",
    "MAX_NB_WORDS = 100000\n",
    "EMBEDDING_DIM = 300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(TRAIN_DATA_FILE)\n",
    "test_df = pd.read_csv(TEST_DATA_FILE)\n",
    "data_loader = DataLoader()\n",
    "embeddings_index = data_loader.load_embedding(EMBEDDING_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "########################################\n",
    "## process texts in datasets\n",
    "########################################\n",
    "print('Processing text dataset')\n",
    "list_sentences_train = train_df[\"comment_text\"].fillna(\"no comment\").values\n",
    "list_classes = [\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]\n",
    "y = train_df[list_classes].values\n",
    "list_sentences_test = test_df[\"comment_text\"].fillna(\"no comment\").values\n",
    "\n",
    "comments = []\n",
    "for text in list_sentences_train:\n",
    "    comments.append(text)\n",
    "    \n",
    "test_comments=[]\n",
    "for text in list_sentences_test:\n",
    "    test_comments.append(text)\n",
    "\n",
    "#tokenizer = Tokenizer(num_words=MAX_NB_WORDS, filters='\"#%&()+,-./:;<=>@[\\\\]^_`{|}~\\t\\n')\n",
    "tokenizer = Tokenizer(num_words=MAX_NB_WORDS)\n",
    "\n",
    "tokenizer.fit_on_texts(comments + test_comments)\n",
    "\n",
    "sequences = tokenizer.texts_to_sequences(comments)\n",
    "test_sequences = tokenizer.texts_to_sequences(test_comments)\n",
    "\n",
    "word_index = tokenizer.word_index\n",
    "print('Found %s unique tokens' % len(word_index))\n",
    "\n",
    "data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "print('Shape of data tensor:', data.shape)\n",
    "print('Shape of label tensor:', y.shape)\n",
    "\n",
    "test_data = pad_sequences(test_sequences, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "print('Shape of test_data tensor:', test_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "########################################\n",
    "## prepare embeddings\n",
    "########################################\n",
    "print('Preparing embedding matrix')\n",
    "nb_words = min(MAX_NB_WORDS, len(word_index))\n",
    "#embedding_matrix = np.random.normal(loc=matrix_mean, scale=matrix_std, size=(nb_words, EMBEDDING_DIM))\n",
    "embedding_matrix = np.zeros((nb_words, EMBEDDING_DIM))\n",
    "null_count = 0\n",
    "for word, i in word_index.items():\n",
    "    if i >= MAX_NB_WORDS:\n",
    "        continue\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        embedding_matrix[i] = embedding_vector\n",
    "    else:\n",
    "        null_count += 1\n",
    "print('Null word embeddings: %d' % null_count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_model():\n",
    "    return model_zoo.get_av_rnn(nb_words, EMBEDDING_DIM, embedding_matrix, MAX_SEQUENCE_LENGTH, out_size=6)\n",
    "\n",
    "keras_model_trainer = trainer.KerasModelTrainer(model_stamp='kmax_text_cnn', epoch_num=50, learning_rate=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "models, val_loss, total_auc, fold_predictions = keras_model_trainer.train_folds(data, y, fold_count=10, batch_size=256, get_model_func=get_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(\"Overall val-loss:\", val_loss, \"AUC\", total_auc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_fold_preditcions = np.concatenate(fold_predictions, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "training_auc = roc_auc_score(y[:-1], train_fold_preditcions)\n",
    "print(\"Training AUC\", training_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#test_data = test_df\n",
    "CLASSES = [\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]\n",
    "submit_path_prefix = \"results/rnn/nds/fasttext-SC2-nds-randomNoisy-capNet-\" + str(MAX_NB_WORDS) + \"-RST-lp-ct-\" + str(MAX_SEQUENCE_LENGTH) \n",
    "\n",
    "print(\"Predicting testing results...\")\n",
    "test_predicts_list = []\n",
    "for fold_id, model in enumerate(models):\n",
    "    test_predicts = model.predict(test_data, batch_size=256, verbose=1)\n",
    "    test_predicts_list.append(test_predicts)\n",
    "    np.save(\"predict_path/\", test_predicts)\n",
    "\n",
    "test_predicts = np.zeros(test_predicts_list[0].shape)\n",
    "for fold_predict in test_predicts_list:\n",
    "    test_predicts += fold_predict\n",
    "test_predicts /= len(test_predicts_list)\n",
    "\n",
    "test_ids = test_df[\"id\"].values\n",
    "test_ids = test_ids.reshape((len(test_ids), 1))\n",
    "\n",
    "test_predicts = pd.DataFrame(data=test_predicts, columns=CLASSES)\n",
    "test_predicts[\"id\"] = test_ids\n",
    "test_predicts = test_predicts[[\"id\"] + CLASSES]\n",
    "submit_path = submit_path_prefix + \"-L{:4f}-A{:4f}.csv\".format(val_loss, total_auc)\n",
    "test_predicts.to_csv(submit_path, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(\"Predicting training results...\")\n",
    "\n",
    "train_ids = train_df[\"id\"].values\n",
    "train_ids = train_ids.reshape((len(train_ids), 1))\n",
    "\n",
    "train_predicts = pd.DataFrame(data=train_fold_preditcions, columns=CLASSES)\n",
    "train_predicts[\"id\"] = train_ids\n",
    "train_predicts = train_predicts[[\"id\"] + CLASSES]\n",
    "submit_path = submit_path_prefix + \"-Train-L{:4f}-A{:4f}.csv\".format(val_loss, training_auc)\n",
    "train_predicts.to_csv(submit_path, index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}