CS2 SMS spam.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs=pd.read_table('SMSSpamCollection+(1)', header=None, names=['Class','sms'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class</th>\n",
       "      <th>sms</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Class                                                sms\n",
       "0   ham  Go until jurong point, crazy.. Available only ...\n",
       "1   ham                      Ok lar... Joking wif u oni...\n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...\n",
       "3   ham  U dun say so early hor... U c already then say...\n",
       "4   ham  Nah I don't think he goes to usf, he lives aro..."
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "docs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "ham_spam=docs.Class.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ham     4825\n",
       "spam     747\n",
       "Name: Class, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ham_spam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "% of spam 13.406317300789663\n"
     ]
    }
   ],
   "source": [
    "print('% of spam',(ham_spam[1]/float(ham_spam[0]+ham_spam[1])*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs['label']=docs.Class.map({'ham':0, 'spam':1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class</th>\n",
       "      <th>sms</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Class                                                sms  label\n",
       "0   ham  Go until jurong point, crazy.. Available only ...      0\n",
       "1   ham                      Ok lar... Joking wif u oni...      0\n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...      1\n",
       "3   ham  U dun say so early hor... U c already then say...      0\n",
       "4   ham  Nah I don't think he goes to usf, he lives aro...      0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "docs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "x=docs.sms\n",
    "y=docs.label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5572,)\n",
      "(5572,)\n"
     ]
    }
   ],
   "source": [
    "print(x.shape)\n",
    "print(y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train, x_test, y_train, y_test= train_test_split(x,y, random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "710     4mths half price Orange line rental & latest c...\n",
       "3740                           Did you stitch his trouser\n",
       "2711    Hope you enjoyed your new content. text stop t...\n",
       "3155    Not heard from U4 a while. Call 4 rude chat pr...\n",
       "3748    Ü neva tell me how i noe... I'm not at home in...\n",
       "Name: sms, dtype: object"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "vec=CountVectorizer(stop_words='english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountVectorizer(stop_words='english')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vec.fit(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'4mths': 509,\n",
       " 'half': 3089,\n",
       " 'price': 5027,\n",
       " 'orange': 4626,\n",
       " 'line': 3852,\n",
       " 'rental': 5310,\n",
       " 'latest': 3763,\n",
       " 'camera': 1527,\n",
       " 'phones': 4822,\n",
       " 'free': 2780,\n",
       " 'phone': 4818,\n",
       " '11mths': 264,\n",
       " 'mobilesdirect': 4248,\n",
       " '08000938767': 50,\n",
       " 'update': 6673,\n",
       " 'or2stoptxt': 4624,\n",
       " 'cs': 1971,\n",
       " 'did': 2169,\n",
       " 'stitch': 6028,\n",
       " 'trouser': 6545,\n",
       " 'hope': 3260,\n",
       " 'enjoyed': 2436,\n",
       " 'new': 4442,\n",
       " 'content': 1867,\n",
       " 'text': 6321,\n",
       " 'stop': 6038,\n",
       " '61610': 563,\n",
       " 'unsubscribe': 6665,\n",
       " 'help': 3180,\n",
       " '08712400602450p': 98,\n",
       " 'provided': 5089,\n",
       " 'tones2you': 6473,\n",
       " 'uk': 6614,\n",
       " 'heard': 3159,\n",
       " 'u4': 6608,\n",
       " 'rude': 5443,\n",
       " 'chat': 1643,\n",
       " 'private': 5040,\n",
       " '01223585334': 5,\n",
       " 'cum': 1989,\n",
       " 'wan': 6852,\n",
       " '2c': 374,\n",
       " 'pics': 4837,\n",
       " 'gettin': 2913,\n",
       " 'shagged': 5628,\n",
       " 'pix': 4858,\n",
       " '8552': 660,\n",
       " '2end': 378,\n",
       " 'send': 5590,\n",
       " 'sam': 5487,\n",
       " 'xxx': 7127,\n",
       " 'neva': 4438,\n",
       " 'tell': 6289,\n",
       " 'noe': 4477,\n",
       " 'home': 3244,\n",
       " 'da': 2015,\n",
       " 'aft': 826,\n",
       " 'wat': 6876,\n",
       " 'wiskey': 7012,\n",
       " 'brandy': 1389,\n",
       " 'rum': 5450,\n",
       " 'gin': 2927,\n",
       " 'beer': 1209,\n",
       " 'vodka': 6798,\n",
       " 'scotch': 5535,\n",
       " 'shampain': 5637,\n",
       " 'wine': 6993,\n",
       " 'kudi': 3715,\n",
       " 'yarasu': 7142,\n",
       " 'dhina': 2156,\n",
       " 'vaazhthukkal': 6720,\n",
       " 'seeking': 5573,\n",
       " 'lady': 3732,\n",
       " 'street': 6056,\n",
       " 'freak': 2776,\n",
       " 'sheets': 5649,\n",
       " 'lol': 3910,\n",
       " 'drunkard': 2327,\n",
       " 'just': 3623,\n",
       " 'doing': 2259,\n",
       " 'hair': 3085,\n",
       " 'moment': 4265,\n",
       " 'yeah': 7149,\n",
       " 'tonight': 6475,\n",
       " 'wats': 6887,\n",
       " 'plan': 4867,\n",
       " 'turning': 6579,\n",
       " 'moms': 4267,\n",
       " 'telling': 6290,\n",
       " 'cancer': 1535,\n",
       " 'sister': 5745,\n",
       " 'won': 7041,\n",
       " 'calling': 1519,\n",
       " 'hurts': 3323,\n",
       " 'talk': 6234,\n",
       " 'love': 3953,\n",
       " 'coming': 1800,\n",
       " 'dinner': 2193,\n",
       " 'rite': 5396,\n",
       " 'dad': 2018,\n",
       " 'ask': 1032,\n",
       " 'confirm': 1843,\n",
       " 'wif': 6977,\n",
       " 'ah': 842,\n",
       " 'poor': 4931,\n",
       " 'baby': 1121,\n",
       " 'urfeeling': 6687,\n",
       " 'bettersn': 1243,\n",
       " 'luv': 3988,\n",
       " 'probthat': 5052,\n",
       " 'overdose': 4667,\n",
       " 'work': 7058,\n",
       " 'hey': 3198,\n",
       " 'careful': 1556,\n",
       " 'spk': 5938,\n",
       " 'sn': 5822,\n",
       " 'lots': 3942,\n",
       " 'lovejen': 3955,\n",
       " 'gam': 2856,\n",
       " 'gone': 2968,\n",
       " 'outstanding': 4664,\n",
       " 'innings': 3438,\n",
       " 'nice': 4453,\n",
       " 'working': 7062,\n",
       " 'haha': 3081,\n",
       " 'kidding': 3672,\n",
       " 'papa': 4705,\n",
       " 'needs': 4417,\n",
       " 'drugs': 2325,\n",
       " 'chief': 1677,\n",
       " 'bell': 1223,\n",
       " 'need': 4412,\n",
       " 'royal': 5432,\n",
       " 'visit': 6789,\n",
       " '1st': 324,\n",
       " 'june': 3618,\n",
       " 'ugh': 6610,\n",
       " 'long': 3917,\n",
       " 'day': 2054,\n",
       " 'exhausted': 2523,\n",
       " 'want': 6857,\n",
       " 'cuddle': 1984,\n",
       " 'nap': 4381,\n",
       " 'awesome': 1103,\n",
       " 'time': 6418,\n",
       " 'like': 3840,\n",
       " 'lt': 3974,\n",
       " 'gt': 3044,\n",
       " 'll': 3887,\n",
       " 'details': 2143,\n",
       " 'wee': 6916,\n",
       " 'bit': 1277,\n",
       " 'ok': 4578,\n",
       " 'lor': 3932,\n",
       " 'thk': 6376,\n",
       " 'tickets': 6407,\n",
       " 'cos': 1901,\n",
       " 'quite': 5149,\n",
       " 'late': 3760,\n",
       " 'look': 3921,\n",
       " 'ur': 6684,\n",
       " 'frens': 2791,\n",
       " 'darren': 2041,\n",
       " 'dont': 2269,\n",
       " 'know': 3700,\n",
       " 'bring': 1414,\n",
       " 'food': 2730,\n",
       " 'reason': 5224,\n",
       " 've': 6744,\n",
       " 'spoken': 5948,\n",
       " 'year': 7150,\n",
       " 'anyways': 947,\n",
       " 'great': 3020,\n",
       " 'week': 6918,\n",
       " 'best': 1237,\n",
       " 'exam': 2511,\n",
       " 'make': 4042,\n",
       " 'fucks': 2825,\n",
       " 'sake': 5476,\n",
       " 'yes': 7160,\n",
       " 'place': 4862,\n",
       " 'town': 6504,\n",
       " 'meet': 4130,\n",
       " 'exciting': 2515,\n",
       " 'adult': 806,\n",
       " 'singles': 5739,\n",
       " 'txt': 6590,\n",
       " '86688': 664,\n",
       " '150p': 293,\n",
       " 'msg': 4315,\n",
       " 'man': 4052,\n",
       " 'print': 5035,\n",
       " 'outs': 4660,\n",
       " 'room': 5418,\n",
       " 'think': 6368,\n",
       " 'saying': 5517,\n",
       " 'clearly': 1734,\n",
       " 'leave': 3793,\n",
       " 'come': 1794,\n",
       " 'st': 5972,\n",
       " 'andre': 912,\n",
       " 'virgil': 6783,\n",
       " 'cream': 1947,\n",
       " 'yoyyooo': 7179,\n",
       " 'change': 1623,\n",
       " 'permissions': 4789,\n",
       " 'drive': 2313,\n",
       " 'mac': 4013,\n",
       " 'usb': 6699,\n",
       " 'flash': 2692,\n",
       " 'sunshine': 6148,\n",
       " 'quiz': 5151,\n",
       " 'wkly': 7024,\n",
       " 'win': 6987,\n",
       " 'sony': 5865,\n",
       " 'dvd': 2350,\n",
       " 'player': 4875,\n",
       " 'country': 1914,\n",
       " 'algarve': 875,\n",
       " 'ansr': 927,\n",
       " '82277': 637,\n",
       " '50': 520,\n",
       " 'sp': 5898,\n",
       " 'tyrone': 6606,\n",
       " 'dear': 2066,\n",
       " 'th': 6333,\n",
       " 'birthday': 1275,\n",
       " 'loving': 3963,\n",
       " 'gopalettan': 2984,\n",
       " 'planning': 4871,\n",
       " 'small': 5796,\n",
       " 'gift': 2921,\n",
       " 'participate': 4724,\n",
       " 'welcome': 6933,\n",
       " 'contact': 1863,\n",
       " 'admin': 793,\n",
       " 'team': 6274,\n",
       " 'class': 1724,\n",
       " 'hours': 3282,\n",
       " 'sorry': 5877,\n",
       " 'okay': 4579,\n",
       " 'wait': 6834,\n",
       " 'rushing': 5458,\n",
       " 'school': 5527,\n",
       " 'rush': 5457,\n",
       " 'hungry': 3314,\n",
       " 'usually': 6712,\n",
       " 'stops': 6044,\n",
       " 'hella': 3177,\n",
       " 'weed': 6917,\n",
       " 'smoke': 5811,\n",
       " 'awarded': 1101,\n",
       " 'city': 1715,\n",
       " 'break': 1396,\n",
       " '200': 335,\n",
       " 'summer': 6139,\n",
       " 'shopping': 5681,\n",
       " 'spree': 5961,\n",
       " 'wk': 7020,\n",
       " 'store': 6046,\n",
       " '88039': 675,\n",
       " 'skilgme': 5759,\n",
       " 'tscs087147403231winawk': 6562,\n",
       " 'age16': 832,\n",
       " '50perwksub': 528,\n",
       " 'shall': 5635,\n",
       " 'pickle': 4836,\n",
       " 'isn': 3506,\n",
       " 'frnd': 2809,\n",
       " 'necesity': 4407,\n",
       " 'life': 3830,\n",
       " 'imagine': 3376,\n",
       " 'urself': 6697,\n",
       " 'witout': 7017,\n",
       " 'hw': 3328,\n",
       " 'feel': 2620,\n",
       " 'colleg': 1786,\n",
       " 'wth': 7102,\n",
       " 'cell': 1605,\n",
       " 'abt': 742,\n",
       " 'functions': 2833,\n",
       " 'thnk': 6378,\n",
       " 'events': 2491,\n",
       " 'espe': 2474,\n",
       " 'cared': 1553,\n",
       " 'missed': 4215,\n",
       " 'amp': 903,\n",
       " 'irritated': 3496,\n",
       " '4wrd': 517,\n",
       " 'frnds': 2810,\n",
       " 'wthout': 7103,\n",
       " 'live': 3880,\n",
       " 'jst': 3605,\n",
       " 'takecare': 6228,\n",
       " 'goodmorning': 2977,\n",
       " 'brum': 1437,\n",
       " 'thanks': 6337,\n",
       " 'putting': 5126,\n",
       " 'keeping': 3651,\n",
       " 'happy': 3120,\n",
       " 'soon': 5868,\n",
       " 'im': 3373,\n",
       " 'tellmiss': 6291,\n",
       " 'way': 6891,\n",
       " 'bloomberg': 1310,\n",
       " 'message': 4163,\n",
       " 'center': 1607,\n",
       " '447797706009': 482,\n",
       " 'apply': 969,\n",
       " 'future': 2842,\n",
       " 'http': 3298,\n",
       " 'careers': 1555,\n",
       " 'com': 1791,\n",
       " 'enjoy': 2435,\n",
       " 'plural': 4892,\n",
       " 'noun': 4515,\n",
       " 'research': 5331,\n",
       " 'sure': 6165,\n",
       " 'checking': 1656,\n",
       " 'happening': 3114,\n",
       " 'area': 991,\n",
       " 'aight': 850,\n",
       " 'sleeping': 5774,\n",
       " 'surfing': 6168,\n",
       " 'cool': 1888,\n",
       " 'breeze': 1406,\n",
       " 'bright': 1410,\n",
       " 'sun': 6142,\n",
       " 'fresh': 2793,\n",
       " 'flower': 2711,\n",
       " 'twittering': 6588,\n",
       " 'birds': 1270,\n",
       " 'waiting': 6837,\n",
       " 'wish': 7006,\n",
       " 'ringtone': 5389,\n",
       " 'order': 4630,\n",
       " 'reference': 5261,\n",
       " 'number': 4526,\n",
       " 'x49': 7114,\n",
       " 'mobile': 4246,\n",
       " 'charged': 1633,\n",
       " 'tone': 6470,\n",
       " 'arrive': 1016,\n",
       " 'customer': 2002,\n",
       " 'services': 5609,\n",
       " '09065989182': 214,\n",
       " 'colour': 1789,\n",
       " 'red': 5257,\n",
       " 'txtstar': 6598,\n",
       " 'does': 2245,\n",
       " 'uncle': 6626,\n",
       " 'timi': 6420,\n",
       " 'clearing': 1733,\n",
       " 'cars': 1569,\n",
       " 'getting': 2914,\n",
       " 'really': 5219,\n",
       " 'bad': 1128,\n",
       " 'totally': 6498,\n",
       " 'rejected': 5281,\n",
       " 'kinda': 3682,\n",
       " 'thing': 6366,\n",
       " 'sent': 5598,\n",
       " 'prices': 5028,\n",
       " 'mean': 4116,\n",
       " 'hi': 3202,\n",
       " 'darlin': 2038,\n",
       " 'london': 3914,\n",
       " 'smashed': 5801,\n",
       " 'driver': 2314,\n",
       " 'big': 1257,\n",
       " 'dent': 2118,\n",
       " 'missing': 4217,\n",
       " 'took': 6479,\n",
       " 'tablets': 6217,\n",
       " 'reaction': 5205,\n",
       " 'morning': 4289,\n",
       " 'going': 2959,\n",
       " 'intention': 3459,\n",
       " 'run': 5453,\n",
       " 'choose': 1700,\n",
       " 'clean': 1728,\n",
       " 'don': 2266,\n",
       " 'say': 5516,\n",
       " 'visitors': 6791,\n",
       " 'maybe': 4111,\n",
       " 'choice': 1698,\n",
       " 'wanted': 6859,\n",
       " 'embarassed': 2413,\n",
       " 'friend': 2800,\n",
       " 'wants': 6861,\n",
       " 'drop': 2319,\n",
       " 'buy': 1483,\n",
       " 'happened': 3113,\n",
       " 'tried': 6537,\n",
       " 'picking': 4835,\n",
       " 'various': 6735,\n",
       " 'points': 4914,\n",
       " 'yeovil': 7157,\n",
       " 'motor': 4298,\n",
       " 'project': 5064,\n",
       " '12': 266,\n",
       " '30': 412,\n",
       " 'max': 4106,\n",
       " 'easy': 2368,\n",
       " 'test': 6316,\n",
       " 'rd': 5199,\n",
       " 'fret': 2794,\n",
       " 'ovulation': 4673,\n",
       " 'strips': 6066,\n",
       " 'wont': 7048,\n",
       " 'til': 6416,\n",
       " 'march': 4072,\n",
       " 'postal': 4953,\n",
       " 'address': 790,\n",
       " 'alright': 887,\n",
       " 'loyalty': 3970,\n",
       " 'offer': 4560,\n",
       " 'nokia6650': 4485,\n",
       " '10': 244,\n",
       " 'txtauction': 6593,\n",
       " 'word': 7056,\n",
       " 'start': 5993,\n",
       " '81151': 632,\n",
       " '4t': 511,\n",
       " 'ctxt': 1982,\n",
       " 'tc': 6266,\n",
       " 'mtmsg': 4325,\n",
       " 'remember': 5295,\n",
       " 'alex': 873,\n",
       " 'pizza': 4860,\n",
       " 'av': 1084,\n",
       " 'wil': 6982,\n",
       " 'use': 6701,\n",
       " 'ta': 6214,\n",
       " 'urgent': 6688,\n",
       " 'trying': 6558,\n",
       " 'todays': 6454,\n",
       " 'draw': 2301,\n",
       " 'shows': 5701,\n",
       " '800': 617,\n",
       " 'prize': 5043,\n",
       " 'guaranteed': 3048,\n",
       " '09050001808': 158,\n",
       " 'land': 3740,\n",
       " 'claim': 1718,\n",
       " 'm95': 4008,\n",
       " 'valid12hrs': 6727,\n",
       " 'babe': 1119,\n",
       " 'lost': 3939,\n",
       " 'painful': 4690,\n",
       " 'personal': 4795,\n",
       " 'thought': 6384,\n",
       " 'try': 6556,\n",
       " 'everybody': 2492,\n",
       " 'recognises': 5250,\n",
       " 'subscribed': 6105,\n",
       " 'service': 5608,\n",
       " 'days': 2055,\n",
       " '82324': 638,\n",
       " 'helpline': 3185,\n",
       " '08706091795': 83,\n",
       " 'basket': 1164,\n",
       " 'mum': 4336,\n",
       " 'messages': 4165,\n",
       " 'got': 2988,\n",
       " 'actually': 781,\n",
       " 'rest': 5351,\n",
       " 'january': 3541,\n",
       " 'male': 4049,\n",
       " 'sale': 5480,\n",
       " 'hot': 3276,\n",
       " 'gay': 2877,\n",
       " 'cheaper': 1649,\n",
       " '08709222922': 88,\n",
       " 'national': 4391,\n",
       " 'rate': 5187,\n",
       " '5p': 548,\n",
       " 'min': 4187,\n",
       " 'cheap': 1648,\n",
       " '8p': 695,\n",
       " 'peak': 4765,\n",
       " 'texts': 6331,\n",
       " '08712460324': 108,\n",
       " '10p': 254,\n",
       " 'honesty': 3249,\n",
       " 'road': 5399,\n",
       " 'bank': 1146,\n",
       " 'tomorrow': 6469,\n",
       " 'tough': 6502,\n",
       " 'decisions': 2080,\n",
       " 'people': 4776,\n",
       " 'womdarfull': 7039,\n",
       " 'actor': 779,\n",
       " 'congrats': 1847,\n",
       " 'special': 5910,\n",
       " 'cinema': 1713,\n",
       " 'pass': 4731,\n",
       " '09061209465': 180,\n",
       " 'suprman': 6163,\n",
       " 'matrix3': 4101,\n",
       " 'starwars3': 5997,\n",
       " 'bx420': 1493,\n",
       " 'ip4': 3484,\n",
       " '5we': 552,\n",
       " '150pm': 295,\n",
       " 'miss': 4213,\n",
       " 'movie': 4304,\n",
       " 'juz': 3626,\n",
       " 'minute': 4202,\n",
       " 'decision': 2079,\n",
       " 'mah': 4031,\n",
       " 'watch': 6877,\n",
       " 'lar': 3751,\n",
       " 'tot': 6496,\n",
       " 'interested': 3460,\n",
       " 'care': 1551,\n",
       " 'sweet': 6191,\n",
       " 'dreams': 2305,\n",
       " 'ummifying': 6620,\n",
       " 'bye': 1495,\n",
       " 'gud': 3050,\n",
       " 'ni8': 4451,\n",
       " 'slp': 5794,\n",
       " 'swt': 6203,\n",
       " 'muah': 4330,\n",
       " 'youdoing': 7172,\n",
       " 'later': 3762,\n",
       " 'sar': 5495,\n",
       " 'money': 4270,\n",
       " '09050000460': 153,\n",
       " 'j89': 3526,\n",
       " 'po': 4897,\n",
       " 'box245c2150pm': 1364,\n",
       " 'issue': 3508,\n",
       " 'weigh': 6924,\n",
       " 'breathe': 1403,\n",
       " 'easier': 2363,\n",
       " 'regret': 5278,\n",
       " 'gr8': 3000,\n",
       " 'leaving': 3795,\n",
       " 'plans': 4872,\n",
       " 'havent': 3142,\n",
       " 'sir': 5742,\n",
       " 'group': 3034,\n",
       " 'mail': 4034,\n",
       " 'check': 1653,\n",
       " 'hit': 3216,\n",
       " 'cash': 1573,\n",
       " 'girl': 2928,\n",
       " '2004': 338,\n",
       " 'account': 760,\n",
       " 'statement': 5998,\n",
       " '07742676969': 27,\n",
       " '786': 608,\n",
       " 'unredeemed': 6661,\n",
       " 'bonus': 1332,\n",
       " '08719180248': 144,\n",
       " 'identifier': 3355,\n",
       " 'code': 1770,\n",
       " '45239': 489,\n",
       " 'expires': 2538,\n",
       " 'mins': 4200,\n",
       " 'busy': 1479,\n",
       " 'finish': 2670,\n",
       " 'looking': 3924,\n",
       " 'forward': 2761,\n",
       " 'finally': 2663,\n",
       " 'meeting': 4132,\n",
       " 'hmv': 3230,\n",
       " '500': 521,\n",
       " 'pounds': 4965,\n",
       " 'genuine': 2901,\n",
       " 'vouchers': 6808,\n",
       " 'answer': 928,\n",
       " 'questions': 5144,\n",
       " 'play': 4873,\n",
       " 'info': 3425,\n",
       " 'www': 7110,\n",
       " '100percent': 249,\n",
       " 'real': 5210,\n",
       " 'accidentally': 754,\n",
       " 'left': 3798,\n",
       " 'silent': 5726,\n",
       " 'night': 4460,\n",
       " 'didn': 2170,\n",
       " 'ya': 7134,\n",
       " 'tht': 6398,\n",
       " 'incident': 3399,\n",
       " 'good': 2972,\n",
       " 'problem': 5047,\n",
       " 'little': 3879,\n",
       " 'experience': 2534,\n",
       " 'understand': 6633,\n",
       " 'american': 895,\n",
       " 'voice': 6799,\n",
       " 'used': 6702,\n",
       " 'agents': 836,\n",
       " 'booked': 1336,\n",
       " 'things': 6367,\n",
       " 'boston': 1350,\n",
       " 'nyc': 4539,\n",
       " 'experiment': 2535,\n",
       " 'online': 4599,\n",
       " 'transaction': 6515,\n",
       " 'came': 1526,\n",
       " 'hostel': 3274,\n",
       " 'sleep': 5772,\n",
       " 'plz': 4895,\n",
       " 'hrishi': 3295,\n",
       " 'fuuuuck': 2843,\n",
       " 'sleepin': 5773,\n",
       " 'sup': 6150,\n",
       " 'dude': 2337,\n",
       " 'makin': 4045,\n",
       " 'weirdy': 6931,\n",
       " 'brownies': 1433,\n",
       " 'cookies': 1886,\n",
       " 'secret': 5560,\n",
       " 'admirer': 795,\n",
       " 'reveal': 5368,\n",
       " 'thinks': 6372,\n",
       " '09058094599': 175,\n",
       " 'oh': 4573,\n",
       " 'training': 6512,\n",
       " 'manual': 4066,\n",
       " 'tech': 6279,\n",
       " 'process': 5053,\n",
       " 'password': 4737,\n",
       " 'reset': 5337,\n",
       " 'troubleshooting': 6544,\n",
       " 'computerless': 1831,\n",
       " 'oreo': 4633,\n",
       " 'truffles': 6550,\n",
       " 'watching': 6880,\n",
       " 'tv': 6582,\n",
       " 'job': 3580,\n",
       " 'hoping': 3264,\n",
       " 'game': 2858,\n",
       " 'yesterday': 7162,\n",
       " 'touch': 6500,\n",
       " 'pls': 4888,\n",
       " 'fondly': 2725,\n",
       " 'bein': 1220,\n",
       " 'thot': 6382,\n",
       " 'abiola': 732,\n",
       " 'nite': 4470,\n",
       " 'pocay': 4907,\n",
       " 'wocay': 7035,\n",
       " '4eva': 502,\n",
       " 'promise': 5068,\n",
       " 'ring': 5387,\n",
       " '2morrowxxxx': 393,\n",
       " 'yup': 7189,\n",
       " 'thanx': 6342,\n",
       " 'reply': 5321,\n",
       " 'pick': 4833,\n",
       " 'heart': 3162,\n",
       " 'mind': 4190,\n",
       " 'wisdom': 7004,\n",
       " 'eyes': 2555,\n",
       " 'alwys': 891,\n",
       " 'panren': 4700,\n",
       " 'paru': 4729,\n",
       " 'wife': 6978,\n",
       " 'relax': 5286,\n",
       " 'wkend': 7021,\n",
       " 'fun': 2831,\n",
       " 'truly': 6551,\n",
       " 'forget': 2746,\n",
       " 'gbp': 2881,\n",
       " 'sms': 5817,\n",
       " 'approve': 975,\n",
       " 'panalam': 4695,\n",
       " 'posts': 4959,\n",
       " 'paid': 4688,\n",
       " 'commercial': 1804,\n",
       " 'hasbro': 3129,\n",
       " 'august': 1075,\n",
       " 'jump': 3616,\n",
       " 'hoops': 3258,\n",
       " 'shut': 5711,\n",
       " 'omg': 4592,\n",
       " 'dream': 2304,\n",
       " 'kids': 3673,\n",
       " 'boys': 1381,\n",
       " 'pissed': 4855,\n",
       " 'told': 6460,\n",
       " 'mark': 4076,\n",
       " 'changing': 1626,\n",
       " 'diapers': 2164,\n",
       " 'cause': 1588,\n",
       " 'owed': 4676,\n",
       " 'face': 2559,\n",
       " 'ey': 2553,\n",
       " 'calm': 1523,\n",
       " 'downon': 2292,\n",
       " 'theacusations': 6347,\n",
       " 'itxt': 3517,\n",
       " 'iwana': 3520,\n",
       " 'wotu': 7077,\n",
       " 'doin': 2257,\n",
       " 'thew': 6362,\n",
       " 'end': 2422,\n",
       " 'haventcn': 3143,\n",
       " 'ages': 837,\n",
       " 'up4': 6669,\n",
       " 'nething': 4432,\n",
       " 'sat': 5502,\n",
       " 'dramatic': 2299,\n",
       " 'schools': 5528,\n",
       " 'closed': 1742,\n",
       " 'apparently': 961,\n",
       " 'inch': 3397,\n",
       " 'snow': 5832,\n",
       " 'supposed': 6161,\n",
       " 'woke': 7036,\n",
       " 'blur': 1319,\n",
       " 'went': 6940,\n",
       " 'liao': 3821,\n",
       " 'oso': 4645,\n",
       " 'helloooo': 3179,\n",
       " 'wake': 6838,\n",
       " 'welcomes': 6934,\n",
       " 'joy': 3600,\n",
       " 'mrng': 4312,\n",
       " 'jason': 3546,\n",
       " 'says': 5518,\n",
       " 'gonna': 2970,\n",
       " 'mallika': 4051,\n",
       " 'sherawat': 5652,\n",
       " 'url': 6694,\n",
       " 'wa': 6823,\n",
       " 'openin': 4608,\n",
       " 'sentence': 5599,\n",
       " 'formal': 2753,\n",
       " 'fine': 2667,\n",
       " 'tt': 6565,\n",
       " 'eatin': 2371,\n",
       " 'puttin': 5125,\n",
       " 'weight': 6926,\n",
       " 'anythin': 944,\n",
       " 'tyler': 6602,\n",
       " 'minor': 4199,\n",
       " 'crisis': 1960,\n",
       " 'sooner': 5869,\n",
       " 'asap': 1024,\n",
       " 'erm': 2460,\n",
       " 'ill': 3369,\n",
       " '45pm': 490,\n",
       " 'park': 4720,\n",
       " 'treat': 6529,\n",
       " 'pending': 4773,\n",
       " 'respect': 5343,\n",
       " 'mother': 4293,\n",
       " 'mails': 4037,\n",
       " 'alrite': 888,\n",
       " 'hunny': 3316,\n",
       " 'wot': 7076,\n",
       " '2nite': 398,\n",
       " 'didnt': 2171,\n",
       " 'goin': 2958,\n",
       " 'jus': 3622,\n",
       " 'pub': 5101,\n",
       " 'instead': 3453,\n",
       " 'chillin': 1686,\n",
       " 'mo': 4242,\n",
       " 'bedroom': 1206,\n",
       " 'jen': 3559,\n",
       " 'aiyo': 860,\n",
       " 'meh': 4137,\n",
       " 'princess': 5034,\n",
       " 'times': 6419,\n",
       " 'thats': 6346,\n",
       " 'ew': 2507,\n",
       " 'friendship': 2803,\n",
       " 'poem': 4911,\n",
       " 'near': 4405,\n",
       " 'hear': 3158,\n",
       " 'fear': 2613,\n",
       " 'cheer': 1660,\n",
       " 'tear': 6275,\n",
       " 'added': 786,\n",
       " 'list': 3868,\n",
       " 'fullonsms': 2829,\n",
       " 'ignoring': 3363,\n",
       " 'dating': 2049,\n",
       " 'started': 5994,\n",
       " 'sport': 5954,\n",
       " 'radio': 5160,\n",
       " 'connection': 1851,\n",
       " 'coincidence': 1774,\n",
       " 'hook': 3257,\n",
       " 'means': 4120,\n",
       " 'right': 5383,\n",
       " 'valuable': 6728,\n",
       " 'situations': 5752,\n",
       " 'second': 5557,\n",
       " 'loosing': 3930,\n",
       " 'si': 5713,\n",
       " 'como': 1808,\n",
       " 'listened2the': 3871,\n",
       " 'plaid': 4866,\n",
       " 'album': 867,\n",
       " 'gd': 2884,\n",
       " 'air1': 854,\n",
       " 'hilarious': 3207,\n",
       " 'bought': 1354,\n",
       " 'braindance': 1386,\n",
       " 'comp': 1809,\n",
       " 'ofstuff': 4570,\n",
       " 'aphex': 953,\n",
       " 'abel': 728,\n",
       " 'hav2hear': 3139,\n",
       " 'xxxx': 7129,\n",
       " 'wana': 6854,\n",
       " 'praps': 4988,\n",
       " 'meant': 4121,\n",
       " 'goodo': 2981,\n",
       " 'stay': 6001,\n",
       " 'england': 2433,\n",
       " 'official': 4566,\n",
       " 'poly': 4918,\n",
       " 'flag': 2688,\n",
       " 'yer': 7159,\n",
       " '84199': 653,\n",
       " 'opt': 4618,\n",
       " 'eng': 2430,\n",
       " 'box39822': 1369,\n",
       " 'w111wx': 6814,\n",
       " 'honeybee': 3251,\n",
       " 'said': 5475,\n",
       " 'sweetest': 6192,\n",
       " 'world': 7065,\n",
       " 'god': 2954,\n",
       " 'laughed': 3767,\n",
       " 'havnt': 3147,\n",
       " 'met': 4170,\n",
       " 'person': 4794,\n",
       " 'reading': 5208,\n",
       " 'moral': 4285,\n",
       " 'crack': 1934,\n",
       " 'jokes': 3590,\n",
       " 'gm': 2943,\n",
       " 'gn': 2945,\n",
       " 'ge': 2886,\n",
       " 'study': 6082,\n",
       " 'tap': 6247,\n",
       " 'spile': 5931,\n",
       " 'seven': 5616,\n",
       " 'gas': 2870,\n",
       " 'broad': 1422,\n",
       " 'canal': 1530,\n",
       " 'wan2': 6853,\n",
       " 'greet': 3023,\n",
       " 'westlife': 6950,\n",
       " 'm8': 4006,\n",
       " 'currently': 1997,\n",
       " 'tour': 6503,\n",
       " 'unbreakable': 6625,\n",
       " 'untamed': 6667,\n",
       " 'unkempt': 6652,\n",
       " '83049': 640,\n",
       " 'cost': 1903,\n",
       " '50p': 526,\n",
       " 'std': 6006,\n",
       " 'drunk': 2326,\n",
       " 'motherfucker': 4294,\n",
       " 'textbuddy': 6324,\n",
       " 'horny': 3266,\n",
       " 'guys': 3068,\n",
       " '25p': 362,\n",
       " 'receive': 5237,\n",
       " 'search': 5552,\n",
       " 'postcode': 4955,\n",
       " 'gaytextbuddy': 2880,\n",
       " '89693': 690,\n",
       " '08715500022': 122,\n",
       " 'rpl': 5434,\n",
       " 'cnl': 1759,\n",
       " 'hiya': 3220,\n",
       " 'weekend': 6920,\n",
       " 'usual': 6711,\n",
       " 'ard': 990,\n",
       " 'smth': 5821,\n",
       " 'doc': 2239,\n",
       " 'gave': 2876,\n",
       " 'pain': 4689,\n",
       " 'meds': 4128,\n",
       " 'mahal': 4032,\n",
       " 'bus': 1473,\n",
       " 'decimal': 2078,\n",
       " 'cooking': 1887,\n",
       " 'oops': 4604,\n",
       " 'tomo': 6467,\n",
       " 'station': 5999,\n",
       " 'dhoni': 2157,\n",
       " 'luck': 3978,\n",
       " 'title': 6433,\n",
       " 'waking': 6839,\n",
       " 'afternoon': 828,\n",
       " 'church': 1710,\n",
       " 'holla': 3240,\n",
       " 'let': 3815,\n",
       " 'salad': 5477,\n",
       " 'desert': 2132,\n",
       " 'beers': 1210,\n",
       " 'news': 4447,\n",
       " 'freefone': 2784,\n",
       " '08006344447': 56,\n",
       " '1000': 246,\n",
       " '2000': 336,\n",
       " 'speak': 5908,\n",
       " 'operator': 4612,\n",
       " 'hr': 3294,\n",
       " 'trip': 6538,\n",
       " 'audition': 1072,\n",
       " 'wednesday': 6914,\n",
       " 'whats': 6955,\n",
       " 'printed': 5036,\n",
       " 'upstairs': 6682,\n",
       " 'wasnt': 6872,\n",
       " 'phoned': 4821,\n",
       " 'voda': 6796,\n",
       " 'numbers': 4527,\n",
       " 'ending': 2424,\n",
       " '1225': 270,\n",
       " 'selected': 5579,\n",
       " '50award': 523,\n",
       " 'match': 4094,\n",
       " '08712300220': 95,\n",
       " 'quoting': 5155,\n",
       " '3100': 424,\n",
       " 'standard': 5984,\n",
       " 'rates': 5188,\n",
       " 'app': 960,\n",
       " 'today': 6453,\n",
       " 'accept': 749,\n",
       " 'brother': 1429,\n",
       " 'lover': 3958,\n",
       " 'dear1': 2067,\n",
       " 'best1': 1238,\n",
       " 'clos1': 1740,\n",
       " 'lvblefrnd': 3993,\n",
       " 'jstfrnd': 3606,\n",
       " 'cutefrnd': 2007,\n",
       " 'lifpartnr': 3833,\n",
       " 'belovd': 1228,\n",
       " 'swtheart': 6204,\n",
       " 'bstfrnd': 1442,\n",
       " 'rply': 5435,\n",
       " 'enemy': 2428,\n",
       " 'bloo': 1307,\n",
       " 'bowl': 1358,\n",
       " 'urgnt': 6691,\n",
       " 'wating': 6885,\n",
       " 'difficult': 2181,\n",
       " 'girls': 2931,\n",
       " 'companion': 1811,\n",
       " 'chef': 1666,\n",
       " 'listener': 3872,\n",
       " 'organizer': 4636,\n",
       " 'boyfriend': 1380,\n",
       " 'sympathetic': 6207,\n",
       " 'athletic': 1054,\n",
       " 'warm': 6864,\n",
       " 'courageous': 1919,\n",
       " 'determined': 2145,\n",
       " 'true': 6548,\n",
       " 'dependable': 2125,\n",
       " 'intelligent': 3457,\n",
       " 'psychologist': 5098,\n",
       " 'pest': 4802,\n",
       " 'exterminator': 2550,\n",
       " 'psychiatrist': 5096,\n",
       " 'healer': 3156,\n",
       " 'stylist': 6094,\n",
       " 'aaniye': 722,\n",
       " 'pudunga': 5105,\n",
       " 'venaam': 6749,\n",
       " 'request': 5326,\n",
       " 'maangalyam': 4011,\n",
       " 'alaipayuthe': 866,\n",
       " 'set': 5611,\n",
       " 'callertune': 1516,\n",
       " 'callers': 1515,\n",
       " 'press': 5017,\n",
       " 'copy': 1893,\n",
       " 'friends': 2801,\n",
       " 'tirunelvai': 6428,\n",
       " 'lick': 3825,\n",
       " 'pussy': 5123,\n",
       " 'inside': 3447,\n",
       " 'office': 4564,\n",
       " 'filling': 2654,\n",
       " 'forms': 2757,\n",
       " 'textin': 6327,\n",
       " 'bout': 1356,\n",
       " 'worries': 7069,\n",
       " 'photo': 4824,\n",
       " 'shoot': 5678,\n",
       " ...}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vec.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4179"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1393"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.75"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(x_train)/(len(x_train)+len(x_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_transform= vec.transform(x_train)\n",
    "x_test_transform=vec.transform(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 50)\t1\n",
      "  (0, 264)\t1\n",
      "  (0, 509)\t1\n",
      "  (0, 1527)\t1\n",
      "  (0, 1971)\t1\n",
      "  (0, 2780)\t2\n",
      "  (0, 3089)\t1\n",
      "  (0, 3763)\t1\n",
      "  (0, 3852)\t1\n",
      "  (0, 4248)\t1\n",
      "  (0, 4624)\t1\n",
      "  (0, 4626)\t1\n",
      "  (0, 4818)\t1\n",
      "  (0, 4822)\t1\n",
      "  (0, 5027)\t1\n",
      "  (0, 5310)\t1\n",
      "  (0, 6673)\t1\n",
      "  (1, 2169)\t1\n",
      "  (1, 6028)\t1\n",
      "  (1, 6545)\t1\n",
      "  (2, 98)\t1\n",
      "  (2, 563)\t1\n",
      "  (2, 1867)\t1\n",
      "  (2, 2436)\t1\n",
      "  (2, 3180)\t1\n",
      "  :\t:\n",
      "  (4176, 3879)\t1\n",
      "  (4176, 4417)\t1\n",
      "  (4176, 5229)\t1\n",
      "  (4176, 6191)\t1\n",
      "  (4176, 7134)\t1\n",
      "  (4177, 254)\t1\n",
      "  (4177, 307)\t1\n",
      "  (4177, 358)\t1\n",
      "  (4177, 831)\t1\n",
      "  (4177, 2046)\t1\n",
      "  (4177, 2704)\t1\n",
      "  (4177, 3585)\t1\n",
      "  (4177, 3623)\t1\n",
      "  (4177, 4130)\t1\n",
      "  (4177, 4315)\t1\n",
      "  (4177, 4771)\t1\n",
      "  (4177, 5234)\t1\n",
      "  (4177, 5321)\t1\n",
      "  (4177, 5487)\t1\n",
      "  (4177, 5620)\t1\n",
      "  (4177, 6321)\t1\n",
      "  (4177, 6374)\t1\n",
      "  (4177, 6453)\t1\n",
      "  (4178, 1643)\t1\n",
      "  (4178, 5817)\t1\n"
     ]
    }
   ],
   "source": [
    "print(x_train_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnb=MultinomialNB()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultinomialNB()"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnb.fit(x_train_transform, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_class=mnb.predict(x_test_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_proba=mnb.predict_proba(x_test_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9877961234745154"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.accuracy_score(y_test, y_pred_class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, ..., 0, 1, 0], dtype=int64)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultinomialNB()"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1201,    7],\n",
       "       [  10,  175]], dtype=int64)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.confusion_matrix(y_test, y_pred_class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1201    7]\n",
      " [  10  175]]\n"
     ]
    }
   ],
   "source": [
    "confusion=metrics.confusion_matrix(y_test, y_pred_class)\n",
    "print(confusion)\n",
    "TN=confusion[0,0]\n",
    "FP=confusion[0,1]\n",
    "FN=confusion[1,0]\n",
    "TP=confusion[1,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9459459459459459\n"
     ]
    }
   ],
   "source": [
    "sensitivity= TP/float(FN+TP)\n",
    "print(sensitivity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9942052980132451"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "specificity=TN/float(TN+FP)\n",
    "specificity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9615384615384616\n",
      "0.9615384615384616\n"
     ]
    }
   ],
   "source": [
    "precision=TP/(FP+TP)\n",
    "print(precision)\n",
    "print(metrics.precision_score(y_test, y_pred_class))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9459459459459459"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.recall_score(y_test, y_pred_class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9536784741144414"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.f1_score(y_test, y_pred_class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, ..., 0, 1, 0], dtype=int64)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[9.95239557e-01, 4.76044325e-03],\n",
       "       [9.99852357e-01, 1.47642544e-04],\n",
       "       [9.27878579e-01, 7.21214213e-02],\n",
       "       ...,\n",
       "       [9.99999671e-01, 3.28799076e-07],\n",
       "       [3.72703622e-09, 9.99999996e-01],\n",
       "       [9.99999985e-01, 1.46852511e-08]])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_proba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix as sk_confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve, auc\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "false_positive_rate, true_positive_rate, thresholds=roc_curve(y_test, y_pred_proba[:,1])\n",
    "roc_auc=auc(false_positive_rate, true_positive_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9921872203329157"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.         0.30810811 0.31351351 0.33513514 0.34054054 0.35135135\n",
      " 0.36756757 0.4        0.41081081 0.59459459 0.60540541 0.61621622\n",
      " 0.62702703 0.67567568 0.68648649 0.71891892 0.72972973 0.94594595\n",
      " 0.94594595 0.94594595 0.94594595 0.95135135 0.95135135 0.96756757\n",
      " 0.96756757 0.96756757 0.96756757 0.96756757 0.96756757 0.96756757\n",
      " 0.96756757 0.96756757 0.96756757 0.96756757 0.97297297 0.97297297\n",
      " 0.97297297 0.97297297 0.97297297 0.97297297 0.97297297 0.97297297\n",
      " 0.97297297 0.97297297 0.97297297 0.97297297 0.97297297 0.97297297\n",
      " 0.97297297 0.97297297 0.97297297 0.97297297 0.97297297 0.97297297\n",
      " 0.97297297 0.97297297 0.97837838 0.97837838 0.97837838 0.97837838\n",
      " 0.97837838 0.97837838 0.97837838 0.97837838 0.97837838 0.97837838\n",
      " 0.97837838 0.97837838 0.98378378 0.98378378 0.98378378 0.98378378\n",
      " 0.98918919 0.98918919 0.98918919 0.98918919 0.99459459 0.99459459\n",
      " 1.         1.         1.         1.         1.         1.\n",
      " 1.         1.         1.         1.         1.         1.\n",
      " 1.         1.         1.         1.         1.         1.\n",
      " 1.         1.         1.         1.         1.         1.\n",
      " 1.         1.         1.         1.         1.         1.\n",
      " 1.         1.        ]\n"
     ]
    }
   ],
   "source": [
    "print(true_positive_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "110"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(true_positive_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1393"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.00165563, 0.00331126,\n",
       "       0.0115894 , 0.0115894 , 0.01324503, 0.01324503, 0.02235099,\n",
       "       0.02400662, 0.0281457 , 0.03890728, 0.04470199, 0.04635762,\n",
       "       0.04966887, 0.05711921, 0.06043046, 0.06125828, 0.06125828,\n",
       "       0.06374172, 0.06870861, 0.07036424, 0.0736755 , 0.07533113,\n",
       "       0.08360927, 0.08857616, 0.09519868, 0.0968543 , 0.10099338,\n",
       "       0.10264901, 0.11672185, 0.11837748, 0.12665563, 0.12831126,\n",
       "       0.13162252, 0.13327815, 0.14072848, 0.14238411, 0.14652318,\n",
       "       0.1531457 , 0.1531457 , 0.1589404 , 0.16059603, 0.16307947,\n",
       "       0.1647351 , 0.20943709, 0.21109272, 0.22930464, 0.23096026,\n",
       "       0.23592715, 0.2384106 , 0.26655629, 0.26655629, 0.27400662,\n",
       "       0.27566225, 0.29304636, 0.29304636, 0.30215232, 0.30380795,\n",
       "       0.30711921, 0.30711921, 0.31291391, 0.31291391, 0.32036424,\n",
       "       0.32201987, 0.37086093, 0.37251656, 0.38493377, 0.3865894 ,\n",
       "       0.43708609, 0.43956954, 0.46854305, 0.47019868, 0.59271523,\n",
       "       0.59437086, 0.64403974, 0.65231788, 0.6986755 , 0.70033113,\n",
       "       0.78062914, 0.78228477, 0.80298013, 0.80463576, 0.85347682,\n",
       "       0.85513245, 0.86589404, 0.86754967, 0.94039735, 0.94205298,\n",
       "       0.95943709, 0.96109272, 0.97019868, 0.97268212, 1.        ])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "false_positive_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2.00000000e+00, 1.00000000e+00, 1.00000000e+00, 1.00000000e+00,\n",
       "       1.00000000e+00, 1.00000000e+00, 1.00000000e+00, 1.00000000e+00,\n",
       "       1.00000000e+00, 9.99999996e-01, 9.99999996e-01, 9.99999995e-01,\n",
       "       9.99999994e-01, 9.99999891e-01, 9.99999451e-01, 9.99998530e-01,\n",
       "       9.99997944e-01, 8.23259521e-01, 6.03514139e-01, 5.57484017e-01,\n",
       "       2.93007628e-01, 2.83243254e-01, 2.56756624e-01, 2.53425148e-01,\n",
       "       1.34568519e-01, 1.34539652e-01, 1.34510790e-01, 1.34481933e-01,\n",
       "       1.01551419e-01, 9.39254835e-02, 9.38832989e-02, 7.38341494e-02,\n",
       "       7.21048341e-02, 7.10172010e-02, 7.04886074e-02, 6.89078701e-02,\n",
       "       6.08617913e-02, 6.02739339e-02, 5.85415571e-02, 5.85278961e-02,\n",
       "       4.92769211e-02, 4.92536993e-02, 3.98139183e-02, 3.97949694e-02,\n",
       "       3.74097322e-02, 3.74097322e-02, 2.88222872e-02, 2.87588101e-02,\n",
       "       2.52608490e-02, 2.52547459e-02, 2.25172886e-02, 2.17200381e-02,\n",
       "       1.91483675e-02, 1.90614072e-02, 1.90567726e-02, 1.65820422e-02,\n",
       "       1.63327142e-02, 1.53112191e-02, 1.53037461e-02, 1.47786698e-02,\n",
       "       1.45856255e-02, 7.97660888e-03, 7.90702710e-03, 6.19870579e-03,\n",
       "       6.17975072e-03, 5.62111311e-03, 5.59444131e-03, 3.88193895e-03,\n",
       "       3.84792356e-03, 3.44265027e-03, 3.39834560e-03, 2.60587132e-03,\n",
       "       2.59421046e-03, 2.32873866e-03, 2.32283556e-03, 2.24586870e-03,\n",
       "       2.23235359e-03, 2.11790190e-03, 2.10924355e-03, 1.92895304e-03,\n",
       "       1.81240895e-03, 7.26923728e-04, 7.25944758e-04, 5.95846511e-04,\n",
       "       5.94317680e-04, 3.03834392e-04, 2.98201675e-04, 1.80130766e-04,\n",
       "       1.79181173e-04, 2.59758085e-05, 2.53136414e-05, 8.39395764e-06,\n",
       "       8.20621267e-06, 3.22446568e-06, 3.20986819e-06, 1.88494983e-07,\n",
       "       1.87353914e-07, 6.41159776e-08, 6.35714673e-08, 6.43979696e-09,\n",
       "       6.35499161e-09, 4.00340347e-09, 3.87006776e-09, 2.32377322e-11,\n",
       "       2.23338186e-11, 6.32559568e-13, 6.08660437e-13, 4.17421525e-14,\n",
       "       3.96833119e-14, 6.22713116e-41])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "thresholds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Threshold</th>\n",
       "      <th>TPR</th>\n",
       "      <th>FPR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.308108</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.313514</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.335135</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.340541</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>6.325596e-13</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.959437</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>6.086604e-13</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.961093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>4.174215e-14</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.970199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>3.968331e-14</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.972682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>6.227131e-41</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>110 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Threshold       TPR       FPR\n",
       "0    2.000000e+00  0.000000  0.000000\n",
       "1    1.000000e+00  0.308108  0.000000\n",
       "2    1.000000e+00  0.313514  0.000000\n",
       "3    1.000000e+00  0.335135  0.000000\n",
       "4    1.000000e+00  0.340541  0.000000\n",
       "..            ...       ...       ...\n",
       "105  6.325596e-13  1.000000  0.959437\n",
       "106  6.086604e-13  1.000000  0.961093\n",
       "107  4.174215e-14  1.000000  0.970199\n",
       "108  3.968331e-14  1.000000  0.972682\n",
       "109  6.227131e-41  1.000000  1.000000\n",
       "\n",
       "[110 rows x 3 columns]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame({'Threshold':thresholds, 'TPR': true_positive_rate, 'FPR': false_positive_rate})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x20536572b50>]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAZHElEQVR4nO3de7QlZX3m8e9jQ0fkqtI62NB2a1qwWQHEFgTFgFdAHTTeQKOjiQuJoGYZHYgaTbwlBsdR4oVpkUEclXhBRYOgJgGMyFUbaECYHhBogUWDLhTQaMNv/qhq3Z4+l93dp/bpc+r7Weuss6vq3bV/L6fZz66qXe+bqkKS1F8PmukCJEkzyyCQpJ4zCCSp5wwCSeo5g0CSes4gkKSeMwgkqecMAmkSSX6c5JdJ7klye5LTkmw3sP3AJP+W5BdJ7k7y9STLxuxjhyQfTnJzu5/V7fLOo++RtCGDQJra86tqO2Af4AnAXwMkOQD4FvA14FHAEuAK4HtJHtO2mQ/8K7AncCiwA3AgcBew32i7IY0v3lksTSzJj4HXVtV32uV/BPasqucm+S5wVVW9fsxzvgmsrapXJXkt8D7gsVV1z4jLl4biEYE0pCS7AocBq5M8hOaT/RfHafoF4Fnt42cC5xgC2pIZBNLUvprkF8AtwB3Au4CH0fz/c9s47W8D1p//f/gEbaQthkEgTe0FVbU9cDCwB82b/M+AB4Bdxmm/C3Bn+/iuCdpIWwyDQBpSVZ0PnAZ8sKruBb4PvGScpi+luUAM8B3gOUm2HUmR0iYwCKSN82HgWUn2AU4A/luSNybZPslDk7wXOAD4u7b9Z2hOKX05yR5JHpTk4UneluTwmemC9PsMAmkjVNVa4HTgb6rqP4DnAH9Ccx3gJpqvlz61qv5v2/4/aS4Y/wj4NvBz4BKa00sXj7wD0jj8+qgk9ZxHBJLUcwaBJPWcQSBJPWcQSFLPbTXTBWysnXfeuRYvXjzTZUjSrHL55ZffWVULxts264Jg8eLFXHbZZTNdhiTNKklummibp4YkqecMAknqOYNAknrOIJCknjMIJKnnOguCJKcmuSPJqgm2J8lJ7UTeVybZt6taJEkT6/KI4DSaybonchiwtP05GvhEh7VIkibQ2X0EVXVBksWTNDkCOL2a4U8vSrJTkl2qymn95qDPXXwzX1v5k5kuQ5rVlj1qB971/D2nfb8zeUPZQpoJO9Zb067bIAiSHE1z1MCiRYtGUpymx/oAuPjGnwKw/5KHzXBFksaaySDIOOvGnRyhqlYAKwCWL1/uBApbsLGf/AcD4Ih9FvLy/Q1yaUszk0GwBthtYHlX4NYZqkVDmuoUz9hP/gaAtOWbySA4CzguyRnA/sDdXh/Y8n1t5U+45rafs2yXHcbd7hu/NPt0FgRJPg8cDOycZA3wLmBrgKo6GTgbOBxYDdwHvKarWma7LelC6/oQ+OfXHTDTpUiaJl1+a+ioKbYXcGxXr9+lUb8xb0kXWpftsgNH7LNwpsuQNI1m3TDU02Vz3sxH/cbs6RZJXeptEEx1rnsyvjFLmkt6GwSA57olCQedk6Te680RwXg3Om0JF18laab15ohgcJgD+N15fknqu94cEUDz5u81AUn6fb05IpAkjc8gkKSeMwgkqecMAknqOYNAknrOIJCknjMIJKnnDAJJ6jmDQJJ6ziCQpJ4zCCSp5wwCSeo5g0CSes4gkKSeMwgkqecMAknqOYNAknrOIJCknjMIJKnnDAJJ6jmDQJJ6ziCQpJ4zCCSp5wwCSeq5ToMgyaFJrkuyOskJ42zfMcnXk1yR5Ookr+myHknShjoLgiTzgI8BhwHLgKOSLBvT7FjgmqraGzgY+B9J5ndVkyRpQ10eEewHrK6qG6rq18AZwBFj2hSwfZIA2wE/BdZ1WJMkaYwug2AhcMvA8pp23aCPAo8HbgWuAt5UVQ+M3VGSo5NcluSytWvXdlWvJPVSl0GQcdbVmOXnACuBRwH7AB9NssMGT6paUVXLq2r5ggULpr9SSeqxLoNgDbDbwPKuNJ/8B70GOLMaq4EbgT06rEmSNEaXQXApsDTJkvYC8JHAWWPa3Aw8AyDJI4HdgRs6rEmSNMZWXe24qtYlOQ44F5gHnFpVVyc5pt1+MvAe4LQkV9GcSjq+qu7sqiZJ0oY6CwKAqjobOHvMupMHHt8KPLvLGiRJk/POYknqOYNAknrOIJCknjMIJKnnDAJJ6jmDQJJ6ziCQpJ4zCCSp5wwCSeo5g0CSem7oIEiybZeFSJJmxpRBkOTAJNcA17bLeyf5eOeVSZJGYpgjgv9JM4HMXQBVdQXwtC6LkiSNzlCnhqrqljGr7u+gFknSDBhmGOpbkhwIVDvBzBtpTxNJkma/YY4IjgGOpZl4fg3N3MKv77IoSdLoDHNEsHtVvWJwRZKnAN/rpiRJ0igNc0TwT0OukyTNQhMeESQ5ADgQWJDkzQObdqCZg1iSNAdMdmpoPrBd22b7gfU/B17cZVGSpNGZMAiq6nzg/CSnVdVNI6xJkjRCw1wsvi/JicCewIPXr6yqp3dWlSRpZIa5WPxZ4EfAEuDvgB8Dl3ZYkyRphIYJgodX1aeA31TV+VX1Z8CTO65LkjQiw5wa+k37+7YkzwVuBXbtriRJ0igNEwTvTbIj8Fc09w/sAPxlp1VJkkZmyiCoqm+0D+8GDoHf3lksSZoDJruhbB7wUpoxhs6pqlVJnge8DdgGeMJoSpQkdWmyI4JPAbsBlwAnJbkJOAA4oaq+OoriJEndmywIlgN7VdUDSR4M3An8YVXdPprSJEmjMNnXR39dVQ8AVNWvgOs3NgSSHJrkuiSrk5wwQZuDk6xMcnWS8zdm/5KkzTfZEcEeSa5sHwd4bLscoKpqr8l23F5j+BjwLJp5DC5NclZVXTPQZifg48ChVXVzkkdsRl8kSZtgsiB4/Gbuez9gdVXdAJDkDOAI4JqBNi8HzqyqmwGq6o7NfE1J0kaabNC5zR1obiEwONfxGmD/MW0eB2yd5DyaEU4/UlWnj91RkqOBowEWLVq0mWVJkgYNNXn9Jso462rM8lbAE4HnAs8B/ibJ4zZ4UtWKqlpeVcsXLFgw/ZVKUo8Nc2fxplpD8/XT9XalGZ5ibJs7q+pe4N4kFwB7A9d3WJckacBQRwRJtkmy+0bu+1JgaZIlSeYDRwJnjWnzNeCgJFsleQjNqaNrN/J1JEmbYcogSPJ8YCVwTru8T5Kxb+gbqKp1wHHAuTRv7l+oqquTHJPkmLbNte1+r6S5ce2Uqlq1qZ2RJG28YU4N/S3NN4DOA6iqlUkWD7PzqjobOHvMupPHLJ8InDjM/iRJ02+YU0PrquruziuRJM2IYY4IViV5OTAvyVLgjcCF3ZYlSRqVYY4I3kAzX/F/Ap+jGY7a+QgkaY4Y5ohg96p6O/D2rouRJI3eMEcEH0ryoyTvSbJn5xVJkkZqyiCoqkOAg4G1wIokVyV5R9eFSZJGY6gbyqrq9qo6CTiG5p6Cd3ZalSRpZIa5oezxSf42ySrgozTfGNq188okSSMxzMXi/w18Hnh2VY0dK0iSNMtNGQRV9eRRFCJJmhkTBkGSL1TVS5Ncxe8PHz3UDGWSpNlhsiOCN7W/nzeKQiRJM2PCi8VVdVv78PVVddPgD/D60ZQnSeraMF8ffdY46w6b7kIkSTNjsmsEf0Hzyf8xSa4c2LQ98L2uC5MkjcZk1wg+B3wT+HvghIH1v6iqn3ZalSRpZCYLgqqqHyc5duyGJA8zDCRpbpjqiOB5wOU0Xx/NwLYCHtNhXZKkEZkwCKrqee3vJaMrR5I0asOMNfSUJNu2j/80yYeSLOq+NEnSKAzz9dFPAPcl2Rv478BNwGc6rUqSNDLDTl5fwBHAR6rqIzRfIZUkzQHDjD76iyR/DbwSOCjJPGDrbsuSJI3KMEcEL6OZuP7Pqup2YCFwYqdVSZJGZpipKm8HPgvsmOR5wK+q6vTOK5MkjcQw3xp6KXAJ8BLgpcDFSV7cdWGSpNEY5hrB24EnVdUdAEkWAN8BvtRlYZKk0RjmGsGD1odA664hnydJmgWGOSI4J8m5NPMWQ3Px+OzuSpIkjdIwcxa/NcmfAE+lGW9oRVV9pfPKJEkjMdl8BEuBDwKPBa4C3lJVPxlVYZKk0ZjsXP+pwDeAF9GMQPpPG7vzJIcmuS7J6iQnTNLuSUnu99tIkjR6k50a2r6qPtk+vi7JDzZmx+0dyB+jmepyDXBpkrOq6ppx2n0AOHdj9i9Jmh6TBcGDkzyB381DsM3gclVNFQz7Aaur6gaAJGfQjFd0zZh2bwC+DDxpI2uXJE2DyYLgNuBDA8u3DywX8PQp9r0QuGVgeQ2w/2CDJAuBF7b7mjAIkhwNHA2waJEjYEvSdJpsYppDNnPfGWddjVn+MHB8Vd2fjNf8t7WsAFYALF++fOw+JEmbYZj7CDbVGmC3geVdgVvHtFkOnNGGwM7A4UnWVdVXO6xLkjSgyyC4FFiaZAnwE+BI4OWDDQanwUxyGvANQ0CSRquzIKiqdUmOo/k20Dzg1Kq6Oskx7faTu3ptSdLwpgyCNOdtXgE8pqre3c5X/F+q6pKpnltVZzNmOIqJAqCqXj1UxZKkaTXM4HEfBw4AjmqXf0Fzf4AkaQ4Y5tTQ/lW1b5IfAlTVz5LM77guSdKIDHNE8Jv27t+C385H8ECnVUmSRmaYIDgJ+ArwiCTvA/4DeH+nVUmSRmaYYag/m+Ry4Bk0N4m9oKqu7bwySdJIDPOtoUXAfcDXB9dV1c1dFiZJGo1hLhb/C831gQAPBpYA1wF7dliXJGlEhjk19EeDy0n2BV7XWUWSpJHa6Eno2+GnHTJakuaIYa4RvHlg8UHAvsDaziqSJI3UMNcIth94vI7mmsGXuylHkjRqkwZBeyPZdlX11hHVI0kasQmvESTZqqrupzkVJEmaoyY7IriEJgRWJjkL+CJw7/qNVXVmx7VJkkZgmGsEDwPuoplXeP39BAUYBJI0B0wWBI9ovzG0it8FwHrOGyxJc8RkQTAP2I7hJqGXJM1SkwXBbVX17pFVIkmaEZPdWTzekYAkaY6ZLAieMbIqJEkzZsIgqKqfjrIQSdLM2OhB5yRJc4tBIEk9ZxBIUs8ZBJLUcwaBJPWcQSBJPWcQSFLPGQSS1HMGgST1XKdBkOTQJNclWZ3khHG2vyLJle3PhUn27rIeSdKGOguCdr7jjwGHAcuAo5IsG9PsRuCPq2ov4D3Aiq7qkSSNr8sjgv2A1VV1Q1X9GjgDOGKwQVVdWFU/axcvAnbtsB5J0ji6DIKFwC0Dy2vadRP5c+Cb421IcnSSy5Jctnbt2mksUZLUZRAMPbNZkkNoguD48bZX1YqqWl5VyxcsWDCNJUqShpm8flOtAXYbWN4VuHVsoyR7AacAh1XVXR3WI0kaR5dHBJcCS5MsSTIfOBI4a7BBkkXAmcArq+r6DmuRJE2gsyOCqlqX5DjgXGAecGpVXZ3kmHb7ycA7gYcDH08CsK6qlndVkyRpQ12eGqKqzgbOHrPu5IHHrwVe22UNkqTJeWexJPWcQSBJPWcQSFLPGQSS1HMGgST1nEEgST1nEEhSzxkEktRzBoEk9ZxBIEk9ZxBIUs8ZBJLUcwaBJPWcQSBJPWcQSFLPGQSS1HMGgST1nEEgST1nEEhSzxkEktRzBoEk9ZxBIEk9ZxBIUs8ZBJLUcwaBJPWcQSBJPWcQSFLPGQSS1HMGgST1nEEgST1nEEhSz3UaBEkOTXJdktVJThhne5Kc1G6/Msm+XdYjSdpQZ0GQZB7wMeAwYBlwVJJlY5odBixtf44GPtFVPZKk8XV5RLAfsLqqbqiqXwNnAEeMaXMEcHo1LgJ2SrJLhzVJksbYqsN9LwRuGVheA+w/RJuFwG2DjZIcTXPEwKJFizapmGWP2mGTnidJc12XQZBx1tUmtKGqVgArAJYvX77B9mG86/l7bsrTJGnO6/LU0Bpgt4HlXYFbN6GNJKlDXQbBpcDSJEuSzAeOBM4a0+Ys4FXtt4eeDNxdVbeN3ZEkqTudnRqqqnVJjgPOBeYBp1bV1UmOabefDJwNHA6sBu4DXtNVPZKk8XV5jYCqOpvmzX5w3ckDjws4tssaJEmT885iSeo5g0CSes4gkKSeMwgkqefSXK+dPZKsBW7axKfvDNw5jeXMBva5H+xzP2xOnx9dVQvG2zDrgmBzJLmsqpbPdB2jZJ/7wT73Q1d99tSQJPWcQSBJPde3IFgx0wXMAPvcD/a5Hzrpc6+uEUiSNtS3IwJJ0hgGgST13JwMgiSHJrkuyeokJ4yzPUlOardfmWTfmahzOg3R51e0fb0yyYVJ9p6JOqfTVH0eaPekJPcnefEo6+vCMH1OcnCSlUmuTnL+qGucbkP8294xydeTXNH2eVaPYpzk1CR3JFk1wfbpf/+qqjn1QzPk9f8DHgPMB64Alo1pczjwTZoZ0p4MXDzTdY+gzwcCD20fH9aHPg+0+zeaUXBfPNN1j+DvvBNwDbCoXX7ETNc9gj6/DfhA+3gB8FNg/kzXvhl9fhqwL7Bqgu3T/v41F48I9gNWV9UNVfVr4AzgiDFtjgBOr8ZFwE5Jdhl1odNoyj5X1YVV9bN28SKa2eBms2H+zgBvAL4M3DHK4joyTJ9fDpxZVTcDVNVs7/cwfS5g+yQBtqMJgnWjLXP6VNUFNH2YyLS/f83FIFgI3DKwvKZdt7FtZpON7c+f03yimM2m7HOShcALgZOZG4b5Oz8OeGiS85JcnuRVI6uuG8P0+aPA42mmub0KeFNVPTCa8mbEtL9/dToxzQzJOOvGfkd2mDazydD9SXIITRA8tdOKujdMnz8MHF9V9zcfFme9Yfq8FfBE4BnANsD3k1xUVdd3XVxHhunzc4CVwNOBxwLfTvLdqvp518XNkGl//5qLQbAG2G1geVeaTwob22Y2Gao/SfYCTgEOq6q7RlRbV4bp83LgjDYEdgYOT7Kuqr46mhKn3bD/tu+sqnuBe5NcAOwNzNYgGKbPrwH+oZoT6KuT3AjsAVwymhJHbtrfv+biqaFLgaVJliSZDxwJnDWmzVnAq9qr708G7q6q20Zd6DSass9JFgFnAq+cxZ8OB03Z56paUlWLq2ox8CXg9bM4BGC4f9tfAw5KslWShwD7A9eOuM7pNEyfb6Y5AiLJI4HdgRtGWuVoTfv715w7IqiqdUmOA86l+cbBqVV1dZJj2u0n03yD5HBgNXAfzSeKWWvIPr8TeDjw8fYT8rqaxSM3DtnnOWWYPlfVtUnOAa4EHgBOqapxv4Y4Gwz5d34PcFqSq2hOmxxfVbN2eOoknwcOBnZOsgZ4F7A1dPf+5RATktRzc/HUkCRpIxgEktRzBoEk9ZxBIEk9ZxBIUs8ZBNoitaOFrhz4WTxJ23um4fVOS3Jj+1o/SHLAJuzjlCTL2sdvG7Ptws2tsd3P+v8uq9oRN3eaov0+SQ6fjtfW3OXXR7VFSnJPVW033W0n2cdpwDeq6ktJng18sKr22oz9bXZNU+03yaeB66vqfZO0fzWwvKqOm+5aNHd4RKBZIcl2Sf61/bR+VZINRhpNskuSCwY+MR/Urn92ku+3z/1ikqneoC8A/rB97pvbfa1K8pftum2T/Es7/v2qJC9r15+XZHmSfwC2aev4bLvtnvb3Pw9+Qm+PRF6UZF6SE5NcmmaM+dcN8Z/l+7SDjSXZL808Ez9sf+/e3on7buBlbS0va2s/tX2dH47331E9NNNjb/vjz3g/wP00A4mtBL5Ccxf8Du22nWnuqlx/RHtP+/uvgLe3j+cB27dtLwC2bdcfD7xznNc7jXa+AuAlwMU0g7ddBWxLM7zx1cATgBcBnxx47o7t7/NoPn3/tqaBNutrfCHw6fbxfJpRJLcBjgbe0a7/A+AyYMk4dd4z0L8vAoe2yzsAW7WPnwl8uX38auCjA89/P/Cn7eOdaMYg2nam/97+zOzPnBtiQnPGL6tqn/ULSbYG3p/kaTRDJywEHgncPvCcS4FT27ZfraqVSf4YWAZ8rx1aYz7NJ+nxnJjkHcBamhFanwF8pZoB3EhyJnAQcA7wwSQfoDmd9N2N6Nc3gZOS/AFwKHBBVf2yPR21V343i9qOwFLgxjHP3ybJSmAxcDnw7YH2n06ylGYkyq0neP1nA/81yVva5QcDi5jd4xFpMxkEmi1eQTP71BOr6jdJfkzzJvZbVXVBGxTPBT6T5ETgZ8C3q+qoIV7jrVX1pfULSZ45XqOquj7JE2nGe/n7JN+qqncP04mq+lWS82iGTn4Z8Pn1Lwe8oarOnWIXv6yqfZLsCHwDOBY4iWa8nX+vqhe2F9bPm+D5AV5UVdcNU6/6wWsEmi12BO5oQ+AQ4NFjGyR5dNvmk8CnaKb7uwh4SpL15/wfkuRxQ77mBcAL2udsS3Na57tJHgXcV1X/B/hg+zpj/aY9MhnPGTQDhR1EM5ga7e+/WP+cJI9rX3NcVXU38EbgLe1zdgR+0m5+9UDTX9CcIlvvXOANaQ+PkjxhotdQfxgEmi0+CyxPchnN0cGPxmlzMLAyyQ9pzuN/pKrW0rwxfj7JlTTBsMcwL1hVP6C5dnAJzTWDU6rqh8AfAZe0p2jeDrx3nKevAK5cf7F4jG/RzEv7nWqmX4RmnohrgB+kmbT8fzHFEXtbyxU0QzP/I83Ryfdorh+s9+/AsvUXi2mOHLZua1vVLqvn/PqoJPWcRwSS1HMGgST1nEEgST1nEEhSzxkEktRzBoEk9ZxBIEk99/8B7Vnr+UbLdmoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.ylabel('True Positive Rate')\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.title('ROC')\n",
    "plt.plot(false_positive_rate, true_positive_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}