AML_Model_Tuning_Iteration-03_RobustScaler_FS-RFElog5.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Notes for the Project Report"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LOADING TRAIN AND TEST DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Train data.\n",
    "import pandas as pd\n",
    "data_train = pd.read_csv(\"FS_RFElog5_train_output.csv\")\n",
    "data_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split values into inpits and outputs.\n",
    "values_train = data_train.values\n",
    "X_train = values_train[:,1:6]\n",
    "y_train = values_train[:,0]\n",
    "\n",
    "data_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test data.\n",
    "data_test = pd.read_csv(\"FS_RFElog5_test_output.csv\")\n",
    "\n",
    "# split values into inpits and outputs.\n",
    "values_test = data_test.values\n",
    "X_test = values_test[:,1:6]\n",
    "y_test = values_test[:,0]\n",
    "\n",
    "data_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LOGISTIC REGRESSION"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LR with default hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initiate the LR model with defualt hyperparameters.\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Fit the model using default hyperparameters.\n",
    "# K, you don't split into train and validate sets??\n",
    "lr.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run predictions on TEST set and see the accuracy.\n",
    "lr_score = lr.score(X_test,y_test)\n",
    "print(lr_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Build confusion matrix.\n",
    "from sklearn.metrics import confusion_matrix\n",
    "lr_cm = confusion_matrix(y_test, lr.predict(X_test))\n",
    "print(lr_cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LR hyperparameters tuning (Random Search)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "\n",
    "# Create array of values for tuned hyperparameters.\n",
    "lr_params = {'C' : [0.01, 0.05, 0.1, 0.5, 1.0, 1.5, 2.0], \n",
    "             'max_iter' : [5, 10, 50, 100, 150, 200, 300]\n",
    "             }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Run random search and initiate the model with tuned parameters.\n",
    "lr_random = RandomizedSearchCV(estimator=lr, param_distributions=lr_params, cv = 3, n_jobs=-1, random_state = 2019)\n",
    "\n",
    "import time\n",
    "start_time = time.time()\n",
    "lr_random.fit(X_train, y_train)\n",
    "finish_time = time.time()\n",
    "\n",
    "# Summarize results\n",
    "print(\"Best: %f using %s\" % (lr_random.best_score_, lr_random.best_params_))\n",
    "print(\"Execution time: \" + str((finish_time - start_time)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Apply best values of hyperparameters to the model.\n",
    "lr_tuned = lr_random.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Train the tuned model on TRAIN set and check the accuracy\n",
    "lr_tuned.fit(X_train, y_train)\n",
    "lr_tuned_score = lr_tuned.score(X_test,y_test)\n",
    "print(lr_tuned_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build confusion matrix.\n",
    "from sklearn.metrics import confusion_matrix\n",
    "lr_tuned_cm = confusion_matrix(y_test, lr_tuned.predict(X_test))\n",
    "print(lr_tuned_cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LR tuning Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "print(\"LR default hyperparameters test accuracy: \", lr_score,', parameters: ', '\\n', lr.get_params())\n",
    "print('Confusion matrix: ', '\\n', lr_cm)\n",
    "print()\n",
    "print(\"LR tuned hyperparameters test accuracy: \", lr_tuned_score,', parameters: ', '\\n', lr_tuned.get_params())\n",
    "print('Confusion matrix: ', '\\n', lr_tuned_cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DECISION TREE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DT with default hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initiate a DT model using default hyperparameters.\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "dt = DecisionTreeClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Train model on train data.\n",
    "dt.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Check model accuracy on the TEST set.\n",
    "dt_score = dt.score(X_test, y_test)\n",
    "print(dt_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build confusion matrix.\n",
    "from sklearn.metrics import confusion_matrix\n",
    "dt_cm = confusion_matrix(y_test, dt.predict(X_test))\n",
    "print(dt_cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DT hyperparameters tuning (Random Search)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://medium.com/@mohtedibf/indepth-parameter-tuning-for-decision-tree-6753118a03c3\n",
    "# Create array of values for tuned hyperparameters.\n",
    "dt_params = {'max_depth': [None, 1, 3, 5, 10, 50, 100, 300], \n",
    "             'min_samples_split': [2, 5, 10, 50, 100], \n",
    "             'min_samples_leaf': [1, 2, 5, 10, 50, 100]\n",
    "            }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run random search.\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "dt_random = RandomizedSearchCV(estimator=dt, param_distributions=dt_params, cv = 10, n_jobs=-1, random_state = 2019)\n",
    "\n",
    "import time\n",
    "start_time = time.time()\n",
    "dt_random.fit(X_train, y_train)\n",
    "finish_time = time.time()\n",
    "\n",
    "# Summarize results\n",
    "print(\"Best: %f using %s\" % (dt_random.best_score_, dt_random.best_params_))\n",
    "print(\"Execution time: \" + str((finish_time - start_time)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Apply best values of hyperparameters to the model.\n",
    "dt_tuned = dt_random.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Train the tuned model on TRAIN set and check the accuracy\n",
    "dt_tuned.fit(X_train, y_train)\n",
    "dt_tuned_score = dt_tuned.score(X_test,y_test)\n",
    "print(dt_tuned_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build confusion matrix.\n",
    "from sklearn.metrics import confusion_matrix\n",
    "dt_tuned_cm = confusion_matrix(y_test, dt_tuned.predict(X_test))\n",
    "print(dt_tuned_cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DT tuning Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"DT default hyperparameters test accuracy: \", dt_score,', parameters: ', '\\n', dt.get_params())\n",
    "print('Confusion matrix: ', '\\n', dt_cm)\n",
    "print()\n",
    "print(\"DT tuned hyperparameters test accuracy: \", dt_tuned_score,', parameters: ', '\\n', dt_tuned.get_params())\n",
    "print('Confusion matrix: ', '\\n', dt_tuned_cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RANDOM FOREST"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RF with default hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initiate a RF model using default hyperparameters.\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rf = RandomForestClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Train model on train data.\n",
    "rf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check model accuracy on the TEST set.\n",
    "rf_score = rf.score(X_test, y_test)\n",
    "print(rf_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build confusion matrix.\n",
    "from sklearn.metrics import confusion_matrix\n",
    "rf_cm = confusion_matrix(y_test, rf.predict(X_test))\n",
    "print(rf_cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RF hyperparameters tuning (Random Search)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a grid of hyperparameters.\n",
    "rf_params = { 'n_estimators': [1, 5, 10, 30, 50, 100, 200, 500], \n",
    "             'max_depth': [None, 1, 2, 4, 8, 20, 50, 100], \n",
    "             'min_samples_leaf': [1, 5, 10, 50, 100], \n",
    "             'max_features': [None, 'auto', 'log2']\n",
    "            }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run random search.\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "rf_random = RandomizedSearchCV(estimator=rf, param_distributions=rf_params, cv = 3, n_jobs=-1, random_state = 2019)\n",
    "\n",
    "import time\n",
    "start_time = time.time()\n",
    "rf_random.fit(X_train, y_train)\n",
    "finish_time = time.time()\n",
    "\n",
    "# Summarize results\n",
    "print(\"Best: %f using %s\" % (rf_random.best_score_, rf_random.best_params_))\n",
    "print(\"Execution time: \" + str((finish_time - start_time)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Apply best values of hyperparameters to the model.\n",
    "rf_tuned = rf_random.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train the tuned model on TRAIN set and check the accuracy\n",
    "rf_tuned.fit(X_train, y_train)\n",
    "rf_tuned_score = rf_tuned.score(X_test,y_test)\n",
    "print(rf_tuned_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build confusion matrix.\n",
    "rf_tuned_cm = confusion_matrix(y_test, rf_tuned.predict(X_test))\n",
    "print(rf_tuned_cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RF tuning Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(\"RF default hyperparameters test accuracy: \", rf_score,', parameters: ', '\\n', rf.get_params())\n",
    "print('Confusion matrix: ', '\\n', rf_cm)\n",
    "print()\n",
    "print(\"RF tuned hyperparameters test accuracy: \", rf_tuned_score,', parameters: ', '\\n', rf_tuned.get_params())\n",
    "print('Confusion matrix: ', '\\n', rf_tuned_cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SVM (SVC)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SVC with default hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import svm\n",
    "svc = svm.SVC()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "svc.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "svc_score = svc.score(X_test, y_test)\n",
    "print(svc_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build confusion matrix.\n",
    "from sklearn.metrics import confusion_matrix\n",
    "svc_cm = confusion_matrix(y_test, svc.predict(X_test))\n",
    "print(svc_cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ## SVC hyperparameters tuning (Random Search)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a grid of hyperparameters.\n",
    "svc_params = { 'C': [0.1, 0.5, 1, 3, 5], \n",
    "             'gamma': ['scale', 'auto', 0.01, 0.1, 1, 10]\n",
    "            }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Run random search.\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "svc_random = RandomizedSearchCV(estimator=svc, n_iter=10, param_distributions=svc_params, cv = 3, n_jobs=-1, \n",
    "                                random_state = 2019)\n",
    "\n",
    "import time\n",
    "start_time = time.time()\n",
    "svc_random.fit(X_train, y_train)\n",
    "finish_time = time.time()\n",
    "\n",
    "# Summarize results\n",
    "print(\"Best: %f using %s\" % (svc_random.best_score_, svc_random.best_params_))\n",
    "print(\"Execution time: \" + str((finish_time - start_time)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Apply best values of hyperparameters to the model.\n",
    "svc_tuned = svc_random.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train the tuned model on TRAIN set and check the accuracy\n",
    "svc_tuned.fit(X_train, y_train)\n",
    "svc_tuned_score = svc_tuned.score(X_test,y_test)\n",
    "print(svc_tuned_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Build confusion matrix.\n",
    "from sklearn.metrics import confusion_matrix\n",
    "svc_tuned_cm = confusion_matrix(y_test, svc_tuned.predict(X_test))\n",
    "print(svc_tuned_cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SVC tuning Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(\"SVC default hyperparameters test accuracy: \", svc_score, \n",
    "      ', parameters: ', '\\n', svc.get_params())\n",
    "print('Confusion matrix: ', '\\n', svc_cm)\n",
    "print()\n",
    "print(\"SVC tuned hyperparameters test accuracy: \", svc_tuned_score, \n",
    "      ', parameters: ', '\\n', svc_tuned.get_params())\n",
    "print('Confusion matrix: ', '\\n', svc_tuned_cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# KNN classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## KNN with default hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn = KNeighborsClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "knn.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "knn_score = knn.score(X_test, y_test)\n",
    "print(knn_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build confusion matrix.\n",
    "from sklearn.metrics import confusion_matrix\n",
    "knn_cm = confusion_matrix(y_test, knn.predict(X_test))\n",
    "print(knn_cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## KNN hyperparameters tuning (Random Search)¶"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a grid of hyperparameters.\n",
    "knn_params = {'n_neighbors': [3, 5, 10, 20, 50, 100], \n",
    "              'weights': ['uniform', 'distance'], \n",
    "              'algorithm': ['auto', 'ball_tree', 'kd_tree', 'brute'], \n",
    "              'leaf_size': [10, 30, 50, 100], \n",
    "              'p': [1, 2]\n",
    "            }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run random search.\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "knn_random = RandomizedSearchCV(estimator=knn, n_iter=10, param_distributions=knn_params, cv = 3, n_jobs=-1, \n",
    "                                random_state = 2019)\n",
    "\n",
    "import time\n",
    "start_time = time.time()\n",
    "knn_random.fit(X_train, y_train)\n",
    "finish_time = time.time()\n",
    "\n",
    "# Summarize results\n",
    "print(\"Best: %f using %s\" % (knn_random.best_score_, knn_random.best_params_))\n",
    "print(\"Execution time: \" + str((finish_time - start_time)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Apply best values of hyperparameters to the model.\n",
    "knn_tuned = knn_random.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train the tuned model on TRAIN set and check the accuracy\n",
    "knn_tuned.fit(X_train, y_train)\n",
    "knn_tuned_score = knn_tuned.score(X_test,y_test)\n",
    "print(knn_tuned_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Build confusion matrix.\n",
    "from sklearn.metrics import confusion_matrix\n",
    "knn_tuned_cm = confusion_matrix(y_test, knn_tuned.predict(X_test))\n",
    "print(knn_tuned_cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## KNN tuning Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(\"KNN default hyperparameters test accuracy: \", knn_score, \n",
    "      ', parameters: ', '\\n', knn.get_params())\n",
    "print('Confusion matrix: ', '\\n', knn_cm)\n",
    "print()\n",
    "print(\"KNN tuned hyperparameters test accuracy: \", knn_tuned_score, \n",
    "      ', parameters: ', '\\n', knn_tuned.get_params())\n",
    "print('Confusion matrix: ', '\\n', knn_tuned_cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Naive Bayes Classifier (NBC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import Gaussian Naive Bayes model\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "nbc = GaussianNB()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fit the model and display score\n",
    "nbc.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run predictions on TEST set and see the accuracy.\n",
    "nbc_score = nbc.score(X_test,y_test)\n",
    "print(nbc_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Build confusion matrix.\n",
    "from sklearn.metrics import confusion_matrix\n",
    "nbc_cm = confusion_matrix(y_test, nbc.predict(X_test))\n",
    "print(nbc_cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NBC hyperparameters tuning (Random Search)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a grid of hyperparameters.\n",
    "nbc_params = {'var_smoothing': [1e-12, 1e-10, 1e-09, 1e-05, 1e-04, 1e-03, 1e-02, 1e-01, 1]\n",
    "            }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run random search.\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "nbc_random = RandomizedSearchCV(estimator=nbc, n_iter=10, param_distributions=nbc_params, cv = 3, n_jobs=-1, \n",
    "                                random_state = 2019)\n",
    "\n",
    "import time\n",
    "start_time = time.time()\n",
    "nbc_random.fit(X_train, y_train)\n",
    "finish_time = time.time()\n",
    "\n",
    "# Summarize results\n",
    "print(\"Best: %f using %s\" % (nbc_random.best_score_, nbc_random.best_params_))\n",
    "print(\"Execution time: \" + str((finish_time - start_time)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Apply best values of hyperparameters to the model.\n",
    "nbc_tuned = nbc_random.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train the tuned model on TRAIN set and check the accuracy\n",
    "nbc_tuned.fit(X_train, y_train)\n",
    "nbc_tuned_score = nbc_tuned.score(X_test,y_test)\n",
    "print(nbc_tuned_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Build confusion matrix.\n",
    "from sklearn.metrics import confusion_matrix\n",
    "nbc_tuned_cm = confusion_matrix(y_test, nbc_tuned.predict(X_test))\n",
    "print(nbc_tuned_cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NBC tuning Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(\"NBC default hyperparameters test accuracy: \", nbc_score, \n",
    "      ', parameters: ', '\\n', nbc.get_params())\n",
    "print('Confusion matrix: ', '\\n', nbc_cm)\n",
    "print()\n",
    "print(\"NBC tuned hyperparameters test accuracy: \", nbc_tuned_score, \n",
    "      ', parameters: ', '\\n', nbc_tuned.get_params())\n",
    "print('Confusion matrix: ', '\\n', nbc_tuned_cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compare Algorithms Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(\"LR tuned hyperparameters test accuracy: \", lr_tuned_score)\n",
    "print('Confusion matrix: ', '\\n', lr_tuned_cm, '\\n')\n",
    "\n",
    "print(\"DT tuned hyperparameters test accuracy: \", dt_tuned_score)\n",
    "print('Confusion matrix: ', '\\n', dt_tuned_cm, '\\n')\n",
    "\n",
    "print(\"RF tuned hyperparameters test accuracy: \", rf_tuned_score)\n",
    "print('Confusion matrix: ', '\\n', rf_tuned_cm, '\\n')\n",
    "\n",
    "print(\"SVC tuned hyperparameters test accuracy: \", svc_tuned_score)\n",
    "print('Confusion matrix: ', '\\n', svc_tuned_cm, '\\n')\n",
    "\n",
    "print(\"KNN tuned hyperparameters test accuracy: \", knn_tuned_score)\n",
    "print('Confusion matrix: ', '\\n', knn_tuned_cm, '\\n')\n",
    "\n",
    "print(\"NBC tuned hyperparameters test accuracy: \", nbc_tuned_score)\n",
    "print('Confusion matrix: ', '\\n', nbc_tuned_cm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}