3 Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, '../utils/')\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=FutureWarning)\n",
    "warnings.filterwarnings(\"ignore\", category=UserWarning)\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import util_michael as um\n",
    "from util import *\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import train_test_split\n",
    "import lightgbm as lgb\n",
    "import xgboost as xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((69, 57), (38, 57), (69,))"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('./data/fill_train.csv')\n",
    "test = pd.read_csv('./data/fill_test.csv')\n",
    "Ylabel = 'College_Persistence_Rate'\n",
    "\n",
    "Y = train[Ylabel]\n",
    "train.drop(columns=[Ylabel], inplace=True)\n",
    "train.shape, test.shape, Y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = train.drop(columns=['unique_id'])\n",
    "Xt = test.drop(columns=['unique_id'])\n",
    "\n",
    "X_scaled, Xt_scaled = um.apply_scale(X, Xt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EstimatorContainer():\n",
    "    def __init__(self, metric: str=''):\n",
    "        self.metric = metric\n",
    "        self.reset()\n",
    "        \n",
    "    def save(self, obj: dict) -> None:\n",
    "        name = type(obj['estimator']).__name__\n",
    "        if name in self.names:\n",
    "            ix = self.names.index(name)\n",
    "            self.names.pop(ix)\n",
    "            self.estimators.pop(ix)\n",
    "            self.Xs.pop(ix)\n",
    "            self.scores.pop(ix)\n",
    "        self.names.append(name)\n",
    "        self.estimators.append(obj['estimator'])\n",
    "        self.Xs.append(obj['X'])\n",
    "        self.scores.append(obj['score'])\n",
    "        \n",
    "    def get_df(self) -> pd.DataFrame:\n",
    "        self.df = pd.DataFrame()\n",
    "        self.df['Estimator'] = self.names\n",
    "        self.df[self.metric] = self.scores\n",
    "        return self.df\n",
    "        \n",
    "    def reset(self) -> None:\n",
    "        self.names = []\n",
    "        self.estimators = []\n",
    "        self.Xs = []\n",
    "        self.scores = []\n",
    "        \n",
    "estimators = EstimatorContainer('MSE')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ridge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters:\n",
      " {'alpha': 12.75, 'solver': 'svd'} \n",
      " -85.91270381437569 \n",
      "\n",
      "-85.91348064581788 {'alpha': 12.5, 'solver': 'auto'}\n",
      "-85.91348064581783 {'alpha': 12.5, 'solver': 'svd'}\n",
      "-85.91348064581788 {'alpha': 12.5, 'solver': 'cholesky'}\n",
      "-86.04793716946004 {'alpha': 12.5, 'solver': 'lsqr'}\n",
      "-86.02041389652803 {'alpha': 12.5, 'solver': 'sparse_cg'}\n",
      "-86.0837901945741 {'alpha': 12.5, 'solver': 'sag'}\n",
      "-86.11659065962623 {'alpha': 12.5, 'solver': 'saga'}\n",
      "-85.9127038143757 {'alpha': 12.75, 'solver': 'auto'}\n",
      "-85.91270381437569 {'alpha': 12.75, 'solver': 'svd'}\n",
      "-85.9127038143757 {'alpha': 12.75, 'solver': 'cholesky'}\n",
      "-86.03943370342851 {'alpha': 12.75, 'solver': 'lsqr'}\n",
      "-86.0394337111661 {'alpha': 12.75, 'solver': 'sparse_cg'}\n",
      "-86.0742749133118 {'alpha': 12.75, 'solver': 'sag'}\n",
      "-86.07525990117492 {'alpha': 12.75, 'solver': 'saga'}\n",
      "-85.91277733191797 {'alpha': 12.8, 'solver': 'auto'}\n",
      "-85.91277733191788 {'alpha': 12.8, 'solver': 'svd'}\n",
      "-85.91277733191797 {'alpha': 12.8, 'solver': 'cholesky'}\n",
      "-86.08432721187437 {'alpha': 12.8, 'solver': 'lsqr'}\n",
      "-86.08432720475126 {'alpha': 12.8, 'solver': 'sparse_cg'}\n",
      "-86.02727937121004 {'alpha': 12.8, 'solver': 'sag'}\n",
      "-86.12630882962921 {'alpha': 12.8, 'solver': 'saga'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/model_selection/_search.py:813: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import Ridge\n",
    "param_grid = [\n",
    "#     { 'solver': ['auto'], 'alpha': [12.5, 12.75, 12.8] },\n",
    "#     { 'solver': ['svd'], 'alpha': [12.5, 12.75, 12.8] },\n",
    "    { 'solver': ['auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga'], 'alpha': [12.5, 12.75, 12.8] }\n",
    "]\n",
    "model = Ridge()\n",
    "grid_search = GridSearchCV(model, param_grid, cv=10, scoring='neg_mean_squared_error', return_train_score=True)\n",
    "grid_search.fit(X_scaled, Y)\n",
    "show_grid_results(grid_search)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "aux = {\n",
    "    'estimator': grid_search.best_estimator_,\n",
    "    'X': X_scaled,\n",
    "    'score': grid_search.best_score_\n",
    "}\n",
    "estimators.save(aux)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lasso"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters:\n",
      " {'alpha': 1.3555} \n",
      " -87.44201088090824 \n",
      "\n",
      "-87.44344232164515 {'alpha': 1.35}\n",
      "-87.44224971257792 {'alpha': 1.355}\n",
      "-87.44201088090824 {'alpha': 1.3555}\n",
      "-87.44201280692893 {'alpha': 1.35555}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/model_selection/_search.py:813: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import Lasso\n",
    "param_grid = [\n",
    "    { 'alpha': [1.35, 1.355, 1.3555, 1.35555] }\n",
    "]\n",
    "model = Lasso()\n",
    "grid_search = GridSearchCV(model, param_grid, cv=10, scoring='neg_mean_squared_error', return_train_score=True)\n",
    "grid_search.fit(X_scaled, Y)\n",
    "show_grid_results(grid_search)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "aux = {\n",
    "    'estimator': grid_search.best_estimator_,\n",
    "    'X': X,\n",
    "    'score': grid_search.best_score_\n",
    "}\n",
    "estimators.save(aux)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters:\n",
      " {'hidden_layer_sizes': (40, 40), 'learning_rate_init': 0.005} \n",
      " -88.6711336892311 \n",
      "\n",
      "-101.37710053253483 {'hidden_layer_sizes': (30, 30), 'learning_rate_init': 0.005}\n",
      "-160.13333305075471 {'hidden_layer_sizes': (30, 30), 'learning_rate_init': 0.001}\n",
      "-88.6711336892311 {'hidden_layer_sizes': (40, 40), 'learning_rate_init': 0.005}\n",
      "-238.34178732600085 {'hidden_layer_sizes': (40, 40), 'learning_rate_init': 0.001}\n",
      "-101.36645092123045 {'hidden_layer_sizes': (30, 40), 'learning_rate_init': 0.005}\n",
      "-215.87637816130294 {'hidden_layer_sizes': (30, 40), 'learning_rate_init': 0.001}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/model_selection/_search.py:813: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.neural_network import MLPRegressor\n",
    "param_grid = [\n",
    "    # Relu is better than other activations\n",
    "    { 'hidden_layer_sizes': [ (30,30), (40, 40), (30, 40) ], 'learning_rate_init': [0.005, 0.001] }\n",
    "]\n",
    "model = MLPRegressor()\n",
    "grid_search = GridSearchCV(model, param_grid, cv=10, scoring='neg_mean_squared_error', return_train_score=True)\n",
    "grid_search.fit(X, Y)\n",
    "show_grid_results(grid_search)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "aux = {\n",
    "    'estimator': grid_search.best_estimator_,\n",
    "    'X': X,\n",
    "    'score': grid_search.best_score_\n",
    "}\n",
    "estimators.save(aux)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RandomForest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/model_selection/_search.py:813: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters:\n",
      " {'n_estimators': 305} \n",
      " -83.13582480632836 \n",
      "\n",
      "-83.13582480632836 {'n_estimators': 305}\n",
      "-83.6144696164924 {'n_estimators': 310}\n",
      "-83.8018791971109 {'n_estimators': 315}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "param_grid = [\n",
    "    { 'n_estimators': [305, 310, 315] }\n",
    "]\n",
    "model = RandomForestRegressor()\n",
    "grid_search = GridSearchCV(model, param_grid, cv=10, scoring='neg_mean_squared_error', return_train_score=True)\n",
    "grid_search.fit(X, Y)\n",
    "show_grid_results(grid_search)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "aux = {\n",
    "    'estimator': grid_search.best_estimator_,\n",
    "    'X': X,\n",
    "    'score': grid_search.best_score_\n",
    "}\n",
    "estimators.save(aux)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AdaBoost Regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/model_selection/_search.py:813: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters:\n",
      " {'learning_rate': 1, 'loss': 'exponential', 'n_estimators': 300} \n",
      " -89.05106469394772 \n",
      "\n",
      "-91.69176166887173 {'learning_rate': 0.8, 'loss': 'exponential', 'n_estimators': 300}\n",
      "-94.33850212047268 {'learning_rate': 0.8, 'loss': 'exponential', 'n_estimators': 320}\n",
      "-89.05106469394772 {'learning_rate': 1, 'loss': 'exponential', 'n_estimators': 300}\n",
      "-91.91590660902946 {'learning_rate': 1, 'loss': 'exponential', 'n_estimators': 320}\n",
      "-90.89424870359683 {'learning_rate': 1.2, 'loss': 'exponential', 'n_estimators': 300}\n",
      "-91.6959814772328 {'learning_rate': 1.2, 'loss': 'exponential', 'n_estimators': 320}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import AdaBoostRegressor\n",
    "param_grid = [\n",
    "#     { 'n_estimators': [350, 400], 'loss': ['linear', 'square', 'exponential'], 'learning_rate': [0.5, 1] }\n",
    "    { 'n_estimators': [300, 320], 'loss': ['exponential'], 'learning_rate': [0.8, 1, 1.2] }\n",
    "]\n",
    "model = AdaBoostRegressor()\n",
    "grid_search = GridSearchCV(model, param_grid, cv=10, scoring='neg_mean_squared_error', return_train_score=True)\n",
    "grid_search.fit(X, Y)\n",
    "show_grid_results(grid_search)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "aux = {\n",
    "    'estimator': grid_search.best_estimator_,\n",
    "    'X': X,\n",
    "    'score': grid_search.best_score_\n",
    "}\n",
    "estimators.save(aux)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "             colsample_bynode=1, colsample_bytree=1, gamma=0,\n",
       "             importance_type='gain', learning_rate=0.1, max_delta_step=0,\n",
       "             max_depth=3, min_child_weight=1, missing=None, n_estimators=100,\n",
       "             n_jobs=1, nthread=None, objective='reg:linear', random_state=0,\n",
       "             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
       "             silent=None, subsample=1, verbosity=1)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb.XGBRegressor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters:\n",
      " {'colsample_bytree': 0.25, 'eta': 0.05, 'n_estimators': 65, 'objective': 'reg:squarederror'} \n",
      " -79.07382221471613 \n",
      "\n",
      "-79.62863849768557 {'colsample_bytree': 0.25, 'eta': 0.05, 'n_estimators': 60, 'objective': 'reg:squarederror'}\n",
      "-79.07382221471613 {'colsample_bytree': 0.25, 'eta': 0.05, 'n_estimators': 65, 'objective': 'reg:squarederror'}\n",
      "-79.30281273210471 {'colsample_bytree': 0.25, 'eta': 0.05, 'n_estimators': 70, 'objective': 'reg:squarederror'}\n",
      "-79.62863849768557 {'colsample_bytree': 0.25, 'eta': 0.1, 'n_estimators': 60, 'objective': 'reg:squarederror'}\n",
      "-79.07382221471613 {'colsample_bytree': 0.25, 'eta': 0.1, 'n_estimators': 65, 'objective': 'reg:squarederror'}\n",
      "-79.30281273210471 {'colsample_bytree': 0.25, 'eta': 0.1, 'n_estimators': 70, 'objective': 'reg:squarederror'}\n",
      "-84.56614521615492 {'colsample_bytree': 0.3, 'eta': 0.05, 'n_estimators': 60, 'objective': 'reg:squarederror'}\n",
      "-84.27056743231799 {'colsample_bytree': 0.3, 'eta': 0.05, 'n_estimators': 65, 'objective': 'reg:squarederror'}\n",
      "-84.17918527627243 {'colsample_bytree': 0.3, 'eta': 0.05, 'n_estimators': 70, 'objective': 'reg:squarederror'}\n",
      "-84.56614521615492 {'colsample_bytree': 0.3, 'eta': 0.1, 'n_estimators': 60, 'objective': 'reg:squarederror'}\n",
      "-84.27056743231799 {'colsample_bytree': 0.3, 'eta': 0.1, 'n_estimators': 65, 'objective': 'reg:squarederror'}\n",
      "-84.17918527627243 {'colsample_bytree': 0.3, 'eta': 0.1, 'n_estimators': 70, 'objective': 'reg:squarederror'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/model_selection/_search.py:813: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "param_grid = [\n",
    "#     { 'n_estimators': [55], 'objective': ['reg:squarederror'] }\n",
    "#         { 'n_estimators': [50, 55, 60], 'colsample_bytree': [0.2], 'objective': ['reg:squarederror'] }\n",
    "    { 'n_estimators': [60, 65, 70],  'eta': [0.05, 0.1], \n",
    "      'colsample_bytree': [0.25, 0.3], 'objective': ['reg:squarederror'] }\n",
    "]\n",
    "model = xgb.XGBRegressor()\n",
    "grid_search = GridSearchCV(model, param_grid, cv=10, scoring='neg_mean_squared_error', return_train_score=True)\n",
    "grid_search.fit(X, Y)\n",
    "show_grid_results(grid_search)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "aux = {\n",
    "    'estimator': grid_search.best_estimator_,\n",
    "    'X': X,\n",
    "    'score': grid_search.best_score_\n",
    "}\n",
    "estimators.save(aux)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# VotingRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Ridge', Ridge(alpha=12.75, copy_X=True, fit_intercept=True, max_iter=None,\n",
       "        normalize=False, random_state=None, solver='svd', tol=0.001)),\n",
       " ('Lasso', Lasso(alpha=1.3555, copy_X=True, fit_intercept=True, max_iter=1000,\n",
       "        normalize=False, positive=False, precompute=False, random_state=None,\n",
       "        selection='cyclic', tol=0.0001, warm_start=False)),\n",
       " ('RandomForestRegressor',\n",
       "  RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n",
       "                        max_features='auto', max_leaf_nodes=None,\n",
       "                        min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                        min_samples_leaf=1, min_samples_split=2,\n",
       "                        min_weight_fraction_leaf=0.0, n_estimators=305,\n",
       "                        n_jobs=None, oob_score=False, random_state=None,\n",
       "                        verbose=0, warm_start=False)),\n",
       " ('XGBRegressor',\n",
       "  XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "               colsample_bynode=1, colsample_bytree=0.25, eta=0.05, gamma=0,\n",
       "               importance_type='gain', learning_rate=0.1, max_delta_step=0,\n",
       "               max_depth=3, min_child_weight=1, missing=None, n_estimators=65,\n",
       "               n_jobs=1, nthread=None, objective='reg:squarederror',\n",
       "               random_state=0, reg_alpha=0, reg_lambda=1, scale_pos_weight=1,\n",
       "               seed=None, silent=None, subsample=1, verbosity=1)),\n",
       " ('AdaBoostRegressor',\n",
       "  AdaBoostRegressor(base_estimator=None, learning_rate=1, loss='exponential',\n",
       "                    n_estimators=300, random_state=None)),\n",
       " ('MLPRegressor',\n",
       "  MLPRegressor(activation='relu', alpha=0.0001, batch_size='auto', beta_1=0.9,\n",
       "               beta_2=0.999, early_stopping=False, epsilon=1e-08,\n",
       "               hidden_layer_sizes=(40, 40), learning_rate='constant',\n",
       "               learning_rate_init=0.005, max_iter=200, momentum=0.9,\n",
       "               n_iter_no_change=10, nesterovs_momentum=True, power_t=0.5,\n",
       "               random_state=None, shuffle=True, solver='adam', tol=0.0001,\n",
       "               validation_fraction=0.1, verbose=False, warm_start=False))]"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "estis = list()\n",
    "skip = ['VotingRegressor']\n",
    "# skip = []\n",
    "for k, v in zip(estimators.names, estimators.estimators):\n",
    "#     print(k, ':', v)\n",
    "    if k not in skip:\n",
    "        estis.append((k, v))\n",
    "estis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/model_selection/_search.py:813: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters:\n",
      " {'weights': [0.15, 0.15, 0.2, 0.4, 0.1, 0]} \n",
      " -72.98273083930076 \n",
      "\n",
      "-72.98273083930076 {'weights': [0.15, 0.15, 0.2, 0.4, 0.1, 0]}\n",
      "-74.57169742776755 {'weights': [0.1, 0.1, 0.2, 0.5, 0.1, 0]}\n",
      "-73.33454943788806 {'weights': [0.1, 0.1, 0.2, 0.5, 0.05, 0.05]}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import VotingRegressor\n",
    "param_grid = [\n",
    "    { 'weights': [ [0.15, 0.15, 0.2, 0.4, 0.1, 0], [0.1, 0.1, 0.2, 0.5, 0.1, 0], [0.1, 0.1, 0.2, 0.5, 0.05, 0.05] ] }\n",
    "#     { 'weights': [ [0.1, 0.3, 0.1, 0.5], [0.125, 0.25, 0.125, 0.5], [0.05, 0.2, 0.15, 0.5] ] }\n",
    "]\n",
    "model = VotingRegressor(estimators=estis)\n",
    "grid_search = GridSearchCV(model, param_grid, cv=10, scoring='neg_mean_squared_error', return_train_score=True)\n",
    "grid_search.fit(X, Y)\n",
    "show_grid_results(grid_search)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "aux = {\n",
    "    'estimator': grid_search.best_estimator_,\n",
    "    'X': X,\n",
    "    'score': grid_search.best_score_\n",
    "}\n",
    "estimators.save(aux)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MSE of each Estimator\n",
    "\n",
    "## Our predictor College_Persistence_Rate is a value ranged from 0 to 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Estimator</th>\n",
       "      <th>MSE</th>\n",
       "      <th>STD</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>VotingRegressor</td>\n",
       "      <td>72.982731</td>\n",
       "      <td>8.542993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>XGBRegressor</td>\n",
       "      <td>79.073822</td>\n",
       "      <td>8.892346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>83.135825</td>\n",
       "      <td>9.117885</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Ridge</td>\n",
       "      <td>85.912704</td>\n",
       "      <td>9.268911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Lasso</td>\n",
       "      <td>87.442011</td>\n",
       "      <td>9.351043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>MLPRegressor</td>\n",
       "      <td>88.671134</td>\n",
       "      <td>9.416535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AdaBoostRegressor</td>\n",
       "      <td>89.051065</td>\n",
       "      <td>9.436687</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Estimator        MSE       STD\n",
       "6        VotingRegressor  72.982731  8.542993\n",
       "3           XGBRegressor  79.073822  8.892346\n",
       "2  RandomForestRegressor  83.135825  9.117885\n",
       "0                  Ridge  85.912704  9.268911\n",
       "1                  Lasso  87.442011  9.351043\n",
       "5           MLPRegressor  88.671134  9.416535\n",
       "4      AdaBoostRegressor  89.051065  9.436687"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = estimators.get_df()\n",
    "results['MSE'] = results['MSE'] * -1\n",
    "results['STD'] = np.sqrt( results['MSE'] )\n",
    "results.sort_values(by='MSE', ascending=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get best estimator and predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VotingRegressor(estimators=[('Ridge',\n",
       "                             Ridge(alpha=12.75, copy_X=True, fit_intercept=True,\n",
       "                                   max_iter=None, normalize=False,\n",
       "                                   random_state=None, solver='svd',\n",
       "                                   tol=0.001)),\n",
       "                            ('Lasso',\n",
       "                             Lasso(alpha=1.3555, copy_X=True,\n",
       "                                   fit_intercept=True, max_iter=1000,\n",
       "                                   normalize=False, positive=False,\n",
       "                                   precompute=False, random_state=None,\n",
       "                                   selection='cyclic', tol=0.0001,\n",
       "                                   warm_start=False)),\n",
       "                            ('R...\n",
       "                                          missing=None, n_estimators=65,\n",
       "                                          n_jobs=1, nthread=None,\n",
       "                                          objective='reg:squarederror',\n",
       "                                          random_state=0, reg_alpha=0,\n",
       "                                          reg_lambda=1, scale_pos_weight=1,\n",
       "                                          seed=None, silent=None, subsample=1,\n",
       "                                          verbosity=1)),\n",
       "                            ('AdaBoostRegressor',\n",
       "                             AdaBoostRegressor(base_estimator=None,\n",
       "                                               learning_rate=1,\n",
       "                                               loss='exponential',\n",
       "                                               n_estimators=300,\n",
       "                                               random_state=None))],\n",
       "                n_jobs=None, weights=[0.15, 0.15, 0.2, 0.4, 0.1])"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "estimators.estimators[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_model = estimators.estimators[2]\n",
    "Yt = best_model.predict(Xt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>unique_id</th>\n",
       "      <th>College_Persistence_Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>21</td>\n",
       "      <td>73.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100</td>\n",
       "      <td>68.92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>77</td>\n",
       "      <td>68.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>63</td>\n",
       "      <td>76.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>16</td>\n",
       "      <td>56.95</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   unique_id  College_Persistence_Rate\n",
       "0         21                     73.64\n",
       "1        100                     68.92\n",
       "2         77                     68.06\n",
       "3         63                     76.07\n",
       "4         16                     56.95"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output = pd.DataFrame()\n",
    "output['unique_id'] = test['unique_id']\n",
    "output[Ylabel] = Yt.astype(float)\n",
    "output[Ylabel] = output[Ylabel].apply(lambda x: round(x, 2))\n",
    "output.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "tnow = datetime.datetime.now()\n",
    "_result_file = 'results/result_'+tnow.strftime(\"%b-%d %H %M %S\")+'.csv'\n",
    "output.to_csv(_result_file, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}