PCA_DRAGON_noVPAS.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initialisations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-24T16:36:15.266255Z",
     "start_time": "2020-01-24T16:35:56.216083Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd ; pd.set_option('display.max_columns', 500) # dataframes\n",
    "import numpy as np # mathsy bits\n",
    "import ipywidgets as widgets # widgets\n",
    "\n",
    "from sklearn.model_selection import train_test_split #split data into train and test sets\n",
    "\n",
    "# feature selection + gridsearch\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# models\n",
    "from sklearn.linear_model import Lars, ElasticNet, Lasso\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor\n",
    "from sklearn.ensemble import AdaBoostRegressor, GradientBoostingRegressor\n",
    "\n",
    "import joblib # saving models\n",
    "from datetime import datetime # get time for labelling saved models\n",
    "\n",
    "# visualization\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import mpld3 # hover-over labels for plots\n",
    "mpld3.disable_notebook()\n",
    "\n",
    "train_scores = dict() ; test_scores  = dict() #to hold scores for viewing at the end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-24T16:36:15.304362Z",
     "start_time": "2020-01-24T16:36:15.273279Z"
    }
   },
   "outputs": [],
   "source": [
    "def grid_search_return_hp_dict(model, X_train_gs, y_train_gs, param_grid):\n",
    "    \"\"\"Run GridSearchCV on given model with defined param_grid, return best params as dict.\n",
    "    Pre-assign sklearn model to variable. Pass param_grid as dict.\"\"\"\n",
    "    print(str(model).split(sep='(')[0], \"grid search.\")\n",
    "    GS = GridSearchCV(model, param_grid, n_jobs=2)\n",
    "    GS.fit(X_train_gs, y_train_gs)\n",
    "    print(GS.best_params_)\n",
    "    return(GS.best_params_)\n",
    "\n",
    "def train_model(model, hyperparameters, \n",
    "                grid_search = False):\n",
    "    \"\"\"Wrapper for training models with optional grid search and feature selection.\n",
    "    Pass model with no brackets, ie Lars not Lars()\n",
    "    If grid_search = True, hyperparameters is the hp ranges dictionary. Outputs trained model, best_hp\n",
    "    If grid_search = False, hyperparameters is the hp dictionary. Outputs trained model\"\"\"\n",
    "    \n",
    "    if grid_search == True:\n",
    "        best_hp = grid_search_return_hp_dict(model(),\n",
    "                                             X_train, y_train, hyperparameters)\n",
    "        print(\"Grid search complete!\\nHyperparameters:\\n\", best_hp)\n",
    "        training_model = model(**best_hp).fit(X_train, y_train)\n",
    "        return training_model, best_hp\n",
    "\n",
    "    if grid_search == False:\n",
    "        print(\"No grid search\\nHyperparameters:\\n\", hyperparameters)\n",
    "        training_model = model(**hyperparameters).fit(X_train, y_train)\n",
    "        return training_model\n",
    "\n",
    "def produce_exp_vs_pred_df(model, codename): # generalise!\n",
    "    \n",
    "    pred_list = model.predict(pca_features_all)\n",
    "    \n",
    "    exp_vs_calc = pd.DataFrame(constants_first)\n",
    "    exp_vs_calc['Predicted'] = pred_list\n",
    "    exp_vs_calc.rename({'Kh_first':'Experimental'}, inplace=True, axis=1)\n",
    "    exp_vs_calc.to_csv('models/PCA_DRAGON_noVPAS/%s'%(\"expvcalc_\" + str(model).split(sep='(')[0] + \".csv\"))\n",
    "    now = datetime.now()\n",
    "\n",
    "    dt_string = now.strftime(\"_%d_%m_%Y_%H_%M_%S\")\n",
    "    filename = \"models/PCA_DRAGON_noVPAS/\" + str(model).split(sep='(')[0] + \"_\" + dt_string + \".joblib\"\n",
    "    print(str(model).split(sep='(')[0], \"run at:\", now, \". Saving to\", filename)\n",
    "    \n",
    "    joblib.dump(model, filename)\n",
    "    return exp_vs_calc\n",
    "\n",
    "def prediction_plot_scores(model_func, pred_df):\n",
    "    \"\"\"Print train+test scores, then plot scatter of predicted vs actual HLCs. Uses {X/y}_{train/test},\n",
    "    redefining these variables will change the output.\"\"\"\n",
    "    fig = plt.figure(figsize=(10, 10))\n",
    "    scatter = plt.scatter(pred_df['Experimental'], pred_df['Predicted'])\n",
    "    plt.xlabel('Experimental')\n",
    "    plt.ylabel('Predicted')\n",
    "    plt.title('%s predictions of HLCs' %str(model_func).split(sep='(')[0])\n",
    "    labels = ['{}'.format(i) for i in species_names]\n",
    "    tooltip = mpld3.plugins.PointLabelTooltip(scatter, labels=labels)\n",
    "    mpld3.plugins.connect(fig, tooltip)\n",
    "    plt.plot([-30, 10], [-30, 10], c='red')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DRAGON descriptors\n",
    "## Bringing in data and splitting it into parts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-24T16:36:16.082280Z",
     "start_time": "2020-01-24T16:36:15.307369Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input Shape (1404, 1479)\n",
      "Removed NaN, new shape (1355, 1479)\n"
     ]
    }
   ],
   "source": [
    "csv = pd.read_csv('organics_noVPAS_desc_kh.csv') # contains non-VP/AS HLCs\n",
    "\n",
    "print(\"Input Shape\", csv.shape) #input shape\n",
    "\n",
    "csv.dropna(axis=0, inplace=True)\n",
    "\n",
    "print(\"Removed NaN, new shape\", csv.shape) #removed NaN shape\n",
    "\n",
    "species_names = csv.pop('Unnamed: 0')\n",
    "smiles_strings = csv.pop('SMILES')\n",
    "constants_mean = csv.pop('Kh_mean')\n",
    "constants_first = csv.pop('Kh_first')\n",
    "\n",
    "varying_columns = csv[['Varying_1', 'Varying_2', 'Varying_3', 'Varying_4',\n",
    " 'Varying_5', 'Varying_6', 'Varying_7', 'Varying_8', 'Varying_9']] # popping one-hot encoding columns\n",
    "\n",
    "dragon_features = csv.drop(['Varying_1', 'Varying_2', 'Varying_3', 'Varying_4',\n",
    " 'Varying_5', 'Varying_6', 'Varying_7', 'Varying_8', 'Varying_9'], axis=1) #seperating features\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(dragon_features, constants_first, test_size=0.3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-24T16:36:17.627892Z",
     "start_time": "2020-01-24T16:36:16.085289Z"
    }
   },
   "outputs": [],
   "source": [
    "pca_standard_x_train = StandardScaler().fit_transform(X_train)\n",
    "pca = PCA(n_components=50)\n",
    "pca_fit = pca.fit(pca_standard_x_train)\n",
    "pca_components_train = pca_fit.transform(pca_standard_x_train)\n",
    "pca_features_train = pd.DataFrame(data = pca_components_train)\n",
    "X_train = pca_features_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-24T16:36:17.679999Z",
     "start_time": "2020-01-24T16:36:17.628864Z"
    }
   },
   "outputs": [],
   "source": [
    "pca_standard_x_test = StandardScaler().fit_transform(X_test)\n",
    "pca_components_test = pca_fit.transform(pca_standard_x_test)\n",
    "\n",
    "pca_features_test = pd.DataFrame(data = pca_components_test)\n",
    "X_test = pca_features_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-24T16:36:17.946708Z",
     "start_time": "2020-01-24T16:36:17.683009Z"
    }
   },
   "outputs": [],
   "source": [
    "pca_standard_x_all = StandardScaler().fit_transform(dragon_features)\n",
    "pca_components_all = pca_fit.transform(pca_standard_x_all)\n",
    "\n",
    "pca_features_all = pd.DataFrame(data = pca_components_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-24T16:36:18.248511Z",
     "start_time": "2020-01-24T16:36:17.948714Z"
    }
   },
   "outputs": [],
   "source": [
    "joblib.dump(pca_fit, \"models/PCA_DRAGON_noVPAS/pca_fit.joblib\")\n",
    "pca_features_all.to_csv(\"models/PCA_DRAGON_noVPAS/pca_all_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-24T16:36:18.430997Z",
     "start_time": "2020-01-24T16:36:18.253525Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:3: FutureWarning: The signature of `Series.to_csv` was aligned to that of `DataFrame.to_csv`, and argument 'header' will change its default value from False to True: please pass an explicit value to suppress this warning.\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    }
   ],
   "source": [
    "for i in ['X_train', 'X_test', 'y_train', 'y_test']:\n",
    "    #joblib.dump(exec('i'), \"models/DRAGON_noVPAS/%s.joblib\"%i)\n",
    "    eval('%s'%i).to_csv('models/PCA_DRAGON_noVPAS/%s.csv'%i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GradientBoosting\n",
    "### Model-chosen features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-24T16:36:18.440021Z",
     "start_time": "2020-01-24T16:36:18.435007Z"
    }
   },
   "outputs": [],
   "source": [
    "gbr_param_grid = {\n",
    "    'n_estimators': [500, 1000, 2000],\n",
    "    'max_depth': [2, 4, 6],\n",
    "    'min_samples_leaf': [3, 5, 9, 17],\n",
    "    'learning_rate': [0.1, 0.05, 0.02],\n",
    "    'max_features': [1.0, 0.3, 0.1],\n",
    "    'loss': ['ls', 'lad', 'huber']\n",
    "} #hyperparameters for each gbr grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-24T18:09:02.979896Z",
     "start_time": "2020-01-24T16:36:18.443028Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GradientBoostingRegressor grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'learning_rate': 0.02, 'loss': 'ls', 'max_depth': 4, 'max_features': 0.3, 'min_samples_leaf': 3, 'n_estimators': 2000}\n",
      "Grid search complete!\n",
      "Hyperparameters:\n",
      " {'learning_rate': 0.02, 'loss': 'ls', 'max_depth': 4, 'max_features': 0.3, 'min_samples_leaf': 3, 'n_estimators': 2000}\n",
      "GradientBoostingRegressor run at: 2020-01-24 18:09:02.790404 . Saving to models/PCA_DRAGON_noVPAS/GradientBoostingRegressor__24_01_2020_18_09_02.joblib\n"
     ]
    }
   ],
   "source": [
    "gbr_model, gbr_hp = train_model(model = GradientBoostingRegressor,\n",
    "            grid_search=True,\n",
    "           hyperparameters=gbr_param_grid)\n",
    "\n",
    "exp_vs_calc_gbr_model = produce_exp_vs_pred_df(gbr_model,\n",
    "                                         \"GradBoost\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decision Tree\n",
    "### Model-Chosen Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-24T18:09:02.993933Z",
     "start_time": "2020-01-24T18:09:02.982905Z"
    }
   },
   "outputs": [],
   "source": [
    "dtr_param_grid = {\n",
    "    'criterion': ['mse', 'friedman_mse', 'mae'], #function measuring quality of a split\n",
    "    'max_depth': [5, 10, 20] , # max depth of tree\n",
    "    'max_features': [5, 10, 15], # N features to be considered when looking for split\n",
    "    'max_leaf_nodes': [None, 10, 15, 20, 30], #grows tree with N nodes in best-first fashion\n",
    "    'min_impurity_decrease': [0.0, 0.1],\n",
    "    'min_samples_leaf': [1, 2, 3], \n",
    "    'min_samples_split': [1.0, 2, 3],\n",
    "    'min_weight_fraction_leaf': [0.0, 0.1, 0.5] , \n",
    "    'splitter': ['best', 'random']\n",
    "} #hyperparameters for dtr gridsearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-24T18:12:37.629441Z",
     "start_time": "2020-01-24T18:09:02.996943Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DecisionTreeRegressor grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'criterion': 'mae', 'max_depth': 20, 'max_features': 15, 'max_leaf_nodes': 30, 'min_impurity_decrease': 0.0, 'min_samples_leaf': 3, 'min_samples_split': 2, 'min_weight_fraction_leaf': 0.0, 'splitter': 'best'}\n",
      "Grid search complete!\n",
      "Hyperparameters:\n",
      " {'criterion': 'mae', 'max_depth': 20, 'max_features': 15, 'max_leaf_nodes': 30, 'min_impurity_decrease': 0.0, 'min_samples_leaf': 3, 'min_samples_split': 2, 'min_weight_fraction_leaf': 0.0, 'splitter': 'best'}\n",
      "DecisionTreeRegressor run at: 2020-01-24 18:12:37.623426 . Saving to models/PCA_DRAGON_noVPAS/DecisionTreeRegressor__24_01_2020_18_12_37.joblib\n"
     ]
    }
   ],
   "source": [
    "dtr_model, dtr_hp = train_model(model = DecisionTreeRegressor,\n",
    "            grid_search=True,\n",
    "           hyperparameters=dtr_param_grid)\n",
    "\n",
    "\n",
    "exp_vs_calc_dtr_model = produce_exp_vs_pred_df(dtr_model,\n",
    "                                         \"Decision Tree\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AdaBoost Regressor\n",
    "### Model Chosen Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-24T18:12:37.642477Z",
     "start_time": "2020-01-24T18:12:37.631448Z"
    }
   },
   "outputs": [],
   "source": [
    "base_models = [ExtraTreesRegressor(n_estimators= 5,\n",
    "                                   criterion= 'mse',\n",
    "                                   max_features = 'log2'),\n",
    "               RandomForestRegressor(n_estimators= 5,\n",
    "                                     criterion= 'mse',\n",
    "                                     max_features = 'sqrt',\n",
    "                                     min_samples_split = 3),\n",
    "              GradientBoostingRegressor(),\n",
    "              DecisionTreeRegressor(),\n",
    "              Lars(),\n",
    "              ElasticNet()]\n",
    "\n",
    "ada_param_grid = {\n",
    "    'base_estimator' : base_models, \n",
    "    'learning_rate' : [0.3, 0.5, 0.8, 1], \n",
    "    'loss' : ['linear', 'square', 'exponential'], \n",
    "    'n_estimators' : [50, 100]\n",
    "} #hyperparameters for adaboost gridsearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-24T18:26:51.770320Z",
     "start_time": "2020-01-24T18:12:37.644482Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AdaBoostRegressor grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'base_estimator': DecisionTreeRegressor(criterion='mse', max_depth=None, max_features=None,\n",
      "                      max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
      "                      min_impurity_split=None, min_samples_leaf=1,\n",
      "                      min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
      "                      presort=False, random_state=None, splitter='best'), 'learning_rate': 0.8, 'loss': 'square', 'n_estimators': 100}\n",
      "Grid search complete!\n",
      "Hyperparameters:\n",
      " {'base_estimator': DecisionTreeRegressor(criterion='mse', max_depth=None, max_features=None,\n",
      "                      max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
      "                      min_impurity_split=None, min_samples_leaf=1,\n",
      "                      min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
      "                      presort=False, random_state=None, splitter='best'), 'learning_rate': 0.8, 'loss': 'square', 'n_estimators': 100}\n",
      "AdaBoostRegressor run at: 2020-01-24 18:26:51.708151 . Saving to models/PCA_DRAGON_noVPAS/AdaBoostRegressor__24_01_2020_18_26_51.joblib\n"
     ]
    }
   ],
   "source": [
    "ada_model, ada_hp = train_model(model = AdaBoostRegressor,\n",
    "            grid_search=True,\n",
    "           hyperparameters=ada_param_grid)\n",
    "\n",
    "exp_vs_calc_ada_model = produce_exp_vs_pred_df(ada_model,\n",
    "                                               \"AdaBoost\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LASSO\n",
    "### Model Chosen Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-24T18:26:51.778342Z",
     "start_time": "2020-01-24T18:26:51.773329Z"
    }
   },
   "outputs": [],
   "source": [
    "lasso_param_grid = {\n",
    "    'alpha':[0.2, 0.4, 0.6, 0.8],\n",
    "    'max_iter':[1000, 5000, 10000, 50000],\n",
    "    'selection':['cyclic', 'random']\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-24T18:26:56.075148Z",
     "start_time": "2020-01-24T18:26:51.782352Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lasso grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'alpha': 0.4, 'max_iter': 1000, 'selection': 'random'}\n",
      "Grid search complete!\n",
      "Hyperparameters:\n",
      " {'alpha': 0.4, 'max_iter': 1000, 'selection': 'random'}\n",
      "Lasso run at: 2020-01-24 18:26:56.069130 . Saving to models/PCA_DRAGON_noVPAS/Lasso__24_01_2020_18_26_56.joblib\n"
     ]
    }
   ],
   "source": [
    "lasso_model, lasso_hp = train_model(model = Lasso,\n",
    "            grid_search=True,\n",
    "           hyperparameters=lasso_param_grid)\n",
    "\n",
    "\n",
    "exp_vs_calc_lasso_model = produce_exp_vs_pred_df(lasso_model,\n",
    "                                                 \"LASSO\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest Regressor\n",
    "### Model Chosen Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-24T18:26:56.087179Z",
     "start_time": "2020-01-24T18:26:56.077153Z"
    }
   },
   "outputs": [],
   "source": [
    "random_forest_param_grid = {\n",
    "    'max_depth': [10, 20, 30, 40, 50, 60, 70, 80, 90, None],\n",
    "    'max_features': ['auto', 'sqrt'],\n",
    "    'min_samples_leaf': [1, 2, 4],\n",
    "    'min_samples_split': [2, 5, 10],\n",
    "    'n_estimators': [200, 400, 600, 800, 1000, 1200, 1400, 1600]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-24T23:55:20.309717Z",
     "start_time": "2020-01-24T18:26:56.090187Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestRegressor grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'max_depth': 40, 'max_features': 'auto', 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 400}\n",
      "Grid search complete!\n",
      "Hyperparameters:\n",
      " {'max_depth': 40, 'max_features': 'auto', 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 400}\n",
      "RandomForestRegressor run at: 2020-01-24 23:55:20.050021 . Saving to models/PCA_DRAGON_noVPAS/RandomForestRegressor__24_01_2020_23_55_20.joblib\n"
     ]
    }
   ],
   "source": [
    "random_forest_model, rfr_hp = train_model(model = RandomForestRegressor,\n",
    "            grid_search=True,\n",
    "           hyperparameters=random_forest_param_grid)\n",
    "exp_vs_calc_random_forest_model = produce_exp_vs_pred_df(random_forest_model,\n",
    "                                                         \"RandomForest\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LARS - Least angle regression\n",
    "### Model chosen features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-24T23:55:20.322750Z",
     "start_time": "2020-01-24T23:55:20.317746Z"
    }
   },
   "outputs": [],
   "source": [
    "lars_param_grid = {'n_nonzero_coefs': [0, 5, 10, 15]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-24T23:55:28.926664Z",
     "start_time": "2020-01-24T23:55:20.327781Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lars grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'n_nonzero_coefs': 15}\n",
      "Grid search complete!\n",
      "Hyperparameters:\n",
      " {'n_nonzero_coefs': 15}\n",
      "Lars run at: 2020-01-24 23:55:28.919632 . Saving to models/PCA_DRAGON_noVPAS/Lars__24_01_2020_23_55_28.joblib\n"
     ]
    }
   ],
   "source": [
    "lars_model, lars_hp = train_model(model = Lars,\n",
    "            grid_search=True,\n",
    "           hyperparameters= lars_param_grid)\n",
    "exp_vs_calc_lars_model = produce_exp_vs_pred_df(lars_model, \"LARS\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Elastic Net (linear regression l1 l2 norm regularization)\n",
    "### Model chosen features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-24T23:55:28.936680Z",
     "start_time": "2020-01-24T23:55:28.931666Z"
    }
   },
   "outputs": [],
   "source": [
    "net_param_grid = {\n",
    "    'alpha':[0.1, 0.3, 0.5, 0.8, 1.0],\n",
    "    'l1_ratio':[0.0, 0.2, 0.4, 0.6, 0.8, 1.0],\n",
    "    'fit_intercept':[True, False],\n",
    "    'max_iter': [1000, 3000, 5000],\n",
    "    'tol':[0.0001, 0.001, 0.01],\n",
    "    'selection':['cyclic', 'random']\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-24T23:56:16.132188Z",
     "start_time": "2020-01-24T23:55:28.938685Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ElasticNet grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'alpha': 1.0, 'fit_intercept': True, 'l1_ratio': 0.2, 'max_iter': 1000, 'selection': 'random', 'tol': 0.01}\n",
      "Grid search complete!\n",
      "Hyperparameters:\n",
      " {'alpha': 1.0, 'fit_intercept': True, 'l1_ratio': 0.2, 'max_iter': 1000, 'selection': 'random', 'tol': 0.01}\n",
      "ElasticNet run at: 2020-01-24 23:56:16.125168 . Saving to models/PCA_DRAGON_noVPAS/ElasticNet__24_01_2020_23_56_16.joblib\n"
     ]
    }
   ],
   "source": [
    "net_model, net_hp = train_model(model = ElasticNet,\n",
    "            grid_search=True,\n",
    "           hyperparameters= net_param_grid)\n",
    "exp_vs_calc_net_model = produce_exp_vs_pred_df(net_model, \"ElasticNet\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "notify_time": "30",
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "303.516px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}