RDKit_VPAS.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initialisations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T10:10:29.676057Z",
     "start_time": "2020-01-29T10:10:08.282050Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd ; pd.set_option('display.max_columns', 500) # dataframes\n",
    "import numpy as np # mathsy bits\n",
    "import ipywidgets as widgets # widgets\n",
    "\n",
    "from sklearn.model_selection import train_test_split #split data into train and test sets\n",
    "\n",
    "# feature selection + gridsearch\n",
    "from sklearn.feature_selection import SelectFromModel, SelectKBest\n",
    "from sklearn.feature_selection import f_regression, mutual_info_regression\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# models\n",
    "from sklearn.linear_model import Lars, ElasticNet, Lasso\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor\n",
    "from sklearn.ensemble import AdaBoostRegressor, GradientBoostingRegressor\n",
    "\n",
    "import joblib # saving models\n",
    "from datetime import datetime # get time for labelling saved models\n",
    "\n",
    "# visualization\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import mpld3 # hover-over labels for plots\n",
    "mpld3.disable_notebook()\n",
    "\n",
    "train_scores = dict() ; test_scores  = dict() #to hold scores for viewing at the end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T10:10:29.704972Z",
     "start_time": "2020-01-29T10:10:29.677898Z"
    }
   },
   "outputs": [],
   "source": [
    "def grid_search_return_hp_dict(model, X_train_gs, y_train_gs, param_grid):\n",
    "    \"\"\"Run GridSearchCV on given model with defined param_grid, return best params as dict.\n",
    "    Pre-assign sklearn model to variable. Pass param_grid as dict.\"\"\"\n",
    "    print(str(model).split(sep='(')[0], \"grid search.\")\n",
    "    GS = GridSearchCV(model, param_grid, n_jobs=2)\n",
    "    GS.fit(X_train_gs, y_train_gs)\n",
    "    print(GS.best_params_)\n",
    "    return(GS.best_params_)\n",
    "\n",
    "def train_model(model, feature_set, hyperparameters, \n",
    "                grid_search = False):\n",
    "    \"\"\"Wrapper for training models with optional grid search and feature selection.\n",
    "    Pass model with no brackets, ie Lars not Lars()\n",
    "    If grid_search = True, hyperparameters is the hp ranges dictionary. Outputs trained model, best_hp\n",
    "    If grid_search = False, hyperparameters is the hp dictionary. Outputs trained model\"\"\"\n",
    "    X_train, X_test = X_train_main[feature_set], X_test_main[feature_set]\n",
    "    \n",
    "    if grid_search == True:\n",
    "        best_hp = grid_search_return_hp_dict(model(),\n",
    "                                             X_train, y_train, hyperparameters)\n",
    "        print(\"Grid search complete!\\nHyperparameters:\\n\", best_hp)\n",
    "        training_model = model(**best_hp).fit(X_train, y_train)\n",
    "        return training_model, best_hp\n",
    "\n",
    "    if grid_search == False:\n",
    "        print(\"No grid search\\nHyperparameters:\\n\", hyperparameters)\n",
    "        training_model = model(**hyperparameters).fit(X_train, y_train)\n",
    "        return training_model\n",
    "\n",
    "def produce_exp_vs_pred_df(features_list, model, codename): # generalise!\n",
    "    \"\"\"Pass features_list as a string\"\"\"\n",
    "    pred_list = []\n",
    "    for i in range(0, len(rdkit_features)):\n",
    "        prediction = model.predict([rdkit_features[eval(features_list)].iloc[i]])\n",
    "        pred_list.append(prediction)\n",
    "    \n",
    "    exp_vs_calc = pd.DataFrame(constants_first)\n",
    "    exp_vs_calc['Predicted'] = pred_list\n",
    "    exp_vs_calc.rename({'Kh_first':'Experimental'}, inplace=True, axis=1)\n",
    "    exp_vs_calc.to_csv('models/RDKit_VPAS/%s'%(\"expvcalc_\" + str(model).split(sep='(')[0] + \"_\" + features_list[16:] + \".csv\"))\n",
    "    now = datetime.now()\n",
    "\n",
    "    dt_string = now.strftime(\"_%d_%m_%Y_%H_%M_%S\")\n",
    "    filename = \"models/RDKit_VPAS/\" + str(model).split(sep='(')[0] + \"_\" + features_list[16:] + dt_string + \".joblib\"\n",
    "    print(str(model).split(sep='(')[0], \"run at:\", now, \". Saving to\", filename)\n",
    "    \n",
    "    joblib.dump(model, filename)\n",
    "    return exp_vs_calc\n",
    "\n",
    "def prediction_plot_scores(model_func, pred_df):\n",
    "    \"\"\"Print train+test scores, then plot scatter of predicted vs actual HLCs. Uses {X/y}_{train/test},\n",
    "    redefining these variables will change the output.\"\"\"\n",
    "    fig = plt.figure(figsize=(10, 10))\n",
    "    scatter = plt.scatter(pred_df['Experimental'], pred_df['Predicted'])\n",
    "    plt.xlabel('Experimental')\n",
    "    plt.ylabel('Predicted')\n",
    "    plt.title('%s predictions of HLCs' %str(model_func).split(sep='(')[0])\n",
    "    labels = ['{}'.format(i) for i in species_names]\n",
    "    tooltip = mpld3.plugins.PointLabelTooltip(scatter, labels=labels)\n",
    "    mpld3.plugins.connect(fig, tooltip)\n",
    "    plt.plot([-30, 10], [-30, 10], c='red')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RDkit descriptors\n",
    "## Bringing in data and splitting it into parts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T10:10:29.957156Z",
     "start_time": "2020-01-29T10:10:29.707980Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input Shape (2196, 34)\n",
      "Removed NaN, new shape (2195, 34)\n"
     ]
    }
   ],
   "source": [
    "csv = pd.read_csv('rdkit_desc_VPAS.csv') # contains non-VP/AS HLCs\n",
    "\n",
    "print(\"Input Shape\", csv.shape) #input shape\n",
    "\n",
    "csv.dropna(axis=0, inplace=True)\n",
    "\n",
    "print(\"Removed NaN, new shape\", csv.shape) #removed NaN shape\n",
    "\n",
    "species_names = csv.pop('0')\n",
    "smiles_strings = csv.pop('SMILES')\n",
    "constants_mean = csv.pop('Kh_mean')\n",
    "constants_first = csv.pop('Kh_first')\n",
    "\n",
    "varying_columns = csv[['Varying_1', 'Varying_2', 'Varying_3', 'Varying_4',\n",
    " 'Varying_5', 'Varying_6', 'Varying_7', 'Varying_8', 'Varying_9']] # popping one-hot encoding columns\n",
    "\n",
    "rdkit_features = csv.drop(['Varying_1', 'Varying_2', 'Varying_3', 'Varying_4',\n",
    " 'Varying_5', 'Varying_6', 'Varying_7', 'Varying_8', 'Varying_9'], axis=1) #seperating features\n",
    "\n",
    "X_train_main, X_test_main, y_train, y_test = train_test_split(rdkit_features, constants_first, test_size=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T10:10:30.176717Z",
     "start_time": "2020-01-29T10:10:29.960164Z"
    },
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:3: FutureWarning: The signature of `Series.to_csv` was aligned to that of `DataFrame.to_csv`, and argument 'header' will change its default value from False to True: please pass an explicit value to suppress this warning.\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    }
   ],
   "source": [
    "for i in ['X_train_main', 'X_test_main', 'y_train', 'y_test']:\n",
    "    #joblib.dump(exec('i'), \"models/RDKit_VPAS/%s.joblib\"%i)\n",
    "    eval('%s'%i).to_csv('models/RDKit_VPAS/%s.csv'%i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Selection\n",
    "Made aware of features selection methods in scikitlearn via https://towardsdatascience.com/feature-selection-techniques-in-machine-learning-with-python-f24e7da3f36e\n",
    "\n",
    "See also [scikit-learn docs](https://scikit-learn.org/stable/modules/feature_selection.html)\n",
    "### Importances via 'dummy' GradientBoostingRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T10:10:30.480534Z",
     "start_time": "2020-01-29T10:10:30.178723Z"
    }
   },
   "outputs": [],
   "source": [
    "fs_model = GradientBoostingRegressor()\n",
    "fs_model.fit(X_train_main, y_train)\n",
    "importances = pd.Series(fs_model.feature_importances_, index = X_train_main.columns)\n",
    "top_15_features_model = list(importances.nlargest(15).index.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importances via 'dummy' LASSO model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T10:10:30.529655Z",
     "start_time": "2020-01-29T10:10:30.482533Z"
    }
   },
   "outputs": [],
   "source": [
    "dummy_lasso = Lasso(max_iter=10000, alpha=0.5)\n",
    "selector = SelectFromModel(dummy_lasso).fit(X_train_main, y_train)\n",
    "coefs = selector.estimator_.coef_\n",
    "importances_lasso = pd.Series(coefs, index = X_train_main.columns)\n",
    "top_15_features_lasso = list(importances_lasso.nlargest(15).index.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SelectKBest using f_regression scoring function\n",
    "F-test estimates degree of linear dependency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T10:10:30.777249Z",
     "start_time": "2020-01-29T10:10:30.531661Z"
    }
   },
   "outputs": [],
   "source": [
    "fs_kbest = SelectKBest(score_func= f_regression, k=15)\n",
    "fit = fs_kbest.fit(X_train_main, y_train)\n",
    "scores_f = pd.Series(fit.scores_, index=X_train_main.columns)\n",
    "top_15_features_kbest = list(scores_f.nlargest(15).index.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SelectKBest using mutual_info_regression\n",
    "Catch any statistical dependancy, but requires more samples than f_reg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T10:10:31.560251Z",
     "start_time": "2020-01-29T10:10:30.780230Z"
    }
   },
   "outputs": [],
   "source": [
    "fs_mut_info = SelectKBest(score_func= mutual_info_regression, k=15)\n",
    "fit = fs_mut_info.fit(X_train_main, y_train)\n",
    "scores_mutual = pd.Series(fit.scores_, index=X_train_main.columns)\n",
    "top_15_features_mutual_info = list(scores_mutual.nlargest(15).index.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Selection results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T10:10:32.367289Z",
     "start_time": "2020-01-29T10:10:31.563257Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7UAAAJOCAYAAABoeyBbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdeZgdZZn38e+PsC8CCio4anRGQUhCgLA5CJFNFHVAUQaRRXQUHYdxXvdREB0XmHFccGccDAIqI4syooIggQHCkkAgrDpqEAXHsBjZJeF+/6hqPGm6O+lObyf9/VxXX13nqaeq7tNX36fqrnqqTqoKSZIkSZK60WpjHYAkSZIkSUNlUStJkiRJ6loWtZIkSZKkrmVRK0mSJEnqWha1kiRJkqSuZVErSZIkSepaFrXSKEkyO8lbVrBvJfmrkY5J6pRkcvu/t/owrOsZSS5Ncn+Sfx+O+IZTkpckuW2s45DGoyQ3JZk51nFI0oqyqNWwSLIwycPtAewfklyR5KgkXfc/luS49sD+6F7t72rbjxuj0LQKaXPmT0k26dU+v/0/m7yC6xm1EyBtzHutYPe3AncDT6mqd49gWCuk99+pqv6nqrYYy5jUXTr2cw90/GzeR7+ZSR5v59+f5LYkbxqLmIeqqrauqtljHYe0sgbabyV5XpurX+5j3t+0++M/Jrk7yUU9++UkGyU5Ocnv2hz/WZL3dyybJO9N8vP2M+PXSY5PstZIvU9Z1Gp4vaqqNgCeCxwPvB/4z7ENach+Bhzeq+2wtl0aLr8CDu55kWQqsM7YhTOsngvcXFU12AWH40qxNEJeVVXrd/zc2U+/O6tqfeApwD8B/5Fk2E+imCvSSjkMuA/4286Csz0B+k3g3cCGwPOALwOPt10+C6wPvKid/2rgFx3rPZHmxO5hwAbAy4E9gP8awfcy4VnUathV1eKqOhc4CDg8yRR48vDbJEckuazjdSV5R3tm6/4k/5LkL5PMac+U/VeSNdu+M5P8Jsn7kvw+yV1J9k/yivaM2b1J/rnt+8wkDyV5Wse2tk+yKMka/byNa4B1k2zd9t+apti4prNTkr9L8r/t9s7tPGufZO8ktyZZnOSLQHote2SSW5Lcl+T8JM8dwp9b3e1Ump1ej8NpdqRPGChvklzaNl/fXhU6qHdetf2euEqZZL8k17U5dcdQRx70bCfJp9v/4V8leXk7b1b7Xt7XxrVXkrWSfC7Jne3P53oOIjry+f1Jfgd8YzA53q5jx/az4g9t3y92fF709XeameQ3Hcu/qP1b/yHN0MtXd8ybleRLSc5rP5uuSvKXQ/m7aWKpxg+Be4FpPe1Jtkzyk/b/+LYkr++Y97Qk/93m6DVJPt7HvvLvk/wc+PkKrO8VSW5u/3d/m+Q9bfsmSX7Q/s/fm+R/0o6uSsfVrRXM3Xd35GlXXZXWhHYY8GHgMeBVHe3TgV9V1UVtDt9fVWdV1a/b+TsA36qq+6rq8aq6tarOBEjyAuAdwCFVNaeqllTVTcBrgX2T7DFq726CsajViKmqq4HfAC8ZxGL7AtsDOwPvA04CDgGeDUyh46oW8ExgbeBZwLHAfwBvbJd/CXBskudX1e+A2cDrO5Z9I/CdqnpsgFg6C46+io09gE+1690MuB34TjtvE+Asmg/LTWjO4P11x7L7A/8MvAbYFPgf4NsDxKJV05XAU9qCahLNiaDTVnThqtqtndymvWp0xgos9iDN//VGwH7A29v/x6HYCbiN5n/8X4H/TJKqOgI4HfjXNq4LgQ/R5PV0YBtgR5r86PFM4Kk0V3jf2tG23Bxv+y6luSK2CbALsCfNgcVy/05pTm79N3AB8HTgH4DTs+yVtYOBjwIbA/8LfGKQfytNQElWa0+QbELzf0OS9YCfAN+i+X87GPhy2pOowJdo8vSZNPue3qOGAPanyb+tVmB9/wm8rR1JNQX4adv+bpp99KbAM2j2SX2NrFiR3N2QJk/fDHwpycYr8OeRxkySlwB/QXPc9l8se4L5WmDLJJ9N8tIk6/da/ErgE0ne1BaxnfYEftMeAz+hqu5ol9t7ON+H/syiViPtTpoD1RV1QlX9sT2rdSNwQVX9sqoWAz8Ctu3o+xjwibYw/Q7NQcPn2zNqNwE38ecz46fQHAzTFg8H0xStAzkNOLg94P1bnlxsHAKcXFXXVtWjwAeBXdLcc/EKmqGXZ7bxfQ74XceybwM+VVW3VNUS4JPA9Hi1diLqOXmyN3Ar8NuR3FhVza6qBe3Z5RtoTqbsPsTV3V5V/1FVS2lybDOag+O+HAJ8rKp+X1WLaArEQzvmPw58pKoeraqH27YVzvGqmldVV7ZnxRcCXxvE+9qZZijZ8VX1p6r6KfADlj2JdnZVXd3m6+k0B/iaeL7XXtn8Q5LvDdBv8yR/AB4GzgH+X1Vd1857JbCwqr7R/r9eS3MS9MB2//Ramlx4qKpupsmt3j5VVfe2udLv+tq+j9EUv09pryxd29G+GfDcqnqsvc+8r6J2ebn7WDv/sfaq9AOA96trvDsc+FFV3UdzQujlSZ4OUFW/BGbSnKj5L+DudsROT3H7DzT7gXcCN6cZsffydt4mwF39bPOudr5GgEWtRtqzaIZdraj/65h+uI/XnWfL7mkPpnvm9bV8T//v0+zUn09TPCzufRatt3aYyf/SFJw/b8+yddqc5upsT/8HgHto3vPmwB0d86rzNc3VqM/3HBzR/I3SLquJ5VTgDcAR9BoNMBKS7JTk4jTD7xcDRzH0newTJ2qq6qF2svcZ7R7L5Es73fmQnUVV9UivZVY4x5O8sB1K+bskf6TJ2xV9X5sDd1TV4x1tt7NsPnaelHqI/t+nVm37V9VG7c9AIxzurKqNaO6pPZHmfroezwV26iiO/0BTOD6T5qrp6iy7v+i97+ndNtD6oCmSXwHcnuSSJLu07f9Gs4+7IMkvk3ygn/eyvNy9pz3Z08P80LiWZB3gdTSFKVU1B/g1zb6Ytu3Kqnp9VW1KMzJoN5pRC1TVw1X1yaraHngaTeH73SRPpXlA4mb9bHqzdr5GgEWtRkySHWgOCnvuBXoQWLejyzOftNAIaQ+W/4tmR38oy79K26PnQQF9FRt30hxMAE8MKXsazZW2u2iGTPfMS+drmgOSt3UcHG1UVetU1RUr/q60Kqiq22keGPUK4Ow+ugw2b5bpn6R3/28B5wLPrqoNga/S637vEbJMvgDPadt6DPqBUr18heZK9wuq6ik0QylX9H3dCTw7yz6t/TmM8FVzrfraUTzvB6Z2DPO/A7ik1+f/+lX1dmARsIRmWGSPZ/Nknfky0Pqoqmuq6m9ohiZ/j/ZhNe2Ih3dX1fNp7if8f0n27GNby8tdqdscQHPC6cvtidDf0RyvHtZX56q6hmb/PKWPeT0nUdejeaDUT2n2Jzt29kvybJpRQRcN4/tQB4taDbskT0nySprhgqdV1YJ21nzgNUnWTfPQmjePcmjfpLka9mpW/L7FM4B96PuJdd8C3pRkevvQjE8CV7VDH88Dtk7ymjRPpzyaZYuRrwIfzJ8fRLVhktcN/i1pFfFmYI+qerCPecvLm/8Dnt/x+nqa/73pSdYGjuvVfwPg3qp6pN3pvoHR8W3gw0k2be85P5ZB3D+8AjYA/gg8kGRL4O295vf+O3W6iuZkwPuSrJHm+zlfRXuPvLQyqupPwL/T/M9DM7T9hUkObf/f1kiyQ5IXtSMTzgaOa3N+S/o50O7Q7/qSrJnkkCQbtsP4/0hz/zlJXpnkr9qTrj3tS/tY/0jnrjTS1kiyds8PzX70ZGAqza0k02meezI9ydQku6Z5EOjToXkQG82x45Xt62PaHFuzXd8/An8Abquqn9Ec452eZOckk9pjvbOAC9tnTGgEWNRqOP13kvtpzhp/CPgM0PkUxM8Cf6I5uDyFdtjHaKmqy2nu27u2LTxXZJmHq+rCjnv8OuddBBxD80F1F/CXNPfeUlV30wxtOZ5mSPILgMs7lj0HOAH4TjtU8kaaR75rAqqqX1TV3H5mLy9vjgNOaYcdvr7doX4MuJDmyaiX9er/DuBjba4ey+h9xcDHgbnADcACmgdxfHwY1/8emgL9fpoHSvV+aNZxdPydOme0RceraXLwbpqvbjisqm4dxvg0sZ0MPCfJq6rqfpqTpX9Lc8XzdzT7g56vFHknzYOXfkczqujbwKP9rXgF1ncosLDd1xxF+3wJmv3ShTT3wM4Bvlx9fzftSOeuNNJ+SHO7Ss/P7sDnqup3HT/zgB/T3Gv7B5p9woIkD7Tt59A8EBGakRLfoNlf3ElzW9t+7W1o0OTw12lO/vQsP5vmVgCNkPT9TABp1ZTkpzSPYf/6WMciSdLyJDkBeGZV9fUUZEkSXqnVBNLe47sdT76CI0nSuJDmO2enpbEjzVDJc8Y6Lkkaz1Yf6wCk0ZDkFJrv9fvHdqiWJEnj0QY0Q443B35Pcz/u98c0Ikka5xx+LEmSJEnqWg4/liRJkiR1LYcfjxObbLJJTZ48eazDkIbVvHnz7m6/uHxcMM+0KhpveQbmmlZN4y3XzDOtioaaZxa148TkyZOZO7e/b/SQulOS28c6hk7mmVZF4y3PwFzTqmm85Zp5plXRUPPM4ceSJEmSpK5lUStJkiRJ6loWtZIkSZKkrmVRK0mSJEnqWha1kiRJkqSuZVErSZIkSepaFrWSJEmSpK5lUStJkiRJ6loWtZIkSZKkrmVRK0mSJEnqWha1kiRJkqSuZVErSZIkSepaFrWSJEmSpK5lUStJkiRJ6loWtZIkSZKkrmVRK0mSJEnqWha1kiRJkqSuZVErSZIkSepaFrWSJEmSpK5lUStJkiRJ6loWtZIkSZKkrmVRK0mSJEnqWha1kiRJkqSutfpYB6DGgt8uZvIHzmPh8fuNdSjSKqsnz1aWeSoNrDPXzBdJ0kjzSq0kSZIkqWtZ1EqSJEmSupZFrSRJkiSpa1nUSpIkSZK6lkWtJEmSJKlrWdRKkiRJkrqWRa0kSZIkqWtZ1EqSJEmSupZFrSRJkiSpa1nUjoIkX0+y1VjHIUmSJEmrGovaUVBVb6mqm8c6Dk0skydP5u67717h/vPnz+eHP/zhCEbUtyRXjPpGpRE02NwbDUk+lmSvsY5DmsjS8NhbGgEm1jBLsl6S85Jcn+TGJAclmZ1kxljHpu62dOnSEV3/UIraJUuWDHl7SSYBVNWLh7wSaRSMdO6NtCSTqurYqrpwrGORJpokk5PckuTLwLXAoUkWtMeIJ3T0e3OSn7XHjP+R5ItjF7XUfSxqh9++wJ1VtU1VTQF+3F/HJG9NMjfJ3KUPLR69CDUu7b///my//fZsvfXWnHTSSQCsv/76HHvssey0007MmTOHiy66iG233ZapU6dy5JFH8uijjw64zi984Qtst912TJ06lVtvvRWABx98kCOPPJIddtiBbbfdlu9///v86U9/4thjj+WMM85g+vTpnHHGGX32A5g1axave93reNWrXsU+++xDVfHe976XKVOmMHXqVM4444x+40kyM8nFSb4FLGjbHuiYNzvJmUluTXJ6krTzXtG2XZbkxCQ/aNt3TzK//bkuyQZ9bNM804BGK/fuvfde9t9/f6ZNm8bOO+/MDTfcAMBxxx3H4Ycfzj777MPkyZM5++yzed/73sfUqVPZd999eeyxxwAGFUOShUmOTXIZ8Loks5Ic2DHvo0mubQ+ut2zbN03yk7b9a0luT7JJXydr+9mmuSb1bQvgm8B+wL8AewDTgR2S7J9kc+AYYGdgb2DL/lbUmWeLFi0a+cilLmFRO/wWAHslOSHJS6qq3z17VZ1UVTOqasakdTccxRA1Hp188snMmzePuXPncuKJJ3LPPffw4IMPMmXKFK666ipmzJjBEUccwRlnnMGCBQtYsmQJX/nKVwZc5yabbMK1117L29/+dj796U8D8IlPfII99tiDa665hosvvpj3vve9PPbYY3zsYx/joIMOYv78+Rx00EF99nvwwQcBmDNnDqeccgo//elPOfvss5k/fz7XX389F154Ie9973u56667BgprR+BDVdXXfebbAu8CtgKeD/x1krWBrwEvr6pdgU07+r8H+Puqmg68BHi49wrNMy3PaOXeRz7yEbbddltuuOEGPvnJT3LYYYc90f8Xv/gF5513Ht///vd54xvfyEtf+lIWLFjAOuusw3nnnccjjzwy6BiAR6pq16r6Th/z7q6q7YCv0OQRwEeAn7bt5wDPadtX6GStuSb16/aquhLYAZhdVYuqaglwOrAbzX7xkqq6t6oeA77b34o682zTTTftr5s04VjUDrOq+hmwPU1x+6kkx45xSOoSJ554Ittssw0777wzd9xxBz//+c+ZNGkSr33tawG47bbbeN7znscLX/hCAA4//HAuvfTSAdf5mte8BoDtt9+ehQsXAnDBBRdw/PHHM336dGbOnMkjjzzCr3/96yctO1C/vffem6c+9akAXHbZZRx88MFMmjSJZzzjGey+++5cc801A4V1dVX9aoB5v6mqx4H5wGSaM9a/7Fjm2x39Lwc+k+RoYKP2IEEalNHKvcsuu4xDDz0UgD322IN77rmHxYub854vf/nLWWONNZg6dSpLly5l3333BWDq1KksXLhwSDEA/Q+bgLPb3/No8gxgV+A7AFX1Y+C+tn2FT9ZK6tOD7e/0M7+/dkkraPWxDmBV0w4hubeqTmuHVR4xxiGpC8yePZsLL7yQOXPmsO666z5RRK699tpMmjQJgKoa9HrXWmstACZNmvTE/a9VxVlnncUWW2yxTN+rrrpqmdcD9VtvvfWW6TdIDw4wr3M85VKaz6h+d/ZVdXyS84BXAFcm2auqbh1sQJq4Rjv3emtH2D/Rf7XVVmONNdZ4on211VZjyZIlQ4qBFcu1njyDfnKtqn6WZHuaPPtUkguq6mNDCUia4K4CPp9kE5qTRgcDXwDmAp9NsjFwP/Ba2lt0JK0Yr9QOv6nA1UnmAx8CPj7G8agLLF68mI033ph1112XW2+9lSuvvPJJfbbccksWLlzI//7v/wJw6qmnsvvuuw96Wy972cv4whe+8MRB8nXXXQfABhtswP3337/cfr3ttttunHHGGSxdupRFixZx6aWXsuOOOw46rgHcCjw/yeT29RP38yX5y6paUFUn0BwU9HsfktSX0cy93XbbjdNPPx1oiulNNtmEpzzlKSu07HDFsByXAa8HSLIPsHE7vTnwUFWdBnwa2G64NyxNBFV1F/BB4GLgeuDaqvp+Vf0W+CRN0XshcDPgiAhpECxqh1lVnV9V06pqelXtUFVzq2pmVc0d69g0fu27774sWbKEadOmccwxx7Dzzjs/qc/aa6/NN77xDV73utcxdepUVlttNY466qhBb+uYY47hscceY9q0aUyZMoVjjjkGgJe+9KXcfPPNTzwoqr9+vR1wwAFMmzaNbbbZhj322IN//dd/5ZnPfOag4+pPVT0MvAP4cfvQm//jzzv7d7UPrrme5n7aHw3bhjUhjGbuHXfcccydO5dp06bxgQ98gFNOOWWFlx2uGJbjo8A+Sa4FXg7cRXPVyJO10hBV1cL2XvSe19+qqqlVNaWq3tfR9VtV9UJgL+AvaE7USlpBGeKQJg2ztTZ7QW12+OdYePx+Yx2KNGySzKuqlf46qyTrV9UD7dOQvwT8vKo+O9j19OTZyjJPNZ4MY56tBSytqiVJdgG+0j6EbdA6c8180apiuHKtn3V/mqagXRu4APjHWs5B+owZM2ruXGtfrVqGmmfeUyupG/xdksOBNYHraJ6GLGl4PQf4rySrAX8C/m6M45EmjKp6z/J7SeqPRa3U5Q444AB+9atlHyZ8wgkn8LKXvWyMIoIFCxb0POV1q3bIIsCjVbXTUNbXXpUd9JVZaSSNh9w74IADYNk8A3h/VZ0/2HVV1c9pvlZLkqSuYlErdblzzjlnrEN4kqlTpzJ//nyS3DxSQ7WksTYecu+cc84xzyRJE54PipIkSZIkdS2LWkmSJElS17KolSRJkiR1LYtaSZIkSVLX8kFR48TUZ23IXL/LTxpR5pk0Osw1SdJo8kqtJEmSJKlrWdRKkiRJkrqWRa0kSZIkqWtZ1EqSJEmSupZFrSRJkiSpa/n043FiwW8XM/kD57HQp0VKI6Ynz0aCuSv9WV+5Zo5IkkaKV2olSZIkSV3LolaSJEmS1LUsaiVJkiRJXcuiVpIkSZLUtSxqJUmSJEldy6JWkiRJktS1LGolSZIkSV3LolaSJEmS1LUsaiVJkiRJXcuiVpIkSZLUtSxqJUmSpDGUZO8k85IsaH/vMdYxSd3EonaI0vDvp1XG3LlzOfroo/ucN3nyZO6+++6hrHbblQqqH0lmJ5kxEuuWYNn/+Re/+MX99ps9ezavfOUrB7XuWbNmceedd65UfIOV5NVJPjCqG5W0QpKsDtwNvKqqpgKHA6eObVRSd1l9rAPoJkkmAz8CLgZ2AT6X5D1AgPOq6v1tvzcD7wfuBH4OPFpV7xyLmKUVNWPGDGbMsE6UerviiiuGdX2zZs1iypQpbL755iu8zJIlS1h99aHtspOsXlXnAucOaQWSlqvjGPEy4MXAb4G/advmAzsCTwGOrKqrkxwHbA5MBu6uqjd0rO4mYO0ka1XVo6P0FqSu5pXGwdsC+CawH/AvwB7AdGCHJPsn2Rw4BtgZ2BvYsr8VJXlrkrlJ5i59aPHIR64JYf/992f77bdn66235qSTTgJg/fXX5/3vfz/bb789e+21F1dffTUzZ87k+c9/Puee2xzndl5xuueee9hnn33Ydtttedvb3kZVPbH+0047jR133JHp06fztre9jaVLlw4YT5J/T3JtkouSbNq2TU9yZZIbkpyTZOO2fXaSE5JcneRnSV7Stq+T5Dtt/zOAddr2SUlmJbmxHbL1T31s3zzTgPrKmU7rr7/+gMs/8MADHHjggWy55ZYccsghT+TLvHnz2H333dl+++152ctexl133cWZZ57J3LlzOeSQQ5g+fToPP/xwn/0AZs6cyT//8z+z++678/nPf57bb7+dPffck2nTprHnnnvy61//ut+Y2rz4TJKLgROSHJHkix3zTkxyRZJfJjmwbV8tyZeT3JTkB0l+2DHv+CQ3tzn46X62aa5ponsB8KWq2hr4A/Datn29qnox8A7g5I7+2wN/06ugpV3uur4K2s48W7Ro0fC/A6lLWdQO3u1VdSWwAzC7qhZV1RLgdGA3mjNxl1TVvVX1GPDd/lZUVSdV1YyqmjFp3Q1HJXit+k4++WTmzZvH3LlzOfHEE7nnnnt48MEHmTlzJvPmzWODDTbgwx/+MD/5yU8455xzOPbYY5+0jo9+9KPsuuuuXHfddbz61a9+4uD5lltu4YwzzuDyyy9n/vz5TJo0idNPP32gcFYDrq2q7YBLgI+07d8E3l9V04AFHe0Aq1fVjsC7OtrfDjzU9v8EzYEANCeUnlVVU9ohW9/oHYB5puXpK2cG47rrruNzn/scN998M7/85S+5/PLLeeyxx/iHf/gHzjzzTObNm8eRRx7Jhz70IQ488EBmzJjB6aefzvz581l99dX77NfjD3/4A5dccgnvfve7eec738lhhx3GDTfcwCGHHNLv7QIdXgjsVVXv7mPeZsCuwCuB49u219BcNZoKvIVmRBJJngocAGzd5uDH+9qYuSbxq6qa307Po8kngG8DVNWlwFOSbNS2n1tVD3euIMnWwAnA2/raQGeebbrppsMdv9S1HH48eA+2v9PP/P7apVFx4okncs455wBwxx138POf/5w111yTfffdF4CpU6ey1lprscYaazB16lQWLlz4pHVceumlnH322QDst99+bLzxxgBcdNFFzJs3jx122AGAhx9+mKc//enLC+mM9vdpwNlJNgQ2qqpL2vZTWPbkz9nt784Dgt2AEwGq6oYkN7TtvwSen+QLwHnABcsLRuqtr5wZjB133JG/+Iu/AGD69OksXLiQjTbaiBtvvJG9994bgKVLl7LZZps9adnbbrttwH4HHXTQE9Nz5sx5Ii8PPfRQ3ve+9y0vtO9WVX9DKb5XVY8DNyd5Rtu2a7vM48Dv2qu8AH8EHgG+nuQ84AfL27A0QXVeWV1KO6oIqF79el4/2NmY5C+Ac4DDquoXIxKhtIqyqB26q4DPJ9kEuA84GPgCMBf4bDuc8n6aISQLxixKTSizZ8/mwgsvZM6cOay77rrMnDmTRx55hDXWWIOkOd+y2mqrsdZaaz0xvWTJkj7X1dO/U1Vx+OGH86lPfWqoIfbesfel56BgKct+Rj1p2aq6L8k2wMuAvwdeDxw51OA08fSXM4PRk08AkyZNYsmSJVQVW2+9NXPmzBlw2eX1W2+99fpdtq8c7eXBAeZ1Hnyn1+9lVNWSJDsCewJ/C7yT5tYbSSvmIODiJLsCi6tqce/8ba/engd8sKouH4MYpa7m8OMhqqq7gA/SPDTqepohlt+vqt8Cn6Qpei8Ebga8uUijYvHixWy88casu+663HrrrVx55ZVDWs9uu+32xLDiH/3oR9x3330A7Lnnnpx55pn8/ve/B+Dee+/l9ttvX97qDmx/vwG4rKoWA/f13C8LHEozNHkglwKHACSZAkxrpzcBVquqs2juZd9uRd+jBMOXM71tscUWLFq06Ili9bHHHuOmm24CYIMNNuD+++9fbr/eXvziF/Od73wHgNNPP51dd911WGLtcBnw2vbe2mcAMwGSrA9sWFU/pLktYPpwb1haxd2X5Argq8Cb++nzTuCvgGOSzG9/ljsUSlLDK7WDUFULgSkdr78FfKuPrt+qqpPaR7Sfg0MiNUr23XdfvvrVrzJt2jS22GILdt555yGt5yMf+QgHH3ww2223HbvvvjvPec5zANhqq634+Mc/zj777MPjjz/OGmuswZe+9CWe+9zn9reqx4Gtk8yjObnTM5bycOCrSdalGUL8puWE9BXgG+2w4/nA1W37s9r2nhN0HxzC29UENlw509uaa67JmWeeydFHH83ixYtZsmQJ73rXu9h666054ogjOOqoo1hnnXWYM2dOv/16O/HEEznyyCP5t3/7NzbddFO+8Y0n3UK+ss6iuRp7I/AzmpOzi4ENgO8nWZvmal6EtPQAACAASURBVO6THsgmTXR9HCN+GpoHIAJnVdUHe/U/rtfrj9PP/eqSli+dTzXV8GifDLkXsDZNQfuPtZw/9FqbvaA2O/xzLDx+v9EIURoVSeZV1bj5nqCePBsJ5q7GynDmWZL1q+qBJE+jOXn011X1u8Gup69cM0fU7YaSa21R+56qmjvc8cyYMaPmzh321Upjaqj7NK/UjoCqes9YxyBJ0hD8oL23b03gX4ZS0Er6s6qaOdYxSBOBRa2klbbTTjvx6KPLfp3eqaeeOkbRSMNvwYIFHHroocu0rbXWWlx11VVjFFHjE5/4BMBWSeZ3NH+3qj4xlPV5AC5J6kYWtZJW2lgf2EsjberUqcyfP3/5HUfZhz70IT784Q/fPJ6G+UuSNNp8+rEkSZIkqWtZ1EqSJEmSupZFrSRJkiSpa3lP7Tgx9VkbMtevO5BGlHkmjQ5zTZI0mrxSK0mSJEnqWha1kiRJkqSuZVErSZIkSepaFrWSJEmSpK5lUStJkiRJ6loWtZIkSZKkruVX+owTC367mMkfOI+FfgWCNGJ68mwkmcNS/7lmfkiSRoJXaiVJkiRJXcuiVpIkSZLUtSxqJUmSJEldy6JWkiRJktS1LGolSZIkSV3LolaSJEmS1LUsaiVJkiRJXcuiVpIkSZLUtSxqJUmSJEldy6JW0iohydeTbDXWcUirgiSbJzmznX5akouTPJDki2Mdm7QqMs+klTOiRW2ShUk2aaevGKDfzCQ/GMlYhmKgmCWNL1X1lqq6eazjkFYFVXVnVR3YvnwEOAZ4zxiGJK2ykqyOeSatlFG7UltVLx6tba2sJJOgu2KWJpIk6yU5L8n1SW5MclCS2UlmjHVs0niRZHKSW5OckuSGJGcmWbc94fzJJHOSzE2yXZLzk/wiyVEdy94IUFUPVtVlNAfdkvrQ5swtSf4jyU1JLkiyTrtv+lySK9r91Y5t/+OSnJTkAuCb5pm0coatqE3yvSTz2kR+ax/zH1jOKtZvd7i3Jjk9Sdrl9kxyXZIFSU5OslbbviI75ST5t/ZDZEGSgwaIf2Y77ONbwILOmNt5s/uJ7xVt22VJTuy54pxk9yTz25/rkmzQxzbf2sY+d+lDi1fkzyypsS9wZ1VtU1VTgB/319E80wS3BXBSVU0D/gi8o22/o6p2Af4HmAUcCOwMfGyoGzLXJF4AfKmqtgb+ALy2bV+vvVDyDuDkjv7bA39TVW9Y0Q105tmiRYuGK26p6w3nldojq2p7YAZwdJKnDXL5bYF3AVsBzwf+OsnaNDvbg6pqKrA68PaOZZa3U34NMB3YBtgL+Lckmw0Qw47Ah6qqr/vy+ovva8DLq2pXYNOO/u8B/r6qpgMvAR7uvcKqOqmqZlTVjEnrbjhAWJJ6WQDsleSEJC+pqn6PoM0zTXB3VNXl7fRpwK7t9Lnt7wXAVVV1f1UtAh5JstFQNmSuSfyqqua30/OAye30twGq6lLgKR05dm5VPen4cCCdebbpppsufwFpghjOovboJNcDVwLPpjlbNRhXV9VvqupxYD7NB8EWNB8QP2v7nALs1rHM8nbKuwLfrqqlVfV/wCXADsuJ4VeDiG9L4Jcdy3y7o//lwGeSHA1sVFVLlvcHkLRi2s+E7Wly/1NJjh3jkKTxqvp5/Wj7+/GO6Z7Xq490UNIqqjOXlvLnXOovDx8c8YikCWJYitokM2muhO5SVdsA1wFrD3I1fX0QZAWX6W+nvLzlexvow2VQ8VXV8cBbgHWAK5NsOchYJPUjyebAQ1V1GvBpYLsxDkkar56TZJd2+mDgsrEMRpqgDgJIsiuweKDRRZKGZriu1G4I3FdVD7XF287DtN5bgclJ/qp9fSjN1dYVdSlwUJJJSTalucp79TDF1hPf85NMbl8/cc9ukr+sqgVVdQIwl+aqrqThMRW4Osl84EPAx8c4Hmm8ugU4PMkNwFOBrwxlJUkWAp8Bjkjym/j1WdJg3JfmGzW+Cry5v07mmTR0wzXE6MfAUe1O8zaaIcgrraoeSfIm4LtpHnd+Dc0Hwoo6B9gFuJ5mqMf7qup3wxFbG9/DSd4B/DjJ3SxbML8ryUtprureDPxouLYrTXRVdT5wfq/mmWMQijTePV5VR/Vqm9wzUVWzaJ5J0fO6Z97dwJQ+2iX1oaoWsmzOfBogyWzgrKr6YK/+x/WxjskjGaO0KhuWoraqHgVe3sesyR191h9g+dnA7I7X7+yYvojmIU29l+lc9yz63ikDvLf9GVDvGDpjHig+4OKq2rJ9GvKXaK7KUlX/sLxtSpIkSZJWjg+DWHl/l+RwYE2ae4m/NsbxSJL0pCtHkkZfVc0c6xikiWBUi9okU4FTezU/WlU7dWsMVfVZ4LMrHZgkSZIkadBGtaitqgU03xs7ZsZDDJIkSZKk4TGc31MrSZIkSdKosqiVJEmSJHUti1pJkiRJUtfy6cfjxNRnbcjc4/cb6zCkVZp5Jo0Oc02SNJq8UitJkiRJ6loWtZIkSZKkrmVRK0mSJEnqWha1kiRJkqSuZVErSZIkSepaFrWSJEmSpK7lV/qMEwt+u5jJHzgPgIV+DYI0IjrzbLww37UqGijX/J+XJA03r9RKkiRJkrqWRa0kSZIkqWtZ1EqSJEmSupZFrSRJkiSpa1nUSpIkSZK6lkWtJEmSJKlrWdRKkiRJkrqWRa0kSZIkqWtZ1EqSJEmSupZF7TBJsnmSM9vppyW5OMkDSb441rFJq4IkVyxn/uuS3JLk4tGKSVpV9dqn7Z1kXpIF7e89xjo+qZu5P5OG3+pjHcCqoqruBA5sXz4CHANMaX8kraSqevFyurwZeEdVeRAgraRe+7S7gVdV1Z1JpgDnA88as+CkLuf+TBp+XqntkGRykluTnJLkhiRnJlk3ycIkn0wyJ8ncJNslOT/JL5Ic1bHsjQBV9WBVXUZT3EoaBkkeaH9vluTSJPOT3JjkJUmOBXYFvprk38Y2Uml8GMZ92nVtkQtwE7B2krXG6n1J3c79mTT8LGqfbAvgpKqaBvwReEfbfkdV7QL8DzCL5gz2zsDHhrqhJG9tDyjmLn1o8cpFLU0cbwDOr6rpwDbA/Kr6GDAXOKSq3tvZ2TzTBDfc+7TXAtdV1aO9Z5hr0qANan8Gy+bZokWLRjlcafyyqH2yO6rq8nb6NJqzZQDntr8XAFdV1f1VtQh4JMlGQ9lQVZ1UVTOqasakdTdcuailieMa4E1JjgOmVtX9A3U2zzTBDds+LcnWwAnA2/qab65Jgzao/Rksm2ebbrrpiAcodQuL2ierfl73nJV+vGO657X3JkujpKouBXYDfgucmuSwMQ5JGs+GZZ+W5C+Ac4DDquoXwx2kNBG5P5OGj0Xtkz0nyS7t9MHAZWMZjKRlJXku8Puq+g/gP4HtxjgkaTxb6X1ae+X2POCDHVd9Ja0k92fS8LGofbJbgMOT3AA8FfjKUFaSZCHwGeCIJL9JstXwhShNaDOB+Umuo7m/7/NjG440rg3HPu2dwF8Bx7QPtJmf5OnDGaQ0Qc3E/Zk0LBw2+2SPV9VRvdom90xU1Syah2r0vO6ZdzcdX9/T0S5pGFTV+u3vU4BT+pg/c7RjkrrASu/TqurjwMdHMEZpQnF/Jg0/r9RKkiRJkrqWV2o7VNVCOq62SpLUrdynSZImCq/USpIkSZK6lkWtJEmSJKlrWdRKkiRJkrqWRa0kSZIkqWtZ1EqSJEmSupZFrSRJkiSpa/mVPuPE1GdtyNzj9xvrMKRVmnkmjQ5zTZI0mrxSK0mSJEnqWha1kiRJkqSuZVErSZIkSepaFrWSJEmSpK5lUStJkiRJ6lo+/XicWPDbxUz+wHkALPSJkdKI6Myz8cjc16piebnm/7okaTh5pVaSJEmS1LUsaiVJkiRJXcuiVpIkSZLUtSxqJUmSJEldy6JWkiRJktS1LGolSZIkSV3LolaSJEmS1LUsaiVJkiRJXcuiVpIkSZLUtSxqJUmSJEldy6JWUtdIsnmSM9vppyW5OMkDSb441rFJq5JeubZjkvntz/VJDhjr+CRJ6jRsRW2ShUk2aaevGKDfzCQ/GOS6j0iy+crGOMhtvjrJB0Zzm5IGVlV3VtWB7ctHgGOA94xhSNIqqVeu3QjMqKrpwL7A15KsPnbRSZK0rBG5UltVLx7mVR4BDKqoXZkdbpLVq+rcqjp+qOuQ1Lckk5PcmuSUJDckOTPJuu2JsU8mmZNkbpLtkpyf5BdJjupY9kaAqnqwqi6jKW4l9TKMufZQVS1pV7s2UGP1nqTxKskbk1zdjmj4WpJJSfZNcm07wuGitt/6Sb6RZEGbl69t2/dpc/LaJN9Nsv7YviOpuwypqE3yvSTzktyU5K19zH9gOatYv9253prk9CRpl9s+ySXtus9PslmSA4EZwOntB8U6ffVrl5/d7qgvAf4xyXOTXNR+aFyU5DkDvKdZST6T5GLghPbq8Bc75p2Y5Iokv2xjIslqSb7c/h1+kOSHHfOOT3Jzu+1P97PNt7YHFHOXPrR4uX93aRWyBXBSVU0D/gi8o22/o6p2Af4HmAUcCOwMfGyoGzLPNMENS64l2SnJTcAC4KiOIrezj7mmCSnJi4CDgL9uRzQsBd4I/Afw2qraBnhd2/0YYHFVTW3z8qdpRjp+GNirqrYD5gL/r59tPZFnixYtGtk3JnWRoV7NPLKq7k2yDnBNkrMGufy2wNbAncDlwF8nuQr4AvA3VbUoyUHAJ6rqyCTvBN5TVXOTrNFXP+DIdt0bVdXuAEn+G/hmVZ2S5EjgRGD/AeJ6Ic0HytIkR/SatxmwK7AlcC5wJvAaYDIwFXg6cAtwcpKnAgcAW1ZVJdmor41V1UnASQBrbfYCz3xrIrmjqi5vp08Djm6nz21/LwDWr6r7gfuTPNJfHi2PeaYJblhyraquArZuD95PSfKjqnqkVx9zTRPVnsD2NMfEAOsAOwGXVtWvAKrq3rbvXsDf9ixYVfcleSWwFXB5u/yawJy+NtSZZzNmzDDPpNZQi9qj8+cHRTwbeMEgl7+6qn4DkGQ+TWH4B2AK8JM2oScBd/Wx7BbL6XdGx/QuNIUnwKnAvy4nru9W1dJ+5n2vqh4Hbk7yjLZt13aZx4HftVd5oTkb/gjw9STnAYO6h1iaAHrviHteP9r+frxjuue19/BJgzesuVZVtyR5kGY/PHe4gpS6XIBTquqDTzQkrwZe30/f3nkZ4CdVdfDIhSit2gY9/DjJTJqzTLu0wymuo7nHZjA6d6BLaXagAW6qquntz9Sq2qevEJbT78EBtru8M1oDLdsZc3r9XnYjzbCsHYGzaK4M/3g525Ummuck2aWdPhi4bCyDkVZhK51rSZ6X9jkVSZ5Lc3J54bBFKHW/i4ADkzwdoB2xdz2we5LndbQBXAC8s2fBJBsDV9KMWvyrtm3dJC8cxfilrjeUe2o3BO6rqoeSbElzD85wuA3YtGfnm2SNJFu38+4HNliBfr1dwZ+HeBzC8B84Xwa8tr239hnAzDam9YENq+qHwLuA6cO8Xanb3QIcnuQG4KnAV4aykiQLgc8ARyT5TZKthi9EaZUwHLm2K3B9O7LqHOAdVXX3MMYodbWqupnmntgL2lz7Cc1ta28Fzk5yPX8eSfhxYOMkN7btL62qRTQPRf12u/yVNLe7SVpBQxnO92PgqDbpbqNJvJVWVX9qH7J0YpIN29g+B9xE8xCLryZ5mGZIcX/9ejua5h7X9wKLgDcNR6wdzqK5j+JG4GfAVcBimgL8+0nWprma+0/DvF2p2z1eVUf1apvcM1FVs2jyvud1z7y7aYY99m6X1LeVzrWqOpXmFh5J/aiqM1j2FrgeP+rV7wHg8D6W/ymww8hEJ636Bl3UVtWjwMv7mDW5o0+/jyGvqtnA7I7X7+yYng/s1scyZ9EUkD366zez1+uFwB79xdKr7xG9Xs+i3dH3MW/99vfjSd5TVQ8keRpwNbCgqn5HM/xYkiRJkjSCfPDKyvtB+6TINYF/aQtaSf1oTzZNWV4/SSvHXJMkTRQjVtQmmcqThys9WlU7jdQ2V0SSD/Hn7wrr8d2q+sRQ1tf76rAkSZIkafSMWFFbVQsYhw9IaovXIRWwkiRJkqTxZShPP5YkSZIkaVywqJUkSZIkdS2LWkmSJElS1/Lpx+PE1GdtyNzj9xvrMKRVmnkmjQ5zTZI0mrxSK0mSJEnqWha1kiRJkqSuZVErSZIkSepaFrWSJEmSpK5lUStJkiRJ6loWtZIkSZKkruVX+owTC367mMkfOA+AhX4NgjQiOvNsPPMzQN1uRXLN/3NJ0nDxSq0kSZIkqWtZ1EqSJEmSupZFrSRJkiSpa1nUSpIkSZK6lkWtJEmSJKlrWdRKkiRJkrqWRa0kSZIkqWtZ1EqSJEmSupZFrSRJkiSpa1nUSuoaSTZPcmY7vWOS+e3P9UkOGOv4pFVdkq8n2Wqs45BWNUmOS/KesY5D6lbjvqhNMiPJif3MW5hkkyGs84GVj6zP9c5OMmMk1i0JqurOqjqwfXkjMKOqpgP7Al9LsvrYRSet+qrqLVV181jHIUlSp3Ff1FbV3Ko6eqzjkDQ8kkxOcmuSU5LckOTMJOu2J6k+mWROkrlJtktyfpJfJDmqY9kbAarqoapa0q52baDG6j1Jq6Ik6yU5rx0JcWOSgzx5K/UtyfeSzEtyU5K3JpmUZFabOwuS/FPbb3aSzyW5op23Y8dqtmrn/zKJx77SIIxaUds72du2B5Kc0LZf2A4n7EnmV7d9Zib5QTv9tCQXJLkuydeAdKz/jUmubocifi3JpOXE8+9Jrk1yUZJN27bpSa5sD7TPSbJx2z67jfPqJD9L8pK2fZ0k32n7nwGs07b3+UHWRwxvbQ/e5y59aPHK/omlbrIFcFJVTQP+CLyjbb+jqnYB/geYBRwI7Ax8rK+VJNkpyU3AAuCojiK3s495Jg3NvsCdVbVNVU0BfjxQZ3NNE9yRVbU9MAM4GpgOPKuqplTVVOAbHX3Xq6oX0+z7Tu5o3xJ4GbAj8JEka/TeSGeeLVq0aKTei9R1RvNK7TLJnuRpwHrA7Lb9fuDjwN7AAfR9EPsR4LKq2hY4F3gOQJIXAQcBf90ORVwKHDJALOsB11bVdsAl7XoBvgm8vz3QXtDRDrB6Ve0IvKuj/e3AQ23/TwDbt+0DfZA9oapOqqoZVTVj0robDhCutMq5o6oub6dPA3Ztp89tfy8Arqqq+6tqEfBIko16r6SqrqqqrYEdgA8mWbuPPuaZNDQLgL3ak7ovqaoBK1VzTRPc0UmuB64Eng2sCTw/yReS7EtzArfHtwGq6lLgKR37t/Oq6tGquhv4PfCM3hvpzLNNN910JN+P1FVGs6jtnewvAP7En8/8LgAuqarH2unJfaxjN5oDYKrqPOC+tn1PmoLymiTz29fPHyCWx4Ez2unTgF2TbAhsVFWXtO2ntNvrcXb7e15HbJ3x3ADc0Lb/kv4/yCQ9eahwz+tH29+Pd0z3vO73ftmqugV4EJgyXAFKE11V/Yxm37oA+FSSY8c4JGlcSjIT2AvYpaq2Aa4D1gK2AWYDfw98vWOR5e0DoblA43MipBU0KkVtP8m+NvBYVfUk8hMHsVU10AFsX/fNBTilqqa3P1tU1XGDCHFF7sXr+aDp/SHzpGWr6j76/yCTBM9Jsks7fTBw2WBXkOR5PQ+GSvJcmiHNC4ctQmmCS7I5zWik04BPA9uNcUjSeLUhcF9VPZRkS5rbZjYBVquqs4BjWDZ/DgJIsiuweHmjICQt32hdqe0r2YfiUtphxUleDmzctl8EHJjk6e28p7YHuf1ZjeZePYA30AxpXgzc13O/LHAozdDkFY1nCjCtnR7og0wS3AIcnuQG4KnAV4awjl2B69vRGecA72iHbEkaHlOBq9sc+xDNLUKSnuzHwOrtPu1faEYlPguY3ebPLOCDHf3vS3IF8FXgzaMcq7RKGq1hDT8GjmqT/TaaZB+KjwLfTnItTcH5a4CqujnJh4ELkqwGPEZzhfT2ftbzILB1knnAYtozZsDhwFeTrEszhPhNy4nnK8A32vc1H7i6bX9W295z0uCDfS0sTWCPV9VRvdom90xU1Syag4Ce1z3z7qYdYlxVpwKnjmCM0oRWVecD5/dqnjkGoUjjWlU9Cry8j1mf72eRs6pqmWPD3iMM24ezSVpBo1LUDpDs63f0Oa7XMuu3v2fTDOOlqu4B9uno9k8d/c/gz/fJLi+enu0e06t9Pn1cRa6qmR3Td9MefFfVw8Df9rMZr85KkiRJ0gjzBnRJo6qqFuIDnSRJE1DnhRJJw2eVLmqTXEXz9LlOh1bVgrGIR5IkSZI0vFbporaqdhrrGCRJkiRJI2c0v6dWkiRJkqRhZVErSZIkSepaFrWSJEmSpK5lUStJkiRJ6lqr9IOiusnUZ23I3OP3G+swpFWaeSaNDnNNkjSavFIrSZIkSepaFrWSJEmSpK5lUStJkiRJ6loWtZIkSZKkrmVRK0mSJEnqWj79eJxY8NvFTP7AeQAs9ImR0ojozLNu4GeButWK5pr/45Kk4eCVWkmSJElS17KolSRJkiR1LYtaSZIkSVLXsqiVJEmSJHUti1pJkiRJUteyqJUkSZIkdS2LWkmSJElS17KolSRJkiR1LYtaSZIkSVLXsqiVJEmSJHUti9phkmTzJGe203snmZdkQft7j7GOT1oVmGeSpG6QZHKSG/ton51kxljEJK3KLGqHSVXdWVUHti/vBl5VVVOBw4FTxy4yadVhnmk8O/HEE3nRi17EIYccslLrmTVrFnfeeecTr9/ylrdw8803r2x4Q5Jk7SRXJ7k+yU1JPjomgUiSNACL2g7tWbVbk5yS5IYkZyZZN8nCJJ9MMifJ3CTbJTk/yS+SHNWx7I0AVf+fvXsPr6I89///volICFCEypfSWhu1KoWsHCAEtEiCYqBaEUW+/tioILsqtd1+3devitWKh61VKb/aIipleyFupDRbFLQe2AIlpGBAEgkEFcSNUat+WxDNlqMk3L8/ZhIWYeVITgs+r+viylrPPPPMTeBeM/fMM7N8g7tXHZG8DSSaWae2+nuJtCfKMzlePfHEE7z66qssWLCguq2ioqLR49Qsap966in69evXLDE2wQHgQndPA9KBUWY2pK2CEYkzCWb27+EJodfNrHPYfo2ZvWFmm80sC8DMssK2DeHPc9swbpG4o6L2aOcCc9w9Ffgf4Oaw/WN3Pw/4KzAPuAoYAtxfz3hjgQ3ufqDmAjO7MTx4L6rcW95c8YvEA+WZHFemTJnC9u3bGT16NN27d+fGG28kNzeX6667jrKyMi644AIGDBjAgAEDeOONN6rXmz59OpFIhLS0NO644w4WLVpEUVEREyZMID09nX379pGTk0NRUREACxcuJBKJkJKSwtSpU6vHMbPdZvZgeEV1rZn1ri1WMxsXHkxvNLOCsG2Smb1oZkvNbKuZ3QPggd3hqh3DP17LuMo1kSOdDTzu7v2BLwn2VQBd3P18gn3f3LBtCzDM3TOAacCvYw0YnWc7duxo2ehF4shJbR1AO/Sxu68JXz8L3BK+fin8WQp0dfevgK/MbL+ZnRJrIDPrDzwC5MZa7u5zgDkAnfqcHfMgQeQ4pTyT48rs2bNZunQpK1euZNasWfz5z39m9erVdO7cmb1797Js2TISExPZtm0b48ePp6ioiNdee40lS5awbt06kpKS2LVrFz179mTWrFnMmDGDzMwjb7v79NNPmTp1KsXFxfTo0YPc3FyWLFlStbgLsNbd7zKz6cANwAO1hDsNGOnun9TIqywgBdgLrDezV9y9yMwSgGLg+wQH6OtiDapcEznKB+5eEr4uBpLD1wsB3L3AzL4R5mE34BkzO5vgxFHHWANG51lmZqbyTCSkK7VHq/kBUfW+6grQoajXVe+POjlgZqcBi4Hr3P2/mztIkTinPJPj2ujRo+ncOZhpePDgQW644QYikQjjxo2rvj92+fLlXH/99SQlJQHQs2fPOsdcv349OTk59OrVi5NOOokJEyZQUFBQtfhr4OXwdfTBcyxrgHlmdgOQENW+zN0/d/d9wAvAUAB3r3T3dOA0IMvMUhr2WxA54UXvxyo5vB+LtQ/8N2Clu6cAlwGJLR+eyPFDRe3RTjez88LX44HVjR0gPOP2CvDLqKtRInKY8kyOa126dKl+/eijj9K7d282btxIUVERX3/9NQDujpk1eEz3Oi/KHPTDHaIPnmONMwX4FfBdoMTMvlm1qGbXGut9CeQDoxoctIjEcjWAmQ0Fyt29HOgOfBIun9RGcYnELRW1R3sXmGhmm4CewJNNGOPnBNO07jazkvDP/2rOIEXinPJMThjl5eX06dOHDh06MH/+fCorKwHIzc1l7ty57N27F4Bdu3YB0K1bN7766qujxhk8eDCrVq1i586dVFZWsnDhQrKzsxsdj5md5e7r3H0awVPEvxsuutjMeoYPsxkDrDGzXlVTlMP2EQT3/olI031hZm8As4F/DtumAw+Z2RqOnEEhIg2ge2qPdig8ix0tueqFu88jeIBN1fuqZTsJ7kXC3R+g9nuZRER5JieQm2++mbFjx/Lcc88xfPjw6qu4o0aNoqSkhMzMTE4++WQuueQSfv3rXzNp0iSmTJlC586dKSwsrB6nT58+PPTQQwwfPhx355JLLuHyyy9vSki/Ce/bM2AFsJHgycarCb4a6/vAH8P7aVMJ7vNLIDgR/p/u/nIt44pIyN3LCPdX4fsZ9fQvBM6Jarq7ZSITOT5ZPdOZTihmlgy8HN7P0Ko69Tnb+0z8HQBlD1/a2psXaRFmVuzumTXakmkHeRYP9FkgDRErz5owxiQg091/3hwxNTTX9H9c4klz5FpzyszM9Kono4scL5qaZ7pSG6XmWTURaX7KMxERERFpTipqRUREjkNmdhcwrkbzc+7+YM2+Naf8i4iIxBMVtSIiIsehsHg9qoAVERE53ujpjRimIQAAIABJREFUxyIiIiIiIhK3VNSKiIiIiIhI3FJRKyIiIiIiInFL99S2E5HvdKdIX20g0qKUZyKtQ7kmIiKtSVdqRUREREREJG6pqBUREREREZG4paJWRERERERE4paKWhEREREREYlbKmpFREREREQkbqmoFRERERERkbilr/RpJ0o/KSf5jlcAKNPXIIi0iOg8iyf6TJB409Bc0/9tERFpDrpSKyIiIiIiInFLRa2IiIiIiIjELRW1IiIiIiIiErdU1IqIiIiIiEjcUlErIiIiIiIicUtFrYiIiIiIiMQtFbUiIiIiIiISt1TUioiIiIiISNxSUSsiIiIiIiJxS0VtA5hZspltjtGeb2aZbRGTiIiIiIiIqKgVkXbOAvqsEmlByjNp75KTk9m5c2eD+5eUlPDqq6+2YESxmdkbrb5REVFR2wgJZvbvZva2mb1uZp3D9mvM7A0z22xmWQBmlhW2bQh/ntuGcYvEnXB2xLtm9gTwFnCtmZWGefZIVL9/NrP3wlkT/25ms9ouapH4ojyTtlRZWdmi4zelqK2oqGjy9swsAcDdz2/yICLSZCpqG+5s4HF37w98CYwN27uEH2A3A3PDti3AMHfPAKYBv441oJndaGZFZlZUube8ZaMXiT/nAv8BXAr8G3AhkA4MMrMxZvZt4G5gCHAx0DfWIMozkTo1S56Bck2ONGbMGAYOHEj//v2ZM2cOAF27dmXatGkMHjyYwsJCVqxYQUZGBpFIhMmTJ3PgwIE6x3zssccYMGAAkUiELVu2ALBnzx4mT57MoEGDyMjI4MUXX+Trr79m2rRp5OXlkZ6eTl5eXsx+APPmzWPcuHFcdtll5Obm4u7cdtttpKSkEIlEyMvLqzUeM8sxs5Vm9kegNGzbHbUs38wWmdkWM1tgZhYuuyRsW21mM83s5bA928xKwj8bzKxbjG1W59mOHTsa+88ictw6qa0DiCMfuHtJ+LoYSA5fLwRw9wIz+4aZnQJ0A54xs7MBBzrGGtDd5wBzADr1OdtbMHaRePShu681s8uBfHffAWBmC4BhYZ9V7r4rbH8OOKfmIMozkTo1S56Bck2ONHfuXHr27Mm+ffsYNGgQY8eOZc+ePaSkpHD//fezf/9+zj77bFasWME555zDddddx5NPPsmtt95a65innnoqb731Fk888QQzZszgqaee4sEHH+TCCy9k7ty5fPnll2RlZTFixAjuv/9+ioqKmDUrmFhw5513xuwHUFhYyKZNm+jZsyfPP/88JSUlbNy4kZ07dzJo0CCGDRtGnz59agsrC0hx9w9iLMsA+gOfAmuAH5pZEfAHgosfH5jZwqj+vwB+5u5rzKwrsL/mgNF5lpmZqTwTCelKbcNFnz6s5PAJgZofKE5wtnulu6cAlwGJLR+eyHFnT/jTalleW7uINJzyTFrEzJkzSUtLY8iQIXz88cds27aNhIQExo4NJrpt3bqVM844g3POCc6RTJw4kYKCgjrHvPLKKwEYOHAgZWVlALz++us8/PDDpKenk5OTw/79+/noo4+OWreufhdffDE9e/YEYPXq1YwfP56EhAR69+5NdnY269evryusN2spaKuW/c3dDwElBBdE+gLbo9aJLmrXAL81s1uAU9y96fOhRU4wKmqP3dUAZjYUKHf3cqA78Em4fFIbxSVyvFgHZJvZqeE9S+OBVcCbYXsPMzuJw7cEiEjjKc+k2eTn57N8+XIKCwvZuHEjGRkZ7N+/n8TERBISEgBwb/xFxk6dOgGQkJBQff+ru1dfXS0pKeGjjz7iBz/4wVHr1tWvS5cuR/RrpD11LIt1QaTWE0Xu/jDwE6AzsNbMap3uLyJHUlF77L4In3Q3G/jnsG068JCZrQES2iwykeOAu38G/BJYCWwE3nL3F939E4L71dcBy4F3AN3IJ9IEyjNpTuXl5fTo0YOkpCS2bNnC2rVrj+rTt29fysrKeP/99wGYP38+2dnZjd7WyJEjeeyxx6qL0Q0bNgDQrVs3vvrqq3r71TRs2DDy8vKorKxkx44dFBQUkJWV1ei46rAFONPMksP3V1ctMLOz3L3U3R8BiqjjHnYROZLuqW0Ady8DUqLez6infyFH3nN0d8tEJnJ8ipFzfwT+GKPrH919TngFaTHweutEKBL/lGfSUkaNGsXs2bNJTU3l3HPPZciQIUf1SUxM5Omnn2bcuHFUVFQwaNAgpkyZ0uht3X333dx6662kpqbi7iQnJ/Pyyy8zfPjw6unGv/zlL2vtV9MVV1xBYWEhaWlpmBnTp0/nW9/6VpN+D7G4+z4zuxlYamY7CWZDVLnVzIYTXNV9B3it2TYscpyzpkz/kObXqc/Z3mfi7wAoe/jSNo5GpHmYWbG7Z7bg+DOAEQT3rb8O/B+v40MtOs/iiT4TpC7tLc+g4bmm/9sST5or18ysq7vvDp+G/Diwzd0fbew4mZmZXlRUdKzhiLQrTc0zXakVkbjl7r9o6xhEjnfKM5Fmd4OZTQROBjYQPA1ZRI6BiloRERERaReuuOIKPvjgyIcJP/LII4wcObKNIoLS0lKuvfZagH5mVvX1jgfcfXBTxguvyjb6yqyI1E5FrYiIiIi0C4sXL27rEI4SiUQoKSnBzN5pyan+ItJ0evqxiIiIiIiIxC0VtSIiIiIiIhK3VNSKiIiIiIhI3NI9te1E5DvdKdJXG4i0KOWZSOtQromISGvSlVoRERERERGJWypqRUREREREJG6pqBUREREREZG4paJWRERERERE4paKWhEREREREYlbKmpFREREREQkbukrfdqJ0k/KSb7jlZjLyvS1CCLNoq48Ox7os0Lai8bkmv7fiojIsdKVWhEREREREYlbKmpFREREREQkbqmoFRERERERkbilolZERERERETilopaERERERERiVsqakVERERERCRuqagVERERERGRuKWiVkREREREROKWiloRERERERGJWypqRUREjnNmtrutYxAREWkpbVrUmtktZvaumS04xnEmmdm3o94/ZWb9jj3CJsWSaGZvmtlGM3vbzO5rizhEYklOTmbnzp1tHcYRzOx+MxvR1nGIiMjxoaioiFtuuSXmsqbuB7t27XqsYcVkZvlmltkig4ucQE5q4+3fDPzI3T+oajCzk9y9opHjTAI2A58CuPtPmi3CxjsAXOjuu82sI7DazF5z97VtGJMcpyorK0lISGjrMJrMzBLcfVpbxyFyojAzA6YDPwIceMDd88ysAzALyAY+IDjpPdfdF5lZGZAHDA+H+Sd3f7/VgxdpoMzMTDIzVSeKnEja7Eqtmc0GzgReMrNyM5tjZq8D/2FmyWb2VzN7K/xzftR6t5tZaXgl9GEzuwrIBBaYWYmZdY4+62Vm48P+m83skahxdpvZg+E4a82sdx2xjgvX32hmBWHbJDN70cyWmtlWM7sHwANV07w6hn+8lnFvNLMiMyuq3Ft+LL9OOU6NGTOGgQMH0r9/f+bMmQMEZ4unTZvG4MGDKSwsZMWKFWRkZBCJRJg8eTIHDhyoc8zHHnuMAQMGEIlE2LJlCwC7du1izJgxpKamMmTIEDZt2gTAvffey8SJE8nNzSU5OZkXXniB22+/nUgkwqhRozh48CBAo2IwszIzm2Zmq4FxZjYvzOOqZfeFeV9qZn3D9l5mtixs/4OZfWhmp5pZFzN7JczNzWZ2dYztKc9EDrsSSAfSgBHAb8ysT9ieDESAnwDn1Vjvf9w9i6Dw/V2sgZVr0txq2wdOnTqVgQMHMmLECN58801ycnI488wzeemllwDIz8/nxz/+MQCff/45ubm5ZGRkcNNNN+F++JDs2WefJSsri/T0dG666SYqKyvrC+m0cD+0wsx6AZhZengcucnMFptZj7A938weCWfvvWdmF4Ttnc3sT2H/PKBz2J4Q7g83h/u/f40VQHSe7dixo8m/W5HjTZsVte4+heDK6nDgUWAgcLm7/xPwD+Bidx8AXA3MBDCzHwFjgMHungZMd/dFQBEwwd3T3X1f1TbCKcmPABcS7MQHmdmYcHEXYG04TgFwQx3hTgNGhn1HR7VnARPCscdFFdIJZlYS/j2Wufu6Wn4Hc9w9090zE5K6N+C3JieauXPnUlxcTFFRETNnzuTzzz9nz549pKSksG7dOjIzM5k0aRJ5eXmUlpZSUVHBk08+WeeYp556Km+99RY//elPmTFjBgD33HMPGRkZbNq0iV//+tdcd9111f3/+7//m1deeYUXX3yRa665huHDh1NaWkrnzp155ZVX2L9/f6NjAPa7+1B3/1OMZTvD3H8S+EXYdg/wl7B9MXB62D4K+NTd09w9BVhaczDlmcgRhgIL3b3S3f8OrAIGhe3Pufshd/+/wMoa6y2M+lmz4AWUa9L8atsH5uTkUFxcTLdu3fjVr37FsmXLWLx4MdOmHT3x57777mPo0KFs2LCB0aNH89FHHwHw7rvvkpeXx5o1aygpKSEhIYEFC2q/G27Pnj0Ae8P90CqC/RLAfwBT3T0VKI1qBzgpPBl0a1T7T8NxUoEHCY5/ITiW/I67p7h7BHg6VhzRedarV696f4ciJ4r29KCol6IK0o7Av5tZKfAcUHV/7AjgaXffC+Duu+oZcxCQ7+47winNC4Bh4bKvgZfD18UEZ6hrswaYZ2Y3ANFzPZe5++dh3C8QHBQQHiykA6cBWWaWUk+cIjHNnDmTtLQ0hgwZwscff8y2bdtISEhg7NixAGzdupUzzjiDc845B4CJEydSUFBQ55hXXnklAAMHDqSsrAyA1atXc+211wJw4YUX8vnnn1NeHlxp+dGPfkTHjh2JRCJUVlYyatQoACKRCGVlZU2KgWAqY21eCH9G5+VQ4E8A7r4U+CJsLwVGhGfDL3B3XR4SqZs1sr2K1/JapMXE2geefPLJR+yHsrOzq/dRVfu0aAUFBVxzzTUAXHrppfTo0QMIZhgVFxczaNAg0tPTWbFiBdu3b681lg4dOgBUHXc+Cww1s+7AKe6+Kmx/hsPHmRB7fzYsXB933wRsCtu3A2ea2WNmNgr4nwb8ikQk1J6K2j1Rr/8V+DvB9KhM4OSw3WjczrSunfRBPzwHpZI67i8Oryr/CvguUGJm36xaVLNrjfW+BPIJriaJNEp+fj7Lly+nsLCQjRs3kpGRwf79+0lMTKy+jzZ6GlVDderUCYCEhAQqKipqHSe49e5w/w4dOtCxY8fq9g4dOlBRUdGkGDgy32uqmrscnZcxc9nd3yM4y10KPGRmuj9XpG4FwNXhjKJeBAfYbwKrgbFm1iG8HSenxnpXR/0sbK1g5cRV2z6w5n4oeh9VtU+rqap/NHdn4sSJlJSUUFJSwtatW7n33nsbE2JDdn6x9mcx13X3LwiOe/OBnwFPNSYYkRNdeypqo3UHPnP3Q8C1HL46+jow2cySAMysZ9j+FdAtxjjrgOzw3rsEYDzBlJFGMbOz3H1d+ECbnQTFLcDFZtbTzDoTTIteE977d0q4XmeCq8tbGrtNkfLycnr06EFSUhJbtmxh7dqjnzXWt29fysrKeP/94Jkt8+fPJzs7u9HbGjZsWPW0q/z8fE499VS+8Y1vNGjd5oqhHquB/w1gZrlA1T1L3yaYxvUsMAMY0NwbFjnOLCa4MrQR+Atwezjd+HngbwQPXfwDwf4zeuZDJzNbB/wfghPPIi2qIfvAhojev7322mt88UUw0eeiiy5i0aJF/OMf/wCCZ0t8+OGHtY5z6NAhCPc9wD8Bq8PZQV9U3S9LcMxa33FmAcGta4Qz+VLD16cCHdz9eeButD8TaZS2fvpxbZ4AnjezcQT39eyBYNqhmaUDRWb2NfAqcCcwD5htZvuIutfH3T8zs1+GYxjwqru/2IR4fmNmZ4djrCA4GEgnONCeD3wf+KO7F5lZKvBMWER3AP7T3V+uZVyRWo0aNYrZs2eTmprKueeey5AhQ47qk5iYyNNPP824ceOoqKhg0KBBTJkypdHbuvfee7n++utJTU0lKSmJZ555psHrNlcM9bgPWBg+CGoV8BnByawcgvw8BBwkuFdJRGpw967hTwduC/9ELz9kZr8In9z/TYKrt6VRXR53d31FnbSahuwDG+Kee+5h/PjxDBgwgOzsbE4/PXgkQ79+/XjggQfIzc3l0KFDdOzYkccff5zvfe97Mcfp0qULe/bs6WxmxQQnfKpmL0wkOAZNIphCfH09IT0JPG1mm4ASglwD+E7YXnXB6ZdN+guLnKCsiVMHT3hmNgnIdPefN8d4nfqc7X0mxnygJGUPX9ocmxBpdWZW7O7H/L0KZtYJqHT3CjM7D3gyvG+9UerKs+OBPitOTM2YZ/nAKQS3/Ex393lhexnB/q7BX+7ZmFzT/1uJF82Va80lMzPTi4qK2joMkWbV1Dxrr1dqRUSinQ78Z3gG+2vqflq5iDSBu+fU0p7cupGIiIg0joraKGZ2FzCuRvNz7v5gzb7hGex5rRCWSKNdccUVfPDBB0e0PfLII4wcObJVYwD6hV9vVWWqu/9XY8dy921ARnPFJiIiEsvgwYOP+q71+fPnE4lE2igiEWkIFbVRwuL1qAJWJN4sXry4rUNg8eLFmNk77WmqloiISF3WrVvX1iGISBO016cfi4iIiIiIiNRLRa2IiIiIiIjELRW1IiIiIiIiErdU1IqIiIiIiEjc0oOi2onId7pTpO/qE2lRyjOR1qFcExGR1qQrtSIiIiIiIhK3VNSKiIiIiIhI3FJRKyIiIiIiInFLRa2IiIiIiIjELRW1IiIiIiIiErf09ON2ovSTcpLveKXW5WV6iqTIMasvz04E+iyR1tCUXNP/TRERaSpdqRUREREREZG4paJWRERERERE4paKWhEREREREYlbKmpFREREREQkbqmoFRERERERkbilolZERERERETilopaERERERERiVsqakVERERERCRuqagVERERERGRuKWiVkREREREROKWiloRERGpl5nd2dYxiIiIxNImRa2ZlZnZqY3on25ml7RkTLVs943W3qZIc5s5cyY/+MEPmDBhwjGNM2/ePD799NPq9z/5yU945513jjW8JjGzRDN708w2mtnbZnZfmwQicmJRUSvNKjk5mZ07dwJw/vnn19ovPz+fH//4x40au+Y+qzWY2Wgzu6NVNyoiQDMVtWaW0Bzj1CEdaFRRa2YnNXVjVX8fd6/9E1YkTjzxxBO8+uqrLFiwoLqtoqKi0ePUPEB46qmn6NevX7PE2AQHgAvdPY3g82GUmQ1pq2BEjgdmdruZ3RK+ftTM/hK+vsjMFgGdzazEzBbUOZBIE7zxRvNeR2hKUduUfWMVMzvJ3V9y94ebPIiINFmDilozW2JmxeEVkRvDtt1mdr+ZrQPOC3d6G8ys1Mzmmlmneob9FzN7K+zfNxyzS7ju+nCsy83sZOB+4OpwZ3p1rH7h+pPM7Dkz+zPwugV+Y2abw+1cXcffMcfMVprZH4HSqr9j1LJ8M1tkZlvMbIGZWbjskrBttZnNNLOXw/bsMN6SMMZuMbZ5o5kVmVlR5d7yhvxTiDTKlClT2L59O6NHj6Z79+7ceOON5Obmct1111FWVsYFF1zAgAEDGDBgwBEHFNOnTycSiZCWlsYdd9zBokWLKCoqYsKECaSnp7Nv3z5ycnIoKioCYOHChUQiEVJSUpg6dWp0CBlm9mB4RXWtmfWuLVYzGxfm6kYzKwjbJpnZi2a21My2mtk9AB7YHa7aMfzjtYyrPBNpmALggvB1JtDVzDoCQ4FlwD53T3f3mNM+lGtSlzFjxjBw4ED69+/PnDlzjlretWvXOtffvXs3V111FX379mXChAm4Bx/5xcXFZGdnM3DgQEaOHMlnn30Wc58Vqx9ATk4Od955J9nZ2fz+97/nww8/5KKLLiI1NZWLLrqIjz76qNaYzGyemf3WzFYCj4T7rFlRy2aa2Rtmtt3MrgrbO5jZE+Ex9ctm9mrUsofN7B0z22RmM2rZZnWe7dixo/5fvMgJoqFXMye7+y4z6wysN7PngS7AZnefZmaJwDbgInd/z8z+A/gp8Ls6xtzp7gPM7GbgF8BPgLuAv7j7ZDM7BXgTWA5MAzLd/ecAZvbrmv3MbHk47nlAahjvWIKrOGnAqWHsBe7+WS0xZQEp7v5BjGUZQH/gU2AN8EMzKwL+AAxz9w/MbGFU/18AP3P3NWbWFdhfc0B3nwPMAejU5+yYB+Qix2L27NksXbqUlStXMmvWLP785z+zevVqOnfuzN69e1m2bBmJiYls27aN8ePHU1RUxGuvvcaSJUtYt24dSUlJ7Nq1i549ezJr1ixmzJhBZmbmEdv49NNPmTp1KsXFxfTo0YPc3FyWLFnCmDFjIDhxttbd7zKz6cANwAO1hDsNGOnun4R5XSULSAH2EuTwK+5eZMGMimLg+8Dj7r4u1qDKM5EGKwYGhidhDwBvERS3FwC31Leyck3qMnfuXHr27Mm+ffsYNGgQY8eObdT6GzZs4O233+bb3/42P/zhD1mzZg2DBw/mX/7lX3jxxRfp1asXeXl53HXXXcydO/eIfdbBgwdr7Qfw5ZdfsmrVKgAuu+wyrrvuOiZOnMjcuXO55ZZbWLJkSV2hnQOMcPdKM5tUY1kfgpNCfYGXgEXAlUAyEAH+F/AuMNfMegJXAH3d3WvsB6tF51lmZqbyTCTU0KL2FjO7Inz9XeBsoBJ4Pmw7F/jA3d8L3z8D/Iy6i9oXwp/FBAkOkAuMNrNfhO8TgdNjrFtXv2Xuvit8PRRY6O6VwN/NbBUwiOCDJZY3ayloq5b9DcDMSgg+kHYD26PWWQjcGL5eA/zWgmlaL1StK9KWRo8eTefOnQE4ePAgP//5zykpKSEhIYH33gvSd/ny5Vx//fUkJSUB0LNnzzrHXL9+PTk5OfTq1QuACRMmUFBQUFXUOvBy2LUYuLiOodYA88zsPzn8+QBBTn8OYGYvEOR1UZjX6eGOf7GZpbj75ob+LkTkSO5+0MzKgOuBN4BNwHDgLIIDb5EmmzlzJosXLwbg448/Ztu2bY1aPysri9NOOw2A9PR0ysrKOOWUU9i8eTMXXxzsWiorK+nTp89R627durXOfldffXgiX2FhIS+8EOyCrr32Wm6//fb6Qnsu3B/FssTdDwHvRM1UGhqucwj4v+FVXoD/IbgA8pSZvcLhfaeINEC9Ra2Z5QAjgPPcfa+Z5RMUkfujktiasO0D4c/KqDgMGOvuW2vEMLhmWHX021OjX2PsqWPZgajXVTHXOr67Pxx+KF0CrDWzEe6+pZHxiDSrLl26VL9+9NFH6d27Nxs3buTQoUMkJiYC4O6Es+sbpGoKWO2LqztE53qsjlPCHL4UKDGz9KpFNbvWWO/L8HNpFKCiVuTYFBDMNJpMcCvOb4Hi8MrRQTPr6O4H2zRCiTv5+fksX76cwsJCkpKSyMnJYf/+oyaw1alTp8N3tSUkJFBRUYG7079/fwoLC+tct75+0fvGmhqwP2zosaPV+HkEd68wsyzgIuD/AX4OXFjfxkUk0JB7arsDX4QFbV8g1sNYtgDJZvb98P21wKomxPNfBPfaVt2vmhG2fwV0a0C/mgoI7sVNMLNewDCCKc3NZQtwppklh++rT/WZ2VnuXurujwBFBFNPRNqN8vJy+vTpQ4cOHZg/fz6VlcE5qtzcXObOncvevXsB2LUrmPjQrVs3vvrqq6PGGTx4MKtWrWLnzp1UVlaycOFCsrOzGx1PmDPr3H0asJNgVgjAxWbWM7z9YQywxsx6VU3NCttHEOSjiBybvxJMmSx0978TXDn6a7hsDrDJ9KAoaaTy8nJ69OhBUlISW7ZsYe3atc0y7rnnnsuOHTuqi9WDBw/y9ttvA0fus+rqV9P555/Pn/70JwAWLFjA0KFDmyXWKKuBseG9tb2BHIDwVrXu7v4qcCvB7XMi0kANKWqXAieZ2Sbg34CjPoncfT/BdKXnzKwUOATMbkI8/0bwwJdNZrY5fA+wEugXPnTp6jr61bSYYPrURuAvwO3u/n+bEFdM7r4PuBlYamargb8DVU/HuLXqoTfAPuC15tquSHO4+eabeeaZZxgyZAjvvfde9ZnqUaNGMXr0aDIzM0lPT2fGjOBZFZMmTWLKlCnVD92o0qdPHx566CGGDx9OWloaAwYM4PLLL29KSL+x4IFumwlOSG0M21cD84ES4Hl3LyI46F4Zfi6tJ5iirKlaIsfI3Ve4e0d33xO+P8fdfxu+nuruP6jtQVEitRk1ahQVFRWkpqZy9913M2RI8zys/uSTT2bRokVMnTqVtLQ00tPTqx96GL3PqqysrLVfTTNnzuTpp58mNTWV+fPn8/vf/75ZYo3yPPA3gplFfwDWERw7dgNeDvdrq4B/be4NixzPrJ6pg1IPM+vq7rvDq8aPA9vc/dHGjtOpz9neZ2LttyCXPXzpMUQp0jbMrNjdM+vvWev6k4h6SNyxqi/PTgT6LDn+HGuetYSm5Jr+b0p711y5FnXs+E2CGYQ/bMpFl8zMTK/6FgKR40VT86zJ3+Uq1W4ws4nAycAGgrNuIiIiIiKxvBzeQnMy8G/NOYtQ5ETVokWtmS0GzqjRPNXd/6slt1sXM4sQTGWMdsDdaz6MqkHCq7KNvjIrcqIys7uAcTWan3P3B2v2dfd5wLxWCEtERNpYaWkp11577RFtnTp1Yt26mN/Y1moefPBBnnvuOQhvhQubY+63GsLdc5orNhEJtGhR6+5X1N+rdbl7Kbr5XqTNhAcBTToQEBGR41ckEqGkpKT+jq3srrvu4q677sLM3mlvU/1FJNCQB0WJiIiIiIiItEsqakVERERERCRuqagVERERERGRuKWnH7cTke90p0hfZyDSopRnIq1DuSYiIq1JV2pFREREREQkbqmoFRERERERkbilolZERERERETilopaERERERERiVsqakVERERERCRuqagVERERERGRuKWv9GknSj8pJ/mOV+qqUTzdAAAgAElEQVTsU6avRxA5Jg3JM9FnjRy71sw1/X8VERFdqRUREREREZG4paJWRERERERE4paKWhEREREREYlbKmpFREREREQkbqmoFRERERERkbilolZERERERETilopaERERERERiVsqakVERERERCRuqagVERERERGRuKWiVkTaBTNLNrPNMdrzzSyzAeuXmdmpLROdiIiIiLRX7aKobY8Ho2Z2v5mNaOs4RETk+FFUVMQtt9wSc1lycjI7d+5syrAZxxRULRp6QkmkOUXnwfnnn19rv/z8fH784x+3VlgNZmZvtHUMIieik1piUDNLcPfKlhi7NYTxT2vrOEROQAlm9u/A+cAnwOVh+zVmNhP4BjDZ3d80s28CC4FewJuAtUXAIo2RmZlJZqbqRJGGeOON+KkPq4593b32SlxEWkyTrtSa2RIzKzazt83sxrBtd3h1cx1wnpldZGYbzKzUzOaaWad6hv0XM3sr7N83HLNnuK1NZrbWzFLD9nvN7Bkzez28ynulmU0P111qZh3Dfg2OIRxnmpmtBsaZ2Twzuypq2X0x4utlZsvC9j+Y2YdmdqqZdTGzV8xso5ltNrOra9nmjWZWZGZFlXvLG/mvIHJcOht43N37A18CY8P2LuGBws3A3LDtHmC1u2cALwGnxxpQeSYtYcyYMQwcOJD+/fszZ84cALp27crUqVMZOHAgI0aM4M033yQnJ4czzzyTl156CTjy6tLnn39Obm4uGRkZ3HTTTbh79fjPPvssWVlZpKenc9NNN1FZWfd5YjP7/8J90Qoz6xW2pYf7zk1mttjMeoTt+Wb2iJm9aWbvmdkFYXtnM/tT2D8P6By2J4T7xM3hPvBfa4lBuSaNEiuPonXt2rXO9Xfv3s1VV11F3759mTBhQnUOrVixgoyMDCKRCJMnT+bAgQNAcBX4zjvv5LzzziMzM5O33nqLkSNHctZZZzF79mwA3J3bbruNlJQUIpEIeXl5tW7fzHLMbKWZ/REoDdt2Ry3LN7NFZrbFzBaYmYXLLgnbVpvZTDN7OWzPNrOS8M8GM+sWY5vVebZjx456f8ciJ4qmTj+e7O4DgUzglvCKSRdgs7sPBoqAecDV7h4huCL803rG3OnuA4AngV+EbfcBG9w9FbgT+I+o/mcBlxJcyXkWWBluax9wqZklNiGG/e4+1N3/1MD47gH+ErYv5vBB9SjgU3dPc/cUYGmsjbn7HHfPdPfMhKTu9YQmckL4wN1LwtfFQHL4eiGAuxcA3zCzU4BhBLmPu78CfBFrQOWZtIS5c+dSXFxMUVERM2fO5PPPP2fPnj3k5ORQXFxMt27d+NWvfsWyZctYvHgx06YdPfnnvvvuY+jQoWzYsIHRo0fz0UcfAfDuu++Sl5fHmjVrKCkpISEhgQULFtQVTgfgrXBftIpg3wTBPnNquA8tjWoHOMnds4Bbo9p/CuwN+z8IDAzb04HvuHtKuD99OlYQyjVprFh51BgbNmzgd7/7He+88w7bt29nzZo17N+/n0mTJpGXl0dpaSkVFRU8+eST1et897vfpbCwkAsuuIBJkyaxaNEi1q5dW52jL7zwAiUlJWzcuJHly5dz22238dlnn9UVRhZwl7v3i7EsgyDH+gFnAj8Mj0//APzI3YcSzDaq8gvgZ+6eDlxAcEx7hOg869WrV83FIiespha1t5jZRmAt8F2CqyuVwPPh8nMJDk7fC98/Q3AAWpcXwp/RB7JDgfkA7v4X4JtmVrWnfM3dDxLsqBM4XDiWhus3JYbaT8fVHt+fwviWcviguhQYEZ4Jv8DddcpapGEORL2u5PAtEl6jn9fSLtIqZs6cSVpaGkOGDOHjjz9m27ZtnHzyyYwaNQqASCRCdnY2HTt2JBKJUFZWdtQYBQUFXHPNNQBceuml9OjRAwiuMhUXFzNo0CDS09NZsWIF27dvry+kqv3Xs8DQcF95iruvCttr7gNj7dOiTxRtAjaF7duBM83sMTMbBfxPfcGINESsPGqMrKwsTjvtNDp06EB6ejplZWVs3bqVM844g3POOQeAiRMnUlBQUL3O6NGjgSBHBw8eTLdu3ejVqxeJiYl8+eWXrF69mvHjx5OQkEDv3r3Jzs5m/fr1dYXxprt/UMeyv7n7IaCEINf6Atuj1lkY1X8N8Fszu4Ugfysa9QsROYE1+p5aM8sBRgDnufteM8sHEgmuclbNj2rKvW1VB7PRB7Kxxqk6iD0A4O6HzOygH563dShcvykx7GmG+HD398xsIHAJ8JCZve7u9zchHhEJXA2sNLOhQLm7l5tZATABeMDMfgT0aNMI5YSRn5/P8uXLKSwsJCkpiZycHPbv30/Hjh0JZxfSoUMHOnXqVP26oiL2sWlV/2juzsSJE3nooYeaGmJDTvbE2qfFXNfdvzCzNGAk8DPgfwOTmxqcCNSeR41RlWMACQkJVFRUHDGNv651onO06n1D1o+hIceOcDjXaj0+dfeHzewVguPHtWY2wt23NDYgkRNRU67Udge+CAvavsCQGH22AMlm9v3w/bUEU6Iaq+qgtaqY3unuDT1D3Fwx1GU1wc4dM8slPKg2s28TTOF6FpgBDGjm7YqcaL6w4ImSs4F/DtvuA4aZ2VtALvBRWwUnJ5by8nJ69OhBUlISW7ZsYe3atU0aZ9iwYdXTil977TW++CKY7HPRRRexaNEi/vGPfwCwa9cuPvzww/qGuyr8+U8E95qXE+TNBWF7Q/aB0fvcFKDqORanAh3c/XngbrRPk2bQXHlUU9++fSkrK+P9998HYP78+WRnZzd4/WHDhpGXl0dlZSU7duygoKCArKysZokttIVg5kNy+L76uStmdpa7l7r7IwS38vVtzg2LHM+a8vTjpcAUM9sEbCWYgnwEd99vZtcDz5nZScB6goPRxroXeDrc1l5gYkNXbMYY6nIfsDB8ENQq4DPgKyAH+I2ZHQIOUv+9vCInPHcvA1Ki3s+op//nBMVslZgPrxFpbqNGjWL27NmkpqZy7rnnMmRIrHO79bvnnnsYP348AwYMIDs7m9NPDx7L0K9fPx544AFyc3M5dOgQHTt25PHHH+d73/tebUMdAvqbWTFQzuGD5InAbDNLIphCfH09IT3J4X1uCcFTxQG+E7ZXnQj/ZRP+uiJHaK48qikxMZGnn36acePGUVFRwaBBg5gyZUqD17/iiisoLCwkLS0NM2P69Ol861vfapbYANx9n5ndDCw1s50czjOAW81sOMFV3XeA15ptwyLHOWvCNAsJWfA05Up3rzCz84Anw5v7G61Tn7O9z8Tf1dmn7OFLmzK0SJsxs2J3bzffX9KQPBN91sSb9pZn0Lq5pv+v0lqaK9fMrKu77w6fhvw4sM3dH23sOJmZmV5UVHSs4Yi0K03Nsxb5ntoTyOnAf4Znr78GbmjjeERERESkfbvBzCYCJwMbCJ6GLCLHoFWLWjNbDJxRo3mqu/9XPMbg7tsIHtcuIiLSogYPHlz9fZtV5s+f30bRiLSO0tJSrr322iPaOnXqxLp169oihn5mVvW1cwfCr7FstPCqbKOvzIpI7Vq1qHX3K1pze+01BhERkcZqzYN4kfYiEolQUlJSf8dWiMHM3mlvU/1FJNDU76kVERERERERaXMqakVERERERCRuqagVERERERGRuKWiVkREREREROKWvtKnnYh8pztF+q49kRalPBNpHco1ERFpTbpSKyIiIiIiInFLRa2IiIiIiIjELRW1IiIiIiIiErdU1IqIiIiIiEjcUlErIiIiIiIicUtPP24nSj8pJ/mOV5q8fpmeMilSr2PNMzlMnzlSl7bINf2fFBE5celKrYiIiIiIiMQtFbUiIiIiIiISt1TUioiIiIiISNxSUSsiIiIiIiJxS0WtiIiIiIiIxC0VtSIiIiIiIhK3VNSKiIiIiIhI3FJRKyIiIiIiInFLRa2IiIiIiIjELRW1IiIiIiIiErdU1LYgM7uzrWMQOREo16QtJScns3Pnzgb3Lykp4dVXX23BiGIzszdafaMizWzmzJn84Ac/YMKECcc0zrx58/j000+r3//kJz/hnXfeOdbwmsTMEs3sTTPbaGZvm9l9bRKISBxTUduydKAt0jqUa9JsKisrW3T8phS1FRUVTd6emSUAuPv5TR5EpJ144oknePXVV1mwYEF1W1Pyo2ZR+9RTT9GvX79mibEJDgAXunsakA6MMrMhbRWMSDxSUXsMzOx2M7slfP2omf0lfH2RmS0COptZiZktqHMgEamTck2a05gxYxg4cCD9+/dnzpw5AHTt2pVp06YxePBgCgsLWbFiBRkZGUQiESZPnsyBAwfqHPOxxx5jwIABRCIRtmzZAsCePXuYPHkygwYNIiMjgxdffJGvv/6aadOmkZeXR3p6Onl5eTH7QXDQPW7cOC677DJyc3Nxd2677TZSUlKIRCLk5eXVGo+Z5ZjZSjP7I1Aatu2OWpZvZovMbIuZLTAzC5ddEratNrOZZvZy2J4d5liJmW0ws27H9q8g0nhTpkxh+/btjB49mu7du3PjjTeSm5vLddddR1lZGRdccAEDBgxgwIABvPHG4YkJ06dPJxKJkJaWxh133MGiRYsoKipiwoQJpKens2/fPnJycigqKgJg4cKFRCIRUlJSmDp1anQIGWb2YHhFda2Z9a4tVjMbZ2abw74FYdskM3vRzJaa2VYzuwfAA7vDVTuGf7xZf3kix7mT2jqAOFcA/L/ATCAT6GRmHYGhwDJglLun17aymd0I3AiQ8I1eLR+tSPxqcq4pz6SmuXPn0rNnT/bt28egQYMYO3Yse/bsISUlhfvvv5/9+/dz9tlns2LFCs455xyuu+46nnzySW699dZaxzz11FN56623eOKJJ5gxYwZPPfUUDz74IBdeeCFz587lyy+/JCsrixEjRnD//fdTVFTErFmzALjzzjtj9gMoLCxk06ZN9OzZk+eff56SkhI2btzIzp07GTRoEMOGDavrr5oFpLj7BzGWZQD9gU+BNcAPzawI+AMwzN0/MLOFUf1/AfzM3deYWVdgf80BlWvS0mbPns3SpUtZuXIls2bN4s9//jOrV6+mc+fO7N27l2XLlpGYmMi2bdsYP348RUVFvPbaayxZsoR169aRlJTErl276NmzJ7NmzWLGjBlkZmYesY1PP/2UqVOnUlxcTI8ePcjNzWXJkiWMGTMGgotBa939LjObDtwAPFBLuNOAke7+iZmdEtWeBaQAe4H1ZvaKuxeFMyqKge8Dj7v7uliDRufZ6aef3tRfpchxR1dqj00xMDA8Y30AKCQ44L4A+Gt9K7v7HHfPdPfMhKTuLRupSHxrcq4pz6SmmTNnkpaWxpAhQ/j444/Ztm0bCQkJjB07FoCtW7dyxhlncM455wAwceJECgoK6hzzyiuvBGDgwIGUlZUB8Prrr/Pwww+Tnp5OTk4O+/fv56OPPjpq3br6XXzxxfTs2ROA1atXM378eBISEujduzfZ2dmsX7++rrDerKWgrVr2N3c/BJQAyUBfYHvUOtFF7Rrgt+GMiVPc/aj5nso1aW2jR4+mc+fOABw8eJAbbriBSCTCuHHjqu+PXb58Oddffz1JSUkA1flUm/Xr15OTk0OvXr046aSTmDBhQnT+O/By+LqYIG9qswaYZ2Y3AAlR7cvc/XN33we8QHByFnevDE/OngZkmVlKrEGj86xXL508EqmiK7XHwN0PmlkZcD3wBrAJGA6cBbzbhqGJHFeUa9Jc8vPzWb58OYWFhSQlJVUXkYmJiSQkBMed7o2f9depUycAEhISqu/vc3eef/55zj333CP6rlt35AWYuvp16dLliH6NtKeOZdHzqSsJjgests7u/rCZvQJcAqw1sxHuvqWxAYk0p+j8ePTRR+nduzcbN27k0KFDJCYmAkHehLPrG6SePHM/3KEqb2rrOMXMBgOXAiVmVjWbqOYGvMZ6X5pZPjAK2NzgwEVOcLpSe+wKCKZlFRBcMZoClIQfegfDKZIicuyUa3LMysvL6dGjB0lJSWzZsoW1a9ce1adv376UlZXx/vvvAzB//nyys7Mbva2RI0fy2GOPVR8kb9iwAYBu3brx1Vdf1duvpmHDhpGXl0dlZSU7duygoKCArKysRsdVhy3AmWaWHL6/umqBmZ3l7qXu/ghQRHBVV6TdKC8vp0+fPnTo0IH58+dXP/AtNzeXuXPnsnfvXgB27doFHJ2HVQYPHsyqVavYuXMnlZWVLFy4sEn5H+bMOnefBuwEvhsuutjMeppZZ2AMsMbMelVNUQ7bRxDko4g0kIraY/dXoA9Q6O5/J7jPqGo65Bxgkx5eI9IslGtyzEaNGkVFRQWpqancfffdDBly9ANGExMTefrppxk3bhyRSIQOHTowZcqURm/r7rvv5uDBg6SmppKSksLdd98NwPDhw3nnnXeqHxRVW7+arrjiClJTU0lLS+PCCy9k+vTpfOtb32p0XLUJp0PeDCw1s9XA34HycPGtVQ+9AfYBrzXbhkWawc0338wzzzzDkCFDeO+996qv4o4aNYrRo0eTmZlJeno6M2bMAGDSpElMmTKl+kFRVfr06cNDDz3E8OHDSUtLY8CAAVx++eVNCek3ZlZqZpsJTsZuDNtXA/MJpv0/7+5FBPu2lWa2CVhPMEX55ViDikhs1pRpVtL8OvU52/tM/F2T1y97+NJmjEakeZhZsbtn1t+zdRxrnslh+sxpP5ozz8ysq7vvDp+G/Diwzd0fbew4bZFr+j8pLe1Yc83MJgGZ7v7z5ognMzPTq57YLHK8aGqe6UqtiIiIVLnBzEqAt4HuBE9DFhERadf0oCgREZEGuOKKK/jggyMfJvzII48wcuTINooISktLAfqFhWiVA+4+uCnjhVdlG31lVuREZWZ3AeNqND/n7g/W7Ovu84B5rRCWyAlHRa2IiEgDLF68uK1DOEokEgF4pz1N8xc5kYTF61EFrIi0Lk0/FhERERERkbilolZERERERETilopaERERERERiVsqakVERERERCRu6UFR7UTkO90p0nfsibQo5ZlI61CuiYhIa9KVWhEREREREYlbKmpFREREREQkbqmoFRERERERkbilolZERERERETilopaERERERERiVsqakVERERERCRu6St92onST8pJvuOVJq9fpq9OEKnXseaZHE2fPRKLcu3EoPwXkfZCV2pFREREREQkbqmoFRERERERkbilolZERERERETilopaERERERERiVsqakVERERERCRuqagVERERERGRuKWiVkREREREROKWiloRERERERGJWypqRUREREREJG6pqK2Dme1u6xhEjnfKM5HWZWbJZrY5Rnu+mWW2RUwiJyIz+7aZLfr/27v3oMnq+s7j7w8gEBQGhNFiURyNQC0MiRBCScJS7JoliDGmhNoJ0YpGN4R42Y2uSWFZZFUsCyS7sTYSDKZYyA0TJNRSqBE3AUcugoNcZhCQ22SRm5eQAdfoJsx3/+jfIz3NMzP9zNO30/N+VXX16V+f7vP9ndPf83u+fU6fp03/+yS3Jlnf7v/dtOOTumS3aQcgSZIk7Wyq6lHgtPbwO8Drq+rRJKuBLwAHTS04qWM8UjuE9JyfZEP7Bm1Na98lyR8muSvJ1Uk+l+S09tzGJOcluaXdXjndXkizzTyTJmrXJJ9qeXVNkh9r7W9OcmPLw2MBkhzb2m5r94dNMW5pprQzH+5JcmmSO5N8JslebXz6aJKbkqxLcnSSLyR5IMmZfa/dAFBVt7UiF+AuYM8ke0yrX1LXWNQO543Aq4CfBH4OOD/Jga19FXAk8B+B4wZe91RVHQt8Avj44JsmOaPt6NY98/1NYwxf6gTzTJqcQ4ALquoI4B+BU1v786vqZ4B3ABe3tnuAE6rqKOB3gY8u9obmmnZihwEXVdVPAE/Ryx+Ah6vqOODLwCX0jsq+Gvjwdt7vVOC2qvrh4BP9efbtb397VPFLnWdRO5zjgcuq6pmqegL4EvDTrf3yqtpcVY8D1w687rK++8E/xKmqi6rqmKo6Zte9VowxfKkTzDNpch6qqtvb9K30vjiClk9VtRbYJ8m+wArg8nZE6feBIxZ7Q3NNO7GHq+qGNv1n9MYtgKva/Xrg5qp6uqq+Dfyg5dZzJDkCOA/4jcWe78+zlStXjq4HUsdZ1A4nS2xfUFuZlvRc5pk0Of1HgJ7h2WtsDOZQAecA11bVauD1wJ7jD0/qlMXyBp7Ns81smXObWeS6NkleAlwJ/GpVPTDqIKV5ZlE7nLXAmiS7JlkJnADcAlwPnNp+8/di4MSB163pu79pUsFKHWWeSdO38Fv244FNVbWJ3pHaR9rzb51SXNIsOzjJwplCp9Mbt5akHbn9LPD+vqO+kobk1Y+HcyW90xrvoPft2+9U1eNJrgBeA2wAvgHcDPT/kGiPJDfT+/Lg9MmGLHWOeSZN35NJbgT2Ad7W2j4GXJrkvcDfTS0yaXbdDbwlyR8B9wEXAu9e4nu8C3glcHaSs1vbSVX1rdGFKc0vi9ptqKoXtPsCfrvd+p/fnOR9VfW9JPvTO6q0vm+WC6rqQxMLWOog80yarKraCKzue/x725n/JuDQvqaztzavtJPaXFVnDrStWpioqkvoXShq4fHCc9+h5WJVfQT4yBhjlOaaRe3yXd1OGdkdOKddyEbSaJlnkiRJWpRF7TJV1YlbaV812Uik+WWeSZJm0eCZD5KmwwtFSZIkSZI6y6JWkiRJktRZFrWSJEmSpM6yqJUkSZIkdZZFrSRJkiSps7z68Yw48qAVrDv3ddMOQ5pr5pk0GeaaJGmSPFIrSZIkSeosi1pJkiRJUmdZ1EqSJEmSOsuiVpIkSZLUWRa1kiRJkqTOsqiVJEmSJHWW/9JnRqx/ZBOrzvrsRJe50X+3oJ3MNPJM4+H+a7aZazsPc3F61j+yadohSDPDI7WSJEmSpM6yqJUkSZIkdZZFrSRJkiSpsyxqJUmSJEmdZVErSZIkSeosi1pJkiRJUmdZ1EqSJEmSOsuiVpIkSZLUWRa1kiRJkqTOsqgdQpJVSTYs0n5dkmOGeP3GJAeMJzppfiw31yRtn2OaNBuS/Kskn2nT+ye5Nsn3knxi2rFJXbPbtAOQJEmSdjZV9ShwWnv4A+BsYHW7SVoCj9QOb9ckn0pyV5JrkvxYa39zkhuTbEhyLPzo27ZrktyW5I+ATC9sqXOWkmvHtrbb2v1hU4xb6hLHNGkE2pkP9yS5NMmdST6TZK92RsNHk9yUZF2So5N8IckDSc7se+0GgKr6v1V1Pb3iVtISWdQO7xDggqo6AvhH4NTW/vyq+hngHcDFre2/AtdX1VHAVcDBi71hkjPajm7dM9/fNN7ope5YSq7dA5zQcu13gY8Ovpl5Ji3KMU0ancOAi6rqJ4Cn6OUPwMNVdRzwZeASekdlXw18eEcXZJ5Ji/P04+E9VFW3t+lbgVVt+jKAqlqbZJ8k+wInAG9s7Z9N8uRib1hVFwEXAexx4CE1xtilLllKru0NXJrkEKCA5w2+mXkmLcoxTRqdh6vqhjb9Z8B/atNXtfv1wAuq6mng6SQ/aLm1ZOaZtDiP1A7vh33Tz/DsFwKDO5TaSruk4Swl184Brq2q1cDrgT3HH540FxzTpNHZWt4s5Nlmtsy5zXhgSRopi9rlWwOQ5HhgU1VtAtYCb2rtrwX2m1540txYLNdWAI+05986pbikeeKYJi3dwUmOa9OnA9dPMxhpZ+S3RMv3ZJIbgX2At7W2DwGXJfka8CXg/0wrOGmOLJZrH6N3+vF7gb+bWmTS/HBMk5bubuAt7UJq9wEXAu9e6psk2Ugv93ZP8kvASVX19VEGKs0ri9ohVNVG+i6vXlW/t535vwuc1Nf0nvFEJs2XHci1m4BD+5rOHk9k0vxwTJNGbnNVnTnQtmphoqouoXehqIXHC899hy1z8UevkbQ0nn4sSZIkSeosj9RKkiRJO2DwzAdJ0+GRWkmSJElSZ1nUSpIkSZI6y6JWkiRJktRZFrWSJEmSpM6yqJUkSZIkdZZFrSRJkiSps/yXPjPiyINWsO7c1007DGmumWfSZJhr0vgdedCKaYcgzQyP1EqSJEmSOsuiVpIkSZLUWRa1kiRJkqTOsqiVJEmSJHWWRa0kSZIkqbO8+vGMWP/IJlad9dlphyENbWMHr2xqnqmLzDVp/MwzafzGmWceqZUkSZIkdZZFrSRJkiSpsyxqJUmSJEmdZVErSZIkSeosi1pJkiRJUmdZ1EqSJEmSOsuiVpIkSZLUWRa1kiRJkqTOsqiVJEmSJHWWRa0kSZIkqbMsarciyb5J3jHtOKR5Zp5J42eeSZOT5HvTjkHaGVnUbt2+gH8ESONlnknjZ55JkuaaRe3WnQv8eJLbk3w1ydokVyb5epJPJtklya5JLkmyIcn6JO8BSPLr7TV3JLkiyV5T7os0q8wzafzMM2nC0nN+X06tae27JPnDJHcluTrJ55Kc1p7bmOS8JLe02yun2wupO3abdgAz7CxgdVW9KsmJwN8AhwN/36bfCDwEHFRVq6F3ild77V9X1ada20eAtwN/MLiAJGcAZwDsus/KsXZGmlHmmTR+Y8+z9ry5Jj3rjcCrgJ8EDgC+mmQt8LPAKuBI4EXA3cDFfa97qqqOTfKrwMeBX+h/U/NMWpxHaod3S1U9WFXPAJcBxwMPAq9I8gdJTgaeavOuTvLlJOuBNwFHLPaGVXVRVR1TVcfsuteKSfRBmnXmmTR+I88zMNekAccDl1XVM1X1BPAl4Kdb++VVtbmqHgeuHXjdZX33xw2+qXkmLc6idng1+LiqnqT3Ddx1wDuBP27PXQK8q6qOBD4E7DmhGKWuM8+k8TPPpPHLEtsX1FamJW2DRe3WPQ3s3ff42CQvT7ILsAa4PskBwC5VdQVwNnB0m3dv4LEkz6P3zbakxZln0viZZ9LkrQXWtN+rrwROAG4BrgdObb+tfTFw4u4e/SgAAAqrSURBVMDr1vTd3zSpYKWu8ze1W1FV301yQ5INwD/R27GcS+83EGuBK9v0/2x/GAC8v92fDdxM7/dK69nyjwlJjXkmjZ95Jk3FlfROH76D3hHX36mqx5NcAbwG2AB8g15+bep73R5JbqZ34On0yYYsdZdF7TZU1a8AtAtrvK+q1gzMcgfPfpvd/7oLgQvHHqA0B8wzafzMM2kyquoF7b6A3263/uc3J3lfVX0vyf70jt6u75vlgqr60MQCluaERa0kSZI0OVe3K4zvDpzTLhglaRksaodQVdfRu3iGpDExz6TxM8+k6auqE7fSvmqykUjzwwtFSZIkSZI6y6JWkiRJktRZFrWSJEmSpM6yqJUkSZIkdZZFrSRJkiSps7z68Yw48qAVrDv3ddMOQ5pr5pk0GeaaNH7mmfQsj9RKkiRJkjrLolaSJEmS1FkWtZIkSZKkzrKolSRJkiR1lkWtJEmSJKmzLGolSZIkSZ1lUStJkiRJ6iyLWkmSJElSZ1nUSpIkSZI6y6JWkiRJktRZFrWSJEmSpM6yqJUkSZIkdZZFrSRJkiSpsyxqJUmSJEmdZVErSZIkSeosi1pJkiRJUmdZ1EqSJEmSOsuiVpIkSZLUWRa1kiRJkqTOsqiVJEmSJHWWRa0kSZIkqbMsaiVJkiRJnWVRK0mSJEnqLItaSZIkSVJnpaqmHYOAJE8D9047jjE7APjOtIMYM/u4pZdV1cpxBrMUc5pn8/iZm7c+jbs/M5VnMBO5Nu3P0LSXPwsxzOPyZyrXZiDPBk17my9m1mIynm07AHj+juTZbmMIRjvm3qo6ZtpBjFOSdfax+zrex7nLs45vj0XNW5/mrT9DmmquTXudT3v5sxDDzr78CZmpMW0W1/msxWQ829biWbUjr/X0Y0mSJElSZ1nUSpIkSZI6y6J2dlw07QAmwD7Ohy73scuxb419mn3z1p9hTLvPO/vyYfox7OzLn4RZ6+OsxQOzF5PxbNsOx+OFoiRJkiRJneWRWkmSJElSZ1nUSpIkSZI6y6J2wpKcnOTeJPcnOWuR5/dI8pft+ZuTrJp8lMszRB/fm+TrSe5M8rdJXjaNOJdje33sm++0JJVkZi6XPqxh+pjkP7RteVeSv5h0jFszj3k2j3k1b3nU5ZwZlWG36TKX8dIk1ya5u63H/9zaP5jkkSS3t9spfa95f4vp3iQ/P6I4NiZZ35a1rrW9MMkXk9zX7vdr7UnyP1oMdyY5epnLPqyvn7cneSrJb417HSS5OMm3kmzoa1tyn5O8pc1/X5K3LHP55ye5py3jyiT7tvZVSf6pb118su81P9W23f0txuzI+piUWRvTZm08msWxZNbGgyG22cFtv3pb226nLPY+I4znObk88PzS95lV5W1CN2BX4AHgFcDuwB3A4QPzvAP4ZJv+ZeAvpx33GPr4b4G92vRvzmMf23x7A2uBrwDHTDvuMWzHQ4DbgP3a4xdNO+4lxN6pPJvHvJq3POpyzkx6m45gOQcCR/d9Pr4BHA58EHjfIvMf3mLZA3h5i3HXEcSxEThgoO1jwFlt+izgvDZ9CvB5IMCrgZtHvN4fB1427nUAnAAcDWzY0T4DLwQebPf7ten9lrH8k4Dd2vR5fctf1T/fwPvcAhzXYvs88NpJ5Mgytu/MjGlDxjOx8WjY/Q4THEuGXEcTGw+GjOci4Dfb9OHAxjGvo+fk8sDzS95neqR2so4F7q+qB6vq/wGfBt4wMM8bgEvb9GeA18z6N4gDttvHqrq2qr7fHn4FeMmEY1yuYbYjwDn0BvsfTDK4ERmmj78OXFBVTwJU1bcmHOPWzGOezWNezVsedTlnRmXYbbosVfVYVX2tTT8N3A0ctI2XvAH4dFX9sKoeAu5vsY5D/77lUuCX+tr/pHq+Auyb5MARLfM1wANV9ffbiWvZ66Cq1gL/sMh7L6XPPw98sar+oeXCF4GTd3T5VXVNVf1Le7jdfV+LYZ+quql6fz3/SV/Ms2jWxrRZG49mcSyZtfFgmHgK2KdNrwAeHWM8W9uX9FvyPtOidrIOAh7ue/xNnjsQ/2ietpPeBOw/kehGY5g+9ns7vW9iumS7fUxyFPDSqrp6koGN0DDb8VDg0CQ3JPlKkqH+KJmAecyzecyrecujLufMqCz1c7ps7TTLo4CbW9O72qlqFy+cBjvGuAq4JsmtSc5obS+uqsegV3wDLxpzDNA7MndZ3+NJrgNYep/HGcvb2HLf9/J2OuWXkvybvri+Oablj8OsjWmzNh7N4lgya+PBMPF8EHhzkm8CnwPePcZ4hrHk/YRF7WQt9q3Z4P9UGmaeWTZ0/EneDBwDnD/WiEZvm31Msgvw+8B/mVhEozfMdtyN3ukzJwKnA3+88FumKZvHPJvHvJq3POpyzozKRPMqyQuAK4DfqqqngAuBHwdeBTwG/Lcxx/WzVXU08FrgnUlO2Fa444ghye7ALwKXt6ZJr4NthreVZY5rXXwA+Bfgz1vTY8DBVXUU8F7gL5LsM67lj9GsjWmzNh7N4lgya+PBMPGcDlxSVS+hd+rvn7Z1Ny1L/kxb1E7WN4GX9j1+Cc89vP+jeZLsRu8UgG0dnp81w/SRJD8HfAD4xar64YRiG5Xt9XFvYDVwXZKN9H4LcNUkLkwwQsN+Vv9XVf1zO53tXno76Gmbxzybx7yatzzqcs6MylCf01FI8jx6Be2fV9VfA1TVE1X1TFVtBj7Fs6fXjiWuqnq03X8LuLIt74mFU+Ta/cIpheNaN68FvlZVT7RYJroOmqX2eeSxpHexqV8A3tROKaadav3dNn0rvd8UHtqW33867Ng+pyMya2ParI1HsziWzNp4MEw8bwf+CqCqbgL2BA4YUzzDWPp+Yns/uvU20h9F70bvgggv59kfah8xMM872fLH/n817bjH0Mej6A0uh0w73nH1cWD+65jhC9wsYzueDFzapg+gd5rI/h2JvVN5No95NW951OWcmdY2XcZyQu93kB8faD+wb/o99H5DCnAEW14k6UGWeaEo4PnA3n3TN7btez5bXjTpY236dWx50ZNbRrQuPg382iTXAQMXYFpqn+ldIOoheheJ2q9Nv3AZyz8Z+DqwcmC+lQt9pHeBnEcWlgN8tcW0cKGoUyaZK0tc3zM1pg0Zz8TGo6Xud5jAWDLkOprYeDBkPJ8H3tqm/zW9AjJjXk9b5PLAc0veZ471g+Zt0Y10Cr0rNT4AfKC1fZjeN1nQ+2bkcnoXcbgFeMW0Yx5DH/838ARwe7tdNe2YR93HgXnHvgOd0nYM8N/p/TGxHvjlace8hNg7l2fzmFfzlkddzplxroMxLON4eqeh3dn3eT8F+NO2Xu8ErmLLAu8DLaZ7GcGVbukVSXe0211923t/4G+B+9r9QhEV4IIWw/pRfJaBvYDvAiv62sa6Duj9dvcx4J/pHUl5+470md5vX+9vt19b5vLvp1cQLHwWFoq7U9u2uQP4GvD6vvc5BtjQYvsEY/7jfQTbeqbGtCHimeh4tL14Bua9bhT5N4J1NNHxYIh4DgduaPlyO3DSmONZLJfPBM7sWz9L2memvVCSJEmSpM7xN7WSJEmSpM6yqJUkSZIkdZZFrSRJkiSpsyxqJUmSJEmdZVErSZIkSeosi1pJkiRJUmdZ1EqSJEmSOuv/A0gnIe7+iN85AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1080x720 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, imp_plot = plt.subplots(nrows=1, ncols=4, figsize=(15, 10))\n",
    "imp_plot[0].barh(importances.nlargest(15).index.values, importances.nlargest(15))\n",
    "imp_plot[0].set_title('Dummy Model')\n",
    "imp_plot[1].barh(scores_mutual.nlargest(15).index.values, scores_mutual.nlargest(15))\n",
    "imp_plot[1].set_title('Mutual Information')\n",
    "imp_plot[2].barh(scores_f.nlargest(15).index.values, scores_f.nlargest(15))\n",
    "imp_plot[2].set_title('F Regression')\n",
    "imp_plot[3].barh(importances_lasso.nlargest(15).index.values, importances_lasso.nlargest(15))\n",
    "imp_plot[3].set_title('LASSO')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exporting feature importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T10:10:32.385306Z",
     "start_time": "2020-01-29T10:10:32.369265Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['models/RDKit_VPAS/mutual_importances.joblib']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(importances, \"models/RDKit_VPAS/model_importances.joblib\")\n",
    "joblib.dump(importances_lasso, \"models/RDKit_VPAS/lasso_importances.joblib\")\n",
    "joblib.dump(scores_f, \"models/RDKit_VPAS/freg_importances.joblib\")\n",
    "joblib.dump(scores_mutual, \"models/RDKit_VPAS/mutual_importances.joblib\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GradientBoosting\n",
    "### Model-chosen features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T10:10:32.643310Z",
     "start_time": "2020-01-29T10:10:32.387311Z"
    }
   },
   "outputs": [],
   "source": [
    "gbr_param_grid = {\n",
    "    'n_estimators': [500, 1000, 2000],\n",
    "    'max_depth': [2, 4, 6],\n",
    "    'min_samples_leaf': [3, 5, 9, 17],\n",
    "    'learning_rate': [0.1, 0.05, 0.02],\n",
    "    'max_features': [1.0, 0.3, 0.1],\n",
    "    'loss': ['ls', 'lad', 'huber']\n",
    "} #hyperparameters for each gbr grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T11:09:10.984864Z",
     "start_time": "2020-01-29T10:10:32.645313Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GradientBoostingRegressor grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'learning_rate': 0.05, 'loss': 'lad', 'max_depth': 6, 'max_features': 0.3, 'min_samples_leaf': 5, 'n_estimators': 2000}\n",
      "Grid search complete!\n",
      "Hyperparameters:\n",
      " {'learning_rate': 0.05, 'loss': 'lad', 'max_depth': 6, 'max_features': 0.3, 'min_samples_leaf': 5, 'n_estimators': 2000}\n",
      "GradientBoostingRegressor run at: 2020-01-29 11:09:10.801377 . Saving to models/RDKit_VPAS/GradientBoostingRegressor_model_29_01_2020_11_09_10.joblib\n"
     ]
    }
   ],
   "source": [
    "gbr_model, gbr_hp = train_model(model = GradientBoostingRegressor,\n",
    "           feature_set= top_15_features_model,\n",
    "            grid_search=True,\n",
    "           hyperparameters=gbr_param_grid)\n",
    "\n",
    "exp_vs_calc_gbr_model = produce_exp_vs_pred_df('top_15_features_model', gbr_model,\n",
    "                                         \"GradBoost, model-chosen features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### F-regression features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T11:09:24.793579Z",
     "start_time": "2020-01-29T11:09:10.990880Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No grid search\n",
      "Hyperparameters:\n",
      " {'learning_rate': 0.05, 'loss': 'lad', 'max_depth': 6, 'max_features': 0.3, 'min_samples_leaf': 5, 'n_estimators': 2000}\n",
      "GradientBoostingRegressor run at: 2020-01-29 11:09:24.662229 . Saving to models/RDKit_VPAS/GradientBoostingRegressor_kbest_29_01_2020_11_09_24.joblib\n"
     ]
    }
   ],
   "source": [
    "gbr_freg = train_model(model = GradientBoostingRegressor,\n",
    "           feature_set= top_15_features_kbest,\n",
    "            grid_search=False,\n",
    "           hyperparameters=gbr_hp)\n",
    "\n",
    "exp_vs_calc_gbr_freg = produce_exp_vs_pred_df('top_15_features_kbest', gbr_freg,\n",
    "                                                    \"GradBoost, f_reg features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mutual info features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T11:09:37.983647Z",
     "start_time": "2020-01-29T11:09:24.796588Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No grid search\n",
      "Hyperparameters:\n",
      " {'learning_rate': 0.05, 'loss': 'lad', 'max_depth': 6, 'max_features': 0.3, 'min_samples_leaf': 5, 'n_estimators': 2000}\n",
      "GradientBoostingRegressor run at: 2020-01-29 11:09:37.859318 . Saving to models/RDKit_VPAS/GradientBoostingRegressor_mutual_info_29_01_2020_11_09_37.joblib\n"
     ]
    }
   ],
   "source": [
    "gbr_mu = train_model(model = GradientBoostingRegressor,\n",
    "           feature_set= top_15_features_mutual_info,\n",
    "            grid_search=False,\n",
    "           hyperparameters=gbr_hp)\n",
    "\n",
    "exp_vs_calc_gbr_mu = produce_exp_vs_pred_df('top_15_features_mutual_info', gbr_mu,\n",
    "                                            \"GradBoost, Mutual Info\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LASSO features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T11:09:51.497603Z",
     "start_time": "2020-01-29T11:09:37.985652Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No grid search\n",
      "Hyperparameters:\n",
      " {'learning_rate': 0.05, 'loss': 'lad', 'max_depth': 6, 'max_features': 0.3, 'min_samples_leaf': 5, 'n_estimators': 2000}\n",
      "GradientBoostingRegressor run at: 2020-01-29 11:09:51.225855 . Saving to models/RDKit_VPAS/GradientBoostingRegressor_lasso_29_01_2020_11_09_51.joblib\n"
     ]
    }
   ],
   "source": [
    "gbr_lasso = train_model(model = GradientBoostingRegressor,\n",
    "           feature_set= top_15_features_lasso,\n",
    "            grid_search=False,\n",
    "           hyperparameters=gbr_hp)\n",
    "exp_vs_calc_gbr_lasso = produce_exp_vs_pred_df('top_15_features_lasso', gbr_lasso,\n",
    "                                               \"LASSO-features, GBR model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decision Tree\n",
    "### Model-Chosen Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T11:09:51.507604Z",
     "start_time": "2020-01-29T11:09:51.499583Z"
    }
   },
   "outputs": [],
   "source": [
    "dtr_param_grid = {\n",
    "    'criterion': ['mse', 'friedman_mse', 'mae'], #function measuring quality of a split\n",
    "    'max_depth': [5, 10, 20] , # max depth of tree\n",
    "    'max_features': [5, 10, 15], # N features to be considered when looking for split\n",
    "    'max_leaf_nodes': [None, 10, 15, 20, 30], #grows tree with N nodes in best-first fashion\n",
    "    'min_impurity_decrease': [0.0, 0.1],\n",
    "    'min_samples_leaf': [1, 2, 3], \n",
    "    'min_samples_split': [1.0, 2, 3],\n",
    "    'min_weight_fraction_leaf': [0.0, 0.1, 0.5] , \n",
    "    'splitter': ['best', 'random']\n",
    "} #hyperparameters for dtr gridsearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T11:13:56.946985Z",
     "start_time": "2020-01-29T11:09:51.510614Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DecisionTreeRegressor grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'criterion': 'friedman_mse', 'max_depth': 20, 'max_features': 15, 'max_leaf_nodes': None, 'min_impurity_decrease': 0.1, 'min_samples_leaf': 3, 'min_samples_split': 2, 'min_weight_fraction_leaf': 0.0, 'splitter': 'random'}\n",
      "Grid search complete!\n",
      "Hyperparameters:\n",
      " {'criterion': 'friedman_mse', 'max_depth': 20, 'max_features': 15, 'max_leaf_nodes': None, 'min_impurity_decrease': 0.1, 'min_samples_leaf': 3, 'min_samples_split': 2, 'min_weight_fraction_leaf': 0.0, 'splitter': 'random'}\n",
      "DecisionTreeRegressor run at: 2020-01-29 11:13:56.936958 . Saving to models/RDKit_VPAS/DecisionTreeRegressor_model_29_01_2020_11_13_56.joblib\n"
     ]
    }
   ],
   "source": [
    "dtr_model, dtr_hp = train_model(model = DecisionTreeRegressor,\n",
    "           feature_set= top_15_features_model,\n",
    "            grid_search=True,\n",
    "           hyperparameters=dtr_param_grid)\n",
    "\n",
    "\n",
    "exp_vs_calc_dtr_model = produce_exp_vs_pred_df('top_15_features_model', dtr_model,\n",
    "                                         \"Decision Tree, model-chosen features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### F-regression features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T11:13:59.996091Z",
     "start_time": "2020-01-29T11:13:56.950995Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No grid search\n",
      "Hyperparameters:\n",
      " {'criterion': 'friedman_mse', 'max_depth': 20, 'max_features': 15, 'max_leaf_nodes': None, 'min_impurity_decrease': 0.1, 'min_samples_leaf': 3, 'min_samples_split': 2, 'min_weight_fraction_leaf': 0.0, 'splitter': 'random'}\n",
      "DecisionTreeRegressor run at: 2020-01-29 11:13:59.988100 . Saving to models/RDKit_VPAS/DecisionTreeRegressor_kbest_29_01_2020_11_13_59.joblib\n"
     ]
    }
   ],
   "source": [
    "dtr_freg = train_model(model = DecisionTreeRegressor,\n",
    "           feature_set= top_15_features_kbest,\n",
    "            grid_search=False,\n",
    "           hyperparameters=dtr_hp)\n",
    "exp_vs_calc_dtr_freg = produce_exp_vs_pred_df('top_15_features_kbest', dtr_freg,\n",
    "                                         \"Decision Tree, freg features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mutual info features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T11:14:03.085304Z",
     "start_time": "2020-01-29T11:13:59.999099Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No grid search\n",
      "Hyperparameters:\n",
      " {'criterion': 'friedman_mse', 'max_depth': 20, 'max_features': 15, 'max_leaf_nodes': None, 'min_impurity_decrease': 0.1, 'min_samples_leaf': 3, 'min_samples_split': 2, 'min_weight_fraction_leaf': 0.0, 'splitter': 'random'}\n",
      "DecisionTreeRegressor run at: 2020-01-29 11:14:03.057263 . Saving to models/RDKit_VPAS/DecisionTreeRegressor_mutual_info_29_01_2020_11_14_03.joblib\n"
     ]
    }
   ],
   "source": [
    "dtr_mu = train_model(model = DecisionTreeRegressor,\n",
    "           feature_set= top_15_features_mutual_info,\n",
    "            grid_search=False,\n",
    "           hyperparameters=dtr_hp)\n",
    "exp_vs_calc_dtr_mu = produce_exp_vs_pred_df('top_15_features_mutual_info', dtr_mu,\n",
    "                                         \"Decision Tree, mutual info features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LASSO features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T11:14:06.067880Z",
     "start_time": "2020-01-29T11:14:03.087311Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No grid search\n",
      "Hyperparameters:\n",
      " {'criterion': 'friedman_mse', 'max_depth': 20, 'max_features': 15, 'max_leaf_nodes': None, 'min_impurity_decrease': 0.1, 'min_samples_leaf': 3, 'min_samples_split': 2, 'min_weight_fraction_leaf': 0.0, 'splitter': 'random'}\n",
      "DecisionTreeRegressor run at: 2020-01-29 11:14:06.059860 . Saving to models/RDKit_VPAS/DecisionTreeRegressor_lasso_29_01_2020_11_14_06.joblib\n"
     ]
    }
   ],
   "source": [
    "dtr_lasso = train_model(model = DecisionTreeRegressor,\n",
    "           feature_set= top_15_features_lasso,\n",
    "            grid_search=False,\n",
    "           hyperparameters=dtr_hp)\n",
    "\n",
    "exp_vs_calc_dtr_lasso = produce_exp_vs_pred_df('top_15_features_lasso', dtr_lasso,\n",
    "                                         \"Decision Tree, lasso features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AdaBoost Regressor\n",
    "### Model Chosen Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T11:14:06.078912Z",
     "start_time": "2020-01-29T11:14:06.070891Z"
    }
   },
   "outputs": [],
   "source": [
    "base_models = [ExtraTreesRegressor(n_estimators= 5,\n",
    "                                   criterion= 'mse',\n",
    "                                   max_features = 'log2'),\n",
    "               RandomForestRegressor(n_estimators= 5,\n",
    "                                     criterion= 'mse',\n",
    "                                     max_features = 'sqrt',\n",
    "                                     min_samples_split = 3),\n",
    "              GradientBoostingRegressor(),\n",
    "              DecisionTreeRegressor(),\n",
    "              Lars(),\n",
    "              ElasticNet()]\n",
    "\n",
    "ada_param_grid = {\n",
    "    'base_estimator' : base_models, \n",
    "    'learning_rate' : [0.3, 0.5, 0.8, 1], \n",
    "    'loss' : ['linear', 'square', 'exponential'], \n",
    "    'n_estimators' : [50, 100]\n",
    "} #hyperparameters for adaboost gridsearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T11:21:28.639421Z",
     "start_time": "2020-01-29T11:14:06.081919Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AdaBoostRegressor grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'base_estimator': ExtraTreesRegressor(bootstrap=False, criterion='mse', max_depth=None,\n",
      "                    max_features='log2', max_leaf_nodes=None,\n",
      "                    min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                    min_samples_leaf=1, min_samples_split=2,\n",
      "                    min_weight_fraction_leaf=0.0, n_estimators=5, n_jobs=None,\n",
      "                    oob_score=False, random_state=None, verbose=0,\n",
      "                    warm_start=False), 'learning_rate': 0.8, 'loss': 'linear', 'n_estimators': 50}\n",
      "Grid search complete!\n",
      "Hyperparameters:\n",
      " {'base_estimator': ExtraTreesRegressor(bootstrap=False, criterion='mse', max_depth=None,\n",
      "                    max_features='log2', max_leaf_nodes=None,\n",
      "                    min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                    min_samples_leaf=1, min_samples_split=2,\n",
      "                    min_weight_fraction_leaf=0.0, n_estimators=5, n_jobs=None,\n",
      "                    oob_score=False, random_state=None, verbose=0,\n",
      "                    warm_start=False), 'learning_rate': 0.8, 'loss': 'linear', 'n_estimators': 50}\n",
      "AdaBoostRegressor run at: 2020-01-29 11:21:28.505061 . Saving to models/RDKit_VPAS/AdaBoostRegressor_model_29_01_2020_11_21_28.joblib\n"
     ]
    }
   ],
   "source": [
    "ada_model, ada_hp = train_model(model = AdaBoostRegressor,\n",
    "           feature_set= top_15_features_model,\n",
    "            grid_search=True,\n",
    "           hyperparameters=ada_param_grid)\n",
    "\n",
    "exp_vs_calc_ada_model = produce_exp_vs_pred_df('top_15_features_model', ada_model,\n",
    "                                               \"AdaBoost, model features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### F-regression features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T11:22:36.342428Z",
     "start_time": "2020-01-29T11:21:28.649446Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No grid search\n",
      "Hyperparameters:\n",
      " {'base_estimator': ExtraTreesRegressor(bootstrap=False, criterion='mse', max_depth=None,\n",
      "                    max_features='log2', max_leaf_nodes=None,\n",
      "                    min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                    min_samples_leaf=1, min_samples_split=2,\n",
      "                    min_weight_fraction_leaf=0.0, n_estimators=5, n_jobs=None,\n",
      "                    oob_score=False, random_state=None, verbose=0,\n",
      "                    warm_start=False), 'learning_rate': 0.8, 'loss': 'linear', 'n_estimators': 50}\n",
      "AdaBoostRegressor run at: 2020-01-29 11:22:36.190019 . Saving to models/RDKit_VPAS/AdaBoostRegressor_kbest_29_01_2020_11_22_36.joblib\n"
     ]
    }
   ],
   "source": [
    "ada_freg = train_model(model = AdaBoostRegressor,\n",
    "           feature_set= top_15_features_kbest,\n",
    "            grid_search=False,\n",
    "           hyperparameters=ada_hp)\n",
    "\n",
    "exp_vs_calc_ada_freg = produce_exp_vs_pred_df('top_15_features_kbest', ada_freg,\n",
    "                                               \"AdaBoost, freg features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mutual info features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T11:23:42.220615Z",
     "start_time": "2020-01-29T11:22:36.347440Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No grid search\n",
      "Hyperparameters:\n",
      " {'base_estimator': ExtraTreesRegressor(bootstrap=False, criterion='mse', max_depth=None,\n",
      "                    max_features='log2', max_leaf_nodes=None,\n",
      "                    min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                    min_samples_leaf=1, min_samples_split=2,\n",
      "                    min_weight_fraction_leaf=0.0, n_estimators=5, n_jobs=None,\n",
      "                    oob_score=False, random_state=None, verbose=0,\n",
      "                    warm_start=False), 'learning_rate': 0.8, 'loss': 'linear', 'n_estimators': 50}\n",
      "AdaBoostRegressor run at: 2020-01-29 11:23:42.095246 . Saving to models/RDKit_VPAS/AdaBoostRegressor_mutual_info_29_01_2020_11_23_42.joblib\n"
     ]
    }
   ],
   "source": [
    "ada_mu = train_model(model = AdaBoostRegressor,\n",
    "           feature_set= top_15_features_mutual_info,\n",
    "            grid_search=False,\n",
    "           hyperparameters=ada_hp)\n",
    "\n",
    "exp_vs_calc_ada_mu = produce_exp_vs_pred_df('top_15_features_mutual_info', ada_mu,\n",
    "                                               \"AdaBoost, mutual info features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LASSO features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T11:24:48.718381Z",
     "start_time": "2020-01-29T11:23:42.222585Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No grid search\n",
      "Hyperparameters:\n",
      " {'base_estimator': ExtraTreesRegressor(bootstrap=False, criterion='mse', max_depth=None,\n",
      "                    max_features='log2', max_leaf_nodes=None,\n",
      "                    min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                    min_samples_leaf=1, min_samples_split=2,\n",
      "                    min_weight_fraction_leaf=0.0, n_estimators=5, n_jobs=None,\n",
      "                    oob_score=False, random_state=None, verbose=0,\n",
      "                    warm_start=False), 'learning_rate': 0.8, 'loss': 'linear', 'n_estimators': 50}\n",
      "AdaBoostRegressor run at: 2020-01-29 11:24:48.584024 . Saving to models/RDKit_VPAS/AdaBoostRegressor_lasso_29_01_2020_11_24_48.joblib\n"
     ]
    }
   ],
   "source": [
    "ada_lasso = train_model(model = AdaBoostRegressor,\n",
    "           feature_set= top_15_features_lasso,\n",
    "            grid_search=False,\n",
    "           hyperparameters=ada_hp)\n",
    "\n",
    "exp_vs_calc_ada_lasso = produce_exp_vs_pred_df('top_15_features_lasso', ada_lasso,\n",
    "                                               \"AdaBoost, model features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LASSO\n",
    "### Model Chosen Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T11:24:48.729410Z",
     "start_time": "2020-01-29T11:24:48.722391Z"
    }
   },
   "outputs": [],
   "source": [
    "lasso_param_grid = {\n",
    "    'alpha':[0.2, 0.4, 0.6, 0.8],\n",
    "    'max_iter':[1000, 5000, 10000, 50000],\n",
    "    'selection':['cyclic', 'random']\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T11:24:55.990751Z",
     "start_time": "2020-01-29T11:24:48.733420Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lasso grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'alpha': 0.2, 'max_iter': 5000, 'selection': 'random'}\n",
      "Grid search complete!\n",
      "Hyperparameters:\n",
      " {'alpha': 0.2, 'max_iter': 5000, 'selection': 'random'}\n",
      "Lasso run at: 2020-01-29 11:24:55.983733 . Saving to models/RDKit_VPAS/Lasso_model_29_01_2020_11_24_55.joblib\n"
     ]
    }
   ],
   "source": [
    "lasso_model, lasso_hp = train_model(model = Lasso,\n",
    "           feature_set= top_15_features_model,\n",
    "            grid_search=True,\n",
    "           hyperparameters=lasso_param_grid)\n",
    "\n",
    "\n",
    "exp_vs_calc_lasso_model = produce_exp_vs_pred_df('top_15_features_model', lasso_model,\n",
    "                                                 \"LASSO, model features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### F-regression features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T11:24:59.296411Z",
     "start_time": "2020-01-29T11:24:55.993757Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No grid search\n",
      "Hyperparameters:\n",
      " {'alpha': 0.2, 'max_iter': 5000, 'selection': 'random'}\n",
      "Lasso run at: 2020-01-29 11:24:59.290390 . Saving to models/RDKit_VPAS/Lasso_kbest_29_01_2020_11_24_59.joblib\n"
     ]
    }
   ],
   "source": [
    "lasso_freg = train_model(model = Lasso,\n",
    "           feature_set= top_15_features_kbest,\n",
    "            grid_search=False,\n",
    "           hyperparameters=lasso_hp)\n",
    "\n",
    "exp_vs_calc_lasso_freg = produce_exp_vs_pred_df('top_15_features_kbest', lasso_freg,\n",
    "                                                 \"LASSO, f-reg features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mutual info features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T11:25:02.779670Z",
     "start_time": "2020-01-29T11:24:59.299416Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No grid search\n",
      "Hyperparameters:\n",
      " {'alpha': 0.2, 'max_iter': 5000, 'selection': 'random'}\n",
      "Lasso run at: 2020-01-29 11:25:02.772652 . Saving to models/RDKit_VPAS/Lasso_mutual_info_29_01_2020_11_25_02.joblib\n"
     ]
    }
   ],
   "source": [
    "lasso_mu = train_model(model = Lasso,\n",
    "           feature_set= top_15_features_mutual_info,\n",
    "            grid_search=False,\n",
    "           hyperparameters=lasso_hp)\n",
    "\n",
    "exp_vs_calc_lasso_mu = produce_exp_vs_pred_df('top_15_features_mutual_info', lasso_mu,\n",
    "                                                 \"LASSO, mutual info features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LASSO Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T11:25:06.118548Z",
     "start_time": "2020-01-29T11:25:02.782678Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No grid search\n",
      "Hyperparameters:\n",
      " {'alpha': 0.2, 'max_iter': 5000, 'selection': 'random'}\n",
      "Lasso run at: 2020-01-29 11:25:06.109524 . Saving to models/RDKit_VPAS/Lasso_lasso_29_01_2020_11_25_06.joblib\n"
     ]
    }
   ],
   "source": [
    "lasso_lasso = train_model(model = Lasso,\n",
    "           feature_set= top_15_features_lasso,\n",
    "            grid_search=False,\n",
    "           hyperparameters=lasso_hp)\n",
    "\n",
    "exp_vs_calc_lasso_lasso = produce_exp_vs_pred_df('top_15_features_lasso', lasso_lasso,\n",
    "                                                 \"LASSO, lasso features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest Regressor\n",
    "### Model Chosen Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T11:25:06.129579Z",
     "start_time": "2020-01-29T11:25:06.122560Z"
    }
   },
   "outputs": [],
   "source": [
    "random_forest_param_grid = {\n",
    "    'max_depth': [10, 20, 30, 40, 50, 60, 70, 80, 90, None],\n",
    "    'max_features': ['auto', 'sqrt'],\n",
    "    'min_samples_leaf': [1, 2, 4],\n",
    "    'min_samples_split': [2, 5, 10],\n",
    "    'n_estimators': [200, 400, 600, 800, 1000, 1200, 1400, 1600]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T13:44:25.518289Z",
     "start_time": "2020-01-29T11:25:06.133587Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestRegressor grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'max_depth': 70, 'max_features': 'sqrt', 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 200}\n",
      "Grid search complete!\n",
      "Hyperparameters:\n",
      " {'max_depth': 70, 'max_features': 'sqrt', 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 200}\n",
      "RandomForestRegressor run at: 2020-01-29 13:44:25.389944 . Saving to models/RDKit_VPAS/RandomForestRegressor_model_29_01_2020_13_44_25.joblib\n"
     ]
    }
   ],
   "source": [
    "random_forest_model, rfr_hp = train_model(model = RandomForestRegressor,\n",
    "           feature_set= top_15_features_model,\n",
    "            grid_search=True,\n",
    "           hyperparameters=random_forest_param_grid)\n",
    "exp_vs_calc_random_forest_model = produce_exp_vs_pred_df('top_15_features_model', random_forest_model,\n",
    "                                                         \"RandomForest, model features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### F-regression features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T13:45:02.559445Z",
     "start_time": "2020-01-29T13:44:25.525307Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No grid search\n",
      "Hyperparameters:\n",
      " {'max_depth': 70, 'max_features': 'sqrt', 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 200}\n",
      "RandomForestRegressor run at: 2020-01-29 13:45:02.422406 . Saving to models/RDKit_VPAS/RandomForestRegressor_kbest_29_01_2020_13_45_02.joblib\n"
     ]
    }
   ],
   "source": [
    "random_forest_freg = train_model(model = RandomForestRegressor,\n",
    "           feature_set= top_15_features_kbest,\n",
    "            grid_search=False,\n",
    "           hyperparameters=rfr_hp)\n",
    "exp_vs_calc_random_forest_freg = produce_exp_vs_pred_df('top_15_features_kbest', random_forest_freg,\n",
    "                                                         \"RandomForest, freg features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mutual info features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T13:45:39.641780Z",
     "start_time": "2020-01-29T13:45:02.562453Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No grid search\n",
      "Hyperparameters:\n",
      " {'max_depth': 70, 'max_features': 'sqrt', 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 200}\n",
      "RandomForestRegressor run at: 2020-01-29 13:45:39.515435 . Saving to models/RDKit_VPAS/RandomForestRegressor_mutual_info_29_01_2020_13_45_39.joblib\n"
     ]
    }
   ],
   "source": [
    "random_forest_mu = train_model(model = RandomForestRegressor,\n",
    "           feature_set= top_15_features_mutual_info,\n",
    "            grid_search=False,\n",
    "           hyperparameters=rfr_hp)\n",
    "exp_vs_calc_random_forest_mu = produce_exp_vs_pred_df('top_15_features_mutual_info', random_forest_mu,\n",
    "                                                         \"RandomForest, mutual info features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LASSO features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T13:46:16.432084Z",
     "start_time": "2020-01-29T13:45:39.646787Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No grid search\n",
      "Hyperparameters:\n",
      " {'max_depth': 70, 'max_features': 'sqrt', 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 200}\n",
      "RandomForestRegressor run at: 2020-01-29 13:46:16.305747 . Saving to models/RDKit_VPAS/RandomForestRegressor_lasso_29_01_2020_13_46_16.joblib\n"
     ]
    }
   ],
   "source": [
    "random_forest_lasso = train_model(model = RandomForestRegressor,\n",
    "           feature_set= top_15_features_lasso,\n",
    "            grid_search=False,\n",
    "           hyperparameters=rfr_hp)\n",
    "exp_vs_calc_random_forest_lasso = produce_exp_vs_pred_df('top_15_features_lasso', random_forest_lasso,\n",
    "                                                         \"RandomForest, lasso features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LARS - Least angle regression\n",
    "### Model chosen features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T13:46:16.443115Z",
     "start_time": "2020-01-29T13:46:16.439104Z"
    }
   },
   "outputs": [],
   "source": [
    "lars_param_grid = {'n_nonzero_coefs': [0, 5, 10, 15]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T13:46:35.820854Z",
     "start_time": "2020-01-29T13:46:16.445120Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lars grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'n_nonzero_coefs': 5}\n",
      "Grid search complete!\n",
      "Hyperparameters:\n",
      " {'n_nonzero_coefs': 5}\n",
      "Lars run at: 2020-01-29 13:46:35.810828 . Saving to models/RDKit_VPAS/Lars_model_29_01_2020_13_46_35.joblib\n"
     ]
    }
   ],
   "source": [
    "lars_model, lars_hp = train_model(model = Lars,\n",
    "           feature_set= top_15_features_model,\n",
    "            grid_search=True,\n",
    "           hyperparameters= lars_param_grid)\n",
    "exp_vs_calc_lars_model = produce_exp_vs_pred_df('top_15_features_model', lars_model, \"LARS model features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### F-regression features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T13:46:39.872592Z",
     "start_time": "2020-01-29T13:46:35.823863Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No grid search\n",
      "Hyperparameters:\n",
      " {'n_nonzero_coefs': 5}\n",
      "Lars run at: 2020-01-29 13:46:39.861559 . Saving to models/RDKit_VPAS/Lars_kbest_29_01_2020_13_46_39.joblib\n"
     ]
    }
   ],
   "source": [
    "lars_freg = train_model(model = Lars,\n",
    "           feature_set= top_15_features_kbest,\n",
    "            grid_search=False,\n",
    "           hyperparameters= lars_hp)\n",
    "exp_vs_calc_lars_freg = produce_exp_vs_pred_df('top_15_features_kbest', lars_freg, \"LARS freg features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mutual info features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T13:46:43.734860Z",
     "start_time": "2020-01-29T13:46:39.875600Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No grid search\n",
      "Hyperparameters:\n",
      " {'n_nonzero_coefs': 5}\n",
      "Lars run at: 2020-01-29 13:46:43.724869 . Saving to models/RDKit_VPAS/Lars_mutual_info_29_01_2020_13_46_43.joblib\n"
     ]
    }
   ],
   "source": [
    "lars_mu = train_model(model = Lars,\n",
    "           feature_set= top_15_features_mutual_info,\n",
    "            grid_search=False,\n",
    "           hyperparameters= lars_hp)\n",
    "\n",
    "exp_vs_calc_lars_mu = produce_exp_vs_pred_df('top_15_features_mutual_info', lars_mu, \"LARS mutual info features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LASSO features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T13:46:47.534964Z",
     "start_time": "2020-01-29T13:46:43.737870Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No grid search\n",
      "Hyperparameters:\n",
      " {'n_nonzero_coefs': 5}\n",
      "Lars run at: 2020-01-29 13:46:47.523972 . Saving to models/RDKit_VPAS/Lars_lasso_29_01_2020_13_46_47.joblib\n"
     ]
    }
   ],
   "source": [
    "lars_lasso = train_model(model = Lars,\n",
    "           feature_set= top_15_features_lasso,\n",
    "            grid_search=False,\n",
    "           hyperparameters= lars_hp)\n",
    "exp_vs_calc_lars_lasso = produce_exp_vs_pred_df('top_15_features_lasso', lars_lasso, \"LARS lasso features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Elastic Net (linear regression l1 l2 norm regularization)\n",
    "### Model chosen features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T13:46:47.544990Z",
     "start_time": "2020-01-29T13:46:47.537973Z"
    }
   },
   "outputs": [],
   "source": [
    "net_param_grid = {\n",
    "    'alpha':[0.1, 0.3, 0.5, 0.8, 1.0],\n",
    "    'l1_ratio':[0.0, 0.2, 0.4, 0.6, 0.8, 1.0],\n",
    "    'fit_intercept':[True, False],\n",
    "    'max_iter': [1000, 3000, 5000],\n",
    "    'tol':[0.0001, 0.001, 0.01],\n",
    "    'selection':['cyclic', 'random']\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T13:47:21.412743Z",
     "start_time": "2020-01-29T13:46:47.549001Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ElasticNet grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'alpha': 0.3, 'fit_intercept': True, 'l1_ratio': 0.4, 'max_iter': 1000, 'selection': 'cyclic', 'tol': 0.0001}\n",
      "Grid search complete!\n",
      "Hyperparameters:\n",
      " {'alpha': 0.3, 'fit_intercept': True, 'l1_ratio': 0.4, 'max_iter': 1000, 'selection': 'cyclic', 'tol': 0.0001}\n",
      "ElasticNet run at: 2020-01-29 13:47:21.401712 . Saving to models/RDKit_VPAS/ElasticNet_model_29_01_2020_13_47_21.joblib\n"
     ]
    }
   ],
   "source": [
    "net_model, net_hp = train_model(model = ElasticNet,\n",
    "           feature_set= top_15_features_model,\n",
    "            grid_search=True,\n",
    "           hyperparameters= net_param_grid)\n",
    "exp_vs_calc_net_model = produce_exp_vs_pred_df('top_15_features_model', net_model, \"ElasticNet model features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### F regression features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T13:47:25.373279Z",
     "start_time": "2020-01-29T13:47:21.415749Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No grid search\n",
      "Hyperparameters:\n",
      " {'alpha': 0.3, 'fit_intercept': True, 'l1_ratio': 0.4, 'max_iter': 1000, 'selection': 'cyclic', 'tol': 0.0001}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 1691.237155666753, tolerance: 8.44530395603002\n",
      "  positive)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ElasticNet run at: 2020-01-29 13:47:25.366251 . Saving to models/RDKit_VPAS/ElasticNet_kbest_29_01_2020_13_47_25.joblib\n"
     ]
    }
   ],
   "source": [
    "net_freg = train_model(model = ElasticNet,\n",
    "           feature_set= top_15_features_kbest,\n",
    "            grid_search=False,\n",
    "           hyperparameters= net_hp)\n",
    "exp_vs_calc_net_freg = produce_exp_vs_pred_df('top_15_features_kbest', net_freg, \"ElasticNet freg features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mutual info features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T13:47:29.264618Z",
     "start_time": "2020-01-29T13:47:25.379289Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No grid search\n",
      "Hyperparameters:\n",
      " {'alpha': 0.3, 'fit_intercept': True, 'l1_ratio': 0.4, 'max_iter': 1000, 'selection': 'cyclic', 'tol': 0.0001}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 1003.1704196947067, tolerance: 8.44530395603002\n",
      "  positive)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ElasticNet run at: 2020-01-29 13:47:29.256615 . Saving to models/RDKit_VPAS/ElasticNet_mutual_info_29_01_2020_13_47_29.joblib\n"
     ]
    }
   ],
   "source": [
    "net_mu = train_model(model = ElasticNet,\n",
    "           feature_set= top_15_features_mutual_info,\n",
    "            grid_search=False,\n",
    "           hyperparameters= net_hp)\n",
    "exp_vs_calc_net_mu = produce_exp_vs_pred_df('top_15_features_mutual_info', net_mu, \"ElasticNet mutual info features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LASSO features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-29T13:47:33.181724Z",
     "start_time": "2020-01-29T13:47:29.266624Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No grid search\n",
      "Hyperparameters:\n",
      " {'alpha': 0.3, 'fit_intercept': True, 'l1_ratio': 0.4, 'max_iter': 1000, 'selection': 'cyclic', 'tol': 0.0001}\n",
      "ElasticNet run at: 2020-01-29 13:47:33.175700 . Saving to models/RDKit_VPAS/ElasticNet_lasso_29_01_2020_13_47_33.joblib\n"
     ]
    }
   ],
   "source": [
    "net_lasso = train_model(model = ElasticNet,\n",
    "           feature_set= top_15_features_lasso,\n",
    "            grid_search=False,\n",
    "           hyperparameters= net_hp)\n",
    "exp_vs_calc_net_lasso = produce_exp_vs_pred_df('top_15_features_lasso', net_lasso, \"ElasticNet lasso features\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "notify_time": "30",
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "303.516px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}