predicting.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import joblib\n",
    "import seaborn as sns\n",
    "import networkx as nx\n",
    "from graph_utils import get_graph_features\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = '../data/depression/'\n",
    "folders_list = sorted(os.listdir(data_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "corr_matrix_list = []\n",
    "\n",
    "for folder in folders_list:\n",
    "    df_path = data_path + folder\n",
    "    corr_matrix2append = pd.read_csv(df_path, header=None).values\n",
    "    corr_matrix_list.append(corr_matrix2append)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = joblib.load('../data/y_d_nod.pkl')\n",
    "y_all = joblib.load('../data/y_all.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_edges=37.0\n",
      "37 21 37\n",
      "n_edges=10.0\n",
      "10 2 10\n",
      "n_edges=16.0\n",
      "16 1 16\n",
      "n_edges=7.0\n",
      "7 0 7\n",
      "n_edges=65.0\n",
      "65 57 65\n",
      "n_edges=22.0\n",
      "22 5 22\n",
      "n_edges=125.0\n",
      "125 167 125\n",
      "n_edges=12.0\n",
      "12 2 12\n",
      "n_edges=58.0\n",
      "58 22 58\n",
      "n_edges=6.0\n",
      "6 0 6\n",
      "n_edges=34.0\n",
      "34 13 34\n",
      "n_edges=45.0\n",
      "45 31 45\n",
      "n_edges=92.0\n",
      "92 103 92\n",
      "n_edges=174.0\n",
      "174 265 174\n",
      "n_edges=66.0\n",
      "66 28 66\n",
      "n_edges=9.0\n",
      "9 0 9\n",
      "n_edges=21.0\n",
      "21 8 21\n",
      "n_edges=18.0\n",
      "18 3 18\n",
      "n_edges=10.0\n",
      "10 5 10\n",
      "n_edges=11.0\n",
      "11 1 11\n",
      "n_edges=15.0\n",
      "15 4 15\n",
      "n_edges=32.0\n",
      "32 8 32\n",
      "n_edges=22.0\n",
      "22 3 22\n",
      "n_edges=41.0\n",
      "41 15 41\n",
      "n_edges=78.0\n",
      "78 111 78\n",
      "n_edges=54.0\n",
      "54 42 54\n",
      "n_edges=19.0\n",
      "19 2 19\n",
      "n_edges=45.0\n",
      "45 10 45\n",
      "n_edges=19.0\n",
      "19 2 19\n",
      "n_edges=56.0\n",
      "56 25 56\n",
      "n_edges=45.0\n",
      "45 20 45\n",
      "n_edges=50.0\n",
      "50 21 50\n",
      "n_edges=22.0\n",
      "22 5 22\n",
      "n_edges=16.0\n",
      "16 2 16\n",
      "n_edges=31.0\n",
      "31 11 31\n",
      "n_edges=36.0\n",
      "36 15 36\n",
      "n_edges=33.0\n",
      "33 19 33\n",
      "n_edges=42.0\n",
      "42 19 42\n",
      "n_edges=30.0\n",
      "30 4 30\n",
      "n_edges=12.0\n",
      "12 0 12\n",
      "n_edges=17.0\n",
      "17 1 17\n",
      "n_edges=14.0\n",
      "14 0 14\n",
      "n_edges=11.0\n",
      "11 1 11\n",
      "n_edges=24.0\n",
      "24 4 24\n",
      "n_edges=24.0\n",
      "24 4 24\n",
      "n_edges=59.0\n",
      "59 47 59\n",
      "n_edges=40.0\n",
      "40 6 40\n",
      "n_edges=11.0\n",
      "11 0 11\n",
      "n_edges=286.0\n",
      "286 627 286\n",
      "n_edges=30.0\n",
      "30 6 30\n",
      "n_edges=23.0\n",
      "23 7 23\n",
      "n_edges=24.0\n",
      "24 11 24\n",
      "n_edges is 771.0, ignored\n",
      "n_edges=19.0\n",
      "19 8 19\n",
      "n_edges=26.0\n",
      "26 8 26\n",
      "n_edges=24.0\n",
      "24 2 24\n",
      "n_edges=8.0\n",
      "8 0 8\n",
      "n_edges=56.0\n",
      "56 51 56\n",
      "n_edges=3.0\n",
      "3 0 3\n",
      "n_edges=25.0\n",
      "25 5 25\n",
      "n_edges=1.0\n",
      "1 0 1\n",
      "n_edges is 2525.0, ignored\n",
      "n_edges=4.0\n",
      "4 0 4\n",
      "n_edges=5.0\n",
      "5 0 5\n",
      "n_edges=1.0\n",
      "1 0 1\n",
      "n_edges=15.0\n",
      "15 2 15\n",
      "n_edges=17.0\n",
      "17 2 17\n",
      "n_edges=23.0\n",
      "23 3 23\n",
      "n_edges=50.0\n",
      "50 28 50\n",
      "n_edges=20.0\n",
      "20 8 20\n",
      "n_edges=16.0\n",
      "16 2 16\n",
      "n_edges=4.0\n",
      "4 0 4\n",
      "n_edges=33.0\n",
      "33 21 33\n",
      "n_edges=149.0\n",
      "149 77 149\n",
      "n_edges=15.0\n",
      "15 2 15\n",
      "n_edges=165.0\n",
      "165 345 165\n",
      "n_edges=49.0\n",
      "49 25 49\n",
      "n_edges=21.0\n",
      "21 13 21\n",
      "n_edges=35.0\n",
      "35 12 35\n",
      "n_edges=27.0\n",
      "27 7 27\n",
      "n_edges=5.0\n",
      "5 0 5\n",
      "n_edges=16.0\n",
      "16 4 16\n",
      "n_edges=15.0\n",
      "15 6 15\n",
      "n_edges=41.0\n",
      "41 20 41\n",
      "n_edges=21.0\n",
      "21 3 21\n",
      "n_edges=17.0\n",
      "17 2 17\n",
      "n_edges=25.0\n",
      "25 7 25\n",
      "n_edges=25.0\n",
      "25 9 25\n",
      "n_edges=96.0\n",
      "96 200 96\n",
      "n_edges=55.0\n",
      "55 31 55\n",
      "n_edges=108.0\n",
      "108 80 108\n",
      "n_edges=62.0\n",
      "62 55 62\n",
      "n_edges=89.0\n",
      "89 54 89\n",
      "n_edges=63.0\n",
      "63 37 63\n",
      "n_edges=1.0\n",
      "1 0 1\n",
      "n_edges=29.0\n",
      "29 8 29\n",
      "n_edges=90.0\n",
      "90 147 90\n",
      "n_edges=77.0\n",
      "77 44 77\n",
      "n_edges=27.0\n",
      "27 9 27\n",
      "n_edges=79.0\n",
      "79 44 79\n"
     ]
    }
   ],
   "source": [
    "graph_features_list = []\n",
    "y_list = []\n",
    "\n",
    "for A, y_value in zip(corr_matrix_list, y):\n",
    "    graph_features = get_graph_features(A, border_value=0.22, min_graph_size=3,\n",
    "                                        max_graph_size=1500, max_number_of_edges=700)\n",
    "    if graph_features is not None:\n",
    "        graph_features_list.append(graph_features)\n",
    "        y_list.append(y_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['y_list_depression']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(graph_features_list, 'graph_features_list_depression')\n",
    "joblib.dump(y_list, 'y_list_depression')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "aggregated_graph_features_list = []\n",
    "\n",
    "for graph_features in graph_features_list:\n",
    "    aggregated_graph_features_dict = {}\n",
    "    for feature_name, feature_values in graph_features.items():\n",
    "        aggregated_feature_name = feature_name + '_median'\n",
    "        aggregated_graph_features_dict[aggregated_feature_name] = np.median(feature_values)\n",
    "        \n",
    "#         aggregated_feature_name = feature_name + '_mean'\n",
    "#         aggregated_graph_features_dict[aggregated_feature_name] = np.mean(feature_values)\n",
    "    \n",
    "    aggregated_graph_features_list.append(aggregated_graph_features_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(aggregated_graph_features_list)\n",
    "df = df.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(98, 32)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AvgNodeDegree_G_2click_1simplex_median</th>\n",
       "      <th>AvgNodeDegree_G_3click_1simplex_median</th>\n",
       "      <th>AvgNodeDegree_G_3click_2simplex_median</th>\n",
       "      <th>AvgNodeDegree_initial_median</th>\n",
       "      <th>Betweenness_G_2click_1simplex_median</th>\n",
       "      <th>Betweenness_G_3click_1simplex_median</th>\n",
       "      <th>Betweenness_G_3click_2simplex_median</th>\n",
       "      <th>Betweenness_initial_median</th>\n",
       "      <th>Closeness_G_2click_1simplex_median</th>\n",
       "      <th>Closeness_G_3click_1simplex_median</th>\n",
       "      <th>...</th>\n",
       "      <th>GlobalEfficiency_G_3click_2simplex_median</th>\n",
       "      <th>GlobalEfficiency_initial_median</th>\n",
       "      <th>LocalEfficiency_G_2click_1simplex_median</th>\n",
       "      <th>LocalEfficiency_G_3click_1simplex_median</th>\n",
       "      <th>LocalEfficiency_G_3click_2simplex_median</th>\n",
       "      <th>LocalEfficiency_initial_median</th>\n",
       "      <th>n_2click_1simplex_median</th>\n",
       "      <th>n_3click_1simplex_median</th>\n",
       "      <th>n_3click_2simplex_median</th>\n",
       "      <th>n_edges_median</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9.000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006349</td>\n",
       "      <td>0.001587</td>\n",
       "      <td>0.027558</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.363363</td>\n",
       "      <td>0.160714</td>\n",
       "      <td>...</td>\n",
       "      <td>0.529365</td>\n",
       "      <td>0.009468</td>\n",
       "      <td>0.735689</td>\n",
       "      <td>0.592235</td>\n",
       "      <td>0.669724</td>\n",
       "      <td>0.102889</td>\n",
       "      <td>37.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>37.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.575</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004630</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.396825</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001768</td>\n",
       "      <td>0.513333</td>\n",
       "      <td>0.233333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.027778</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.003136</td>\n",
       "      <td>0.232639</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.014245</td>\n",
       "      <td>16.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001032</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9.875</td>\n",
       "      <td>3.222222</td>\n",
       "      <td>12.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001538</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.012879</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.281250</td>\n",
       "      <td>0.148053</td>\n",
       "      <td>...</td>\n",
       "      <td>0.401650</td>\n",
       "      <td>0.013631</td>\n",
       "      <td>0.671903</td>\n",
       "      <td>0.155989</td>\n",
       "      <td>0.668472</td>\n",
       "      <td>0.167727</td>\n",
       "      <td>65.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>65.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   AvgNodeDegree_G_2click_1simplex_median  \\\n",
       "0                                   9.000   \n",
       "1                                   3.575   \n",
       "2                                   0.000   \n",
       "3                                   0.000   \n",
       "4                                   9.875   \n",
       "\n",
       "   AvgNodeDegree_G_3click_1simplex_median  \\\n",
       "0                                5.000000   \n",
       "1                                1.000000   \n",
       "2                                0.000000   \n",
       "3                                0.000000   \n",
       "4                                3.222222   \n",
       "\n",
       "   AvgNodeDegree_G_3click_2simplex_median  AvgNodeDegree_initial_median  \\\n",
       "0                                     7.0                           0.0   \n",
       "1                                     0.0                           0.0   \n",
       "2                                     0.0                           0.0   \n",
       "3                                     0.0                           0.0   \n",
       "4                                    12.5                           0.0   \n",
       "\n",
       "   Betweenness_G_2click_1simplex_median  Betweenness_G_3click_1simplex_median  \\\n",
       "0                              0.006349                              0.001587   \n",
       "1                              0.004630                              0.000000   \n",
       "2                              0.000000                              0.000000   \n",
       "3                              0.000000                              0.000000   \n",
       "4                              0.001538                              0.000000   \n",
       "\n",
       "   Betweenness_G_3click_2simplex_median  Betweenness_initial_median  \\\n",
       "0                              0.027558                         0.0   \n",
       "1                              0.000000                         0.0   \n",
       "2                              0.000000                         0.0   \n",
       "3                              0.000000                         0.0   \n",
       "4                              0.012879                         0.0   \n",
       "\n",
       "   Closeness_G_2click_1simplex_median  Closeness_G_3click_1simplex_median  \\\n",
       "0                            0.363363                            0.160714   \n",
       "1                            0.396825                            0.111111   \n",
       "2                            0.000000                            0.000000   \n",
       "3                            0.000000                            0.000000   \n",
       "4                            0.281250                            0.148053   \n",
       "\n",
       "        ...        GlobalEfficiency_G_3click_2simplex_median  \\\n",
       "0       ...                                         0.529365   \n",
       "1       ...                                         0.000000   \n",
       "2       ...                                         0.000000   \n",
       "3       ...                                         0.000000   \n",
       "4       ...                                         0.401650   \n",
       "\n",
       "   GlobalEfficiency_initial_median  LocalEfficiency_G_2click_1simplex_median  \\\n",
       "0                         0.009468                                  0.735689   \n",
       "1                         0.001768                                  0.513333   \n",
       "2                         0.003136                                  0.232639   \n",
       "3                         0.001032                                  0.000000   \n",
       "4                         0.013631                                  0.671903   \n",
       "\n",
       "   LocalEfficiency_G_3click_1simplex_median  \\\n",
       "0                                  0.592235   \n",
       "1                                  0.233333   \n",
       "2                                  0.000000   \n",
       "3                                  0.000000   \n",
       "4                                  0.155989   \n",
       "\n",
       "   LocalEfficiency_G_3click_2simplex_median  LocalEfficiency_initial_median  \\\n",
       "0                                  0.669724                        0.102889   \n",
       "1                                  0.000000                        0.027778   \n",
       "2                                  0.000000                        0.014245   \n",
       "3                                  0.000000                        0.000000   \n",
       "4                                  0.668472                        0.167727   \n",
       "\n",
       "   n_2click_1simplex_median  n_3click_1simplex_median  \\\n",
       "0                      37.0                      37.0   \n",
       "1                      10.0                      10.0   \n",
       "2                      16.0                      16.0   \n",
       "3                       7.0                       7.0   \n",
       "4                      65.0                      65.0   \n",
       "\n",
       "   n_3click_2simplex_median  n_edges_median  \n",
       "0                      21.0            37.0  \n",
       "1                       2.0            10.0  \n",
       "2                       1.0            16.0  \n",
       "3                       0.0             7.0  \n",
       "4                      57.0            65.0  \n",
       "\n",
       "[5 rows x 32 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('df.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_with_target = df.copy()\n",
    "# df_with_target['target'] = y_list\n",
    "\n",
    "# df_corr = df_with_target.corr()\n",
    "\n",
    "# feature_names = df_corr.columns\n",
    "\n",
    "# plt.figure(figsize=(12, 10))\n",
    "# sns.heatmap(df_corr.values, cmap='coolwarm', square=True, vmin=-1, vmax=1, cbar_kws={\"shrink\": .7})\n",
    "# # plt.xlabel(feature_names)\n",
    "# plt.yticks(np.arange(len(feature_names)) + 0.5, feature_names, rotation=0, va=\"center\")\n",
    "# plt.xticks(np.arange(len(feature_names)) + 0.5, feature_names, rotation=45, rotation_mode=\"anchor\", ha=\"right\")\n",
    "# plt.savefig('../pic/heatmap.png')\n",
    "# plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LogisticRegression, Lasso\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import RepeatedKFold\n",
    "from sklearn.neighbors import KNeighborsClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = np.array(y_list)\n",
    "ss = StandardScaler()\n",
    "\n",
    "allowed_suffixes_list_initial = ['initial']\n",
    "allowed_suffixes_list_2_click = ['initial', '2click_1simplex']\n",
    "allowed_suffixes_list_3_click = ['initial', '3click_1simplex', '3click_2simplex']\n",
    "\n",
    "columns_initial = [column for column in df.columns if any(suffix in column for suffix in allowed_suffixes_list_initial)]\n",
    "columns_2click = [column for column in df.columns if any(suffix in column for suffix in allowed_suffixes_list_2_click)]\n",
    "columns_3click = [column for column in df.columns if any(suffix in column for suffix in allowed_suffixes_list_3_click)]\n",
    "# columns_all = list(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "rk = RepeatedKFold(n_splits=5, n_repeats=10)\n",
    "\n",
    "# lasso = Lasso(alpha=0.01)\n",
    "rf = RandomForestClassifier(n_estimators=100, max_depth=3, n_jobs=-1)\n",
    "lr = LogisticRegression(solver='lbfgs', max_iter=10000, n_jobs=-1)\n",
    "svm = SVC(gamma='scale', kernel='rbf')\n",
    "knn = KNeighborsClassifier(n_neighbors=3, n_jobs=-1)\n",
    "\n",
    "methods_list = [lr, rf, svm, knn]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_list = []\n",
    "\n",
    "for method in methods_list:\n",
    "    method_name = method.__class__.__name__\n",
    "    result = {\n",
    "        'method_name': method_name,\n",
    "    }\n",
    "\n",
    "    for columns_descryption, columns_list in zip(\n",
    "        ['initial', '2-level features', '3-level features'],\n",
    "        [columns_initial, columns_2click, columns_3click],\n",
    "    ):\n",
    "        train_normed = ss.fit_transform(df[columns_list].values)\n",
    "        scoring_method = 'roc_auc'\n",
    "        method_scores = cross_val_score(method, X=train_normed, y=target, cv=rk, scoring=scoring_method)\n",
    "        method_score_mean = method_scores.mean()\n",
    "        method_score_std = method_scores.std()\n",
    "        result4columns = {\n",
    "            f'{columns_descryption}_mean': method_score_mean,\n",
    "            f'{columns_descryption}_std': method_score_std,\n",
    "        }\n",
    "        result.update(result4columns)\n",
    "\n",
    "    results_list.append(result)#, name=method_name))\n",
    "\n",
    "df_results = pd.DataFrame(results_list)        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>method_name</th>\n",
       "      <th>initial_mean</th>\n",
       "      <th>2-level features_mean</th>\n",
       "      <th>3-level features_mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>0.597</td>\n",
       "      <td>0.590</td>\n",
       "      <td>0.516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RandomForestClassifier</td>\n",
       "      <td>0.552</td>\n",
       "      <td>0.527</td>\n",
       "      <td>0.541</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>SVC</td>\n",
       "      <td>0.609</td>\n",
       "      <td>0.563</td>\n",
       "      <td>0.575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>KNeighborsClassifier</td>\n",
       "      <td>0.492</td>\n",
       "      <td>0.486</td>\n",
       "      <td>0.516</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              method_name  initial_mean  2-level features_mean  \\\n",
       "0      LogisticRegression         0.597                  0.590   \n",
       "1  RandomForestClassifier         0.552                  0.527   \n",
       "2                     SVC         0.609                  0.563   \n",
       "3    KNeighborsClassifier         0.492                  0.486   \n",
       "\n",
       "   3-level features_mean  \n",
       "0                  0.516  \n",
       "1                  0.541  \n",
       "2                  0.575  \n",
       "3                  0.516  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "column_order = ['method_name', 'initial_mean','2-level features_mean', '3-level features_mean']#, '2-level features_std', '3-level features_std']\n",
    "df_results[column_order].round(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "lasso = Lasso(alpha=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "initial 3\n",
      "2-level features 7\n",
      "3-level features 11\n"
     ]
    }
   ],
   "source": [
    "best_features_dict = {}\n",
    "\n",
    "for columns_descryption, columns_list in zip(\n",
    "    ['initial', '2-level features', '3-level features'],\n",
    "    [columns_initial, columns_2click, columns_3click],\n",
    "):\n",
    "    train_normed = ss.fit_transform(df[columns_list].values)\n",
    "\n",
    "    lasso.fit(train_normed, target)\n",
    "    useful_features_nums = np.nonzero(lasso.coef_)[0]\n",
    "    n_features = len(useful_features_nums)\n",
    "    print(columns_descryption, n_features)\n",
    "    best_features_dict[columns_descryption] = np.array(columns_list)[useful_features_nums]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_list = []\n",
    "\n",
    "for method in methods_list:\n",
    "    method_name = method.__class__.__name__\n",
    "    result = {\n",
    "        'method_name': method_name,\n",
    "    }\n",
    "\n",
    "    for columns_descryption, best_features in best_features_dict.items():\n",
    "        train_normed = ss.fit_transform(df[best_features].values)\n",
    "        scoring_method = 'roc_auc'\n",
    "        method_scores = cross_val_score(method, X=train_normed, y=target, cv=rk, scoring=scoring_method)\n",
    "        method_score_mean = method_scores.mean()\n",
    "        method_score_std = method_scores.std()\n",
    "        result4columns = {\n",
    "            f'{columns_descryption}_mean': method_score_mean,\n",
    "            f'{columns_descryption}_std': method_score_std,\n",
    "        }\n",
    "        result.update(result4columns)\n",
    "\n",
    "    results_list.append(result)#, name=method_name))\n",
    "\n",
    "df_results = pd.DataFrame(results_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>method_name</th>\n",
       "      <th>initial_mean</th>\n",
       "      <th>2-level features_mean</th>\n",
       "      <th>3-level features_mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LogisticRegression</td>\n",
       "      <td>0.632</td>\n",
       "      <td>0.610</td>\n",
       "      <td>0.594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RandomForestClassifier</td>\n",
       "      <td>0.575</td>\n",
       "      <td>0.560</td>\n",
       "      <td>0.584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>SVC</td>\n",
       "      <td>0.607</td>\n",
       "      <td>0.604</td>\n",
       "      <td>0.589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>KNeighborsClassifier</td>\n",
       "      <td>0.503</td>\n",
       "      <td>0.505</td>\n",
       "      <td>0.504</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              method_name  initial_mean  2-level features_mean  \\\n",
       "0      LogisticRegression         0.632                  0.610   \n",
       "1  RandomForestClassifier         0.575                  0.560   \n",
       "2                     SVC         0.607                  0.604   \n",
       "3    KNeighborsClassifier         0.503                  0.505   \n",
       "\n",
       "   3-level features_mean  \n",
       "0                  0.594  \n",
       "1                  0.584  \n",
       "2                  0.589  \n",
       "3                  0.504  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results[column_order].round(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'initial': array(['Closeness_initial_median', 'Degrees_initial_median',\n",
      "       'LocalEfficiency_initial_median'], dtype='<U31'), '2-level features': array(['Closeness_G_2click_1simplex_median', 'Closeness_initial_median',\n",
      "       'Degrees_G_2click_1simplex_median', 'Degrees_initial_median',\n",
      "       'GlobalEfficiency_G_2click_1simplex_median',\n",
      "       'GlobalEfficiency_initial_median',\n",
      "       'LocalEfficiency_G_2click_1simplex_median'], dtype='<U41'), '3-level features': array(['AvgNodeDegree_G_3click_2simplex_median',\n",
      "       'Betweenness_G_3click_1simplex_median',\n",
      "       'Betweenness_G_3click_2simplex_median',\n",
      "       'Closeness_G_3click_2simplex_median',\n",
      "       'Clustering_G_3click_1simplex_median',\n",
      "       'Clustering_G_3click_2simplex_median', 'Degrees_initial_median',\n",
      "       'GlobalEfficiency_G_3click_1simplex_median',\n",
      "       'LocalEfficiency_G_3click_1simplex_median',\n",
      "       'LocalEfficiency_initial_median', 'n_3click_2simplex_median'],\n",
      "      dtype='<U41')}\n"
     ]
    }
   ],
   "source": [
    "print(best_features_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def selection_step(model, X, y, used_features=(), cv=3):\n",
    "    \"\"\"\n",
    "    Parameters\n",
    "    ==========\n",
    "        X: ndarray - training inputs\n",
    "        y: ndarray - training targets\n",
    "        used_features: - list of features\n",
    "        cv: int - number of folds\n",
    "\n",
    "    Returns\n",
    "    =======\n",
    "        scores - dictionary of scores\n",
    "    \"\"\"\n",
    "    \n",
    "    scores = {}\n",
    "    \n",
    "    ### BEGIN Solution\n",
    "    scores = cross_val_score(model, X[used_features], y, scoring='roc_auc', cv=cv)\n",
    "    ### END Solution\n",
    "\n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_steps(X, y, n_rounds, method):\n",
    "    \"\"\"\n",
    "    Parameters\n",
    "    ==========\n",
    "        X: ndarray - training inputs\n",
    "        y: ndarray - training targets\n",
    "        n_rounds: int - early stop when score doesn't increase n_rounds\n",
    "        method: sklearn model\n",
    "\n",
    "    Returns\n",
    "    =======\n",
    "        feat_best_list - list of features\n",
    "    \"\"\"\n",
    "    \n",
    "    feat_best_list = []\n",
    "    features_list = X.columns.values.tolist()\n",
    "    prev_cv = np.nan\n",
    "    is_first_iteration = True\n",
    "\n",
    "    n_round = 0\n",
    "    \n",
    "    ### BEGIN Solution\n",
    "    while n_round < n_rounds:\n",
    "        n_round += 1\n",
    "        \n",
    "        if features_list == []:\n",
    "            break\n",
    "        \n",
    "        current_best_cv = np.nan\n",
    "        current_best_feature = np.nan\n",
    "        \n",
    "        for feature in features_list:\n",
    "            \n",
    "            curr_cv = selection_step(method, X, y, used_features= feat_best_list + [feature], cv=3).mean()\n",
    "            \n",
    "            if feature == features_list[0]:\n",
    "                current_best_cv = curr_cv\n",
    "                current_best_feature = feature\n",
    "            elif curr_cv > current_best_cv:\n",
    "                current_best_cv = curr_cv\n",
    "                current_best_feature = feature\n",
    "                \n",
    "        features_list.remove(current_best_feature)\n",
    "        feat_best_list.append(current_best_feature)\n",
    "        \n",
    "        if is_first_iteration: \n",
    "            prev_cv = current_best_cv\n",
    "            is_first_iteration = False\n",
    "        elif current_best_cv >  prev_cv:\n",
    "            n_round = 0\n",
    "            prev_cv = current_best_cv\n",
    "\n",
    "    ### END Solution\n",
    "    \n",
    "    return feat_best_list\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df[best_features_dict['3-level features']]\n",
    "y = target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['LocalEfficiency_initial_median', 'GlobalEfficiency_G_3click_1simplex_median', 'Degrees_initial_median', 'Betweenness_G_3click_2simplex_median', 'Clustering_G_3click_2simplex_median', 'Closeness_G_3click_2simplex_median', 'Betweenness_G_3click_1simplex_median', 'Clustering_G_3click_1simplex_median', 'AvgNodeDegree_G_3click_2simplex_median', 'n_3click_2simplex_median', 'LocalEfficiency_G_3click_1simplex_median']\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "### BEGIN Solution\n",
    "dtr = DecisionTreeClassifier()\n",
    "n_rounds = 7\n",
    "best_features = forward_steps(X, y, n_rounds, dtr)\n",
    "\n",
    "print(best_features)\n",
    "### END Solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5577461205461205\n",
      "0.5436649045399046\n",
      "0.56633947996448\n",
      "0.49798187923187925\n"
     ]
    }
   ],
   "source": [
    "for method in methods_list:\n",
    "    #best_features = forward_steps(X, y, n_rounds, method)\n",
    "    score = cross_val_score(method, X=train_normed, y=target, cv=rk, scoring=scoring_method)\n",
    "    print(score.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5854537878787878\n",
      "0.5625404512154513\n",
      "0.5923399295149294\n",
      "0.5075247641247641\n"
     ]
    }
   ],
   "source": [
    "for method in methods_list:\n",
    "    #best_features = forward_steps(X, y, n_rounds, method)\n",
    "    score = cross_val_score(method, X=train_normed, y=target, cv=rk, scoring=scoring_method)\n",
    "    print(score.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.560134817959818"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "method_scores.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "DMIA",
   "language": "python",
   "name": "dmia"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}