display_results.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Replacing missing premium pupil data NaNs with -1.0\n",
      "\n",
      "\n",
      "Evaluating trained dataset\n",
      "Binarizing predictions using a cutoff of 0.5\n",
      "\n",
      "\n",
      "Evaluating public test dataset\n",
      "Binarizing predictions using a cutoff of 0.5\n",
      "\n",
      "\n",
      "Evaluating private test dataset\n",
      "Binarizing predictions using a cutoff of 0.5\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gender</th>\n",
       "      <th>n</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>auc</th>\n",
       "      <th>auc_max-min</th>\n",
       "      <th>accuracy_max-min</th>\n",
       "      <th>dataset</th>\n",
       "      <th>PremiumPupil</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>7289288.0</td>\n",
       "      <td>0.667466</td>\n",
       "      <td>0.498710</td>\n",
       "      <td>0.001396</td>\n",
       "      <td>0.049722</td>\n",
       "      <td>trained</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>12648838.0</td>\n",
       "      <td>0.673901</td>\n",
       "      <td>0.498861</td>\n",
       "      <td>0.001396</td>\n",
       "      <td>0.049722</td>\n",
       "      <td>trained</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.0</td>\n",
       "      <td>11792098.0</td>\n",
       "      <td>0.684127</td>\n",
       "      <td>0.498971</td>\n",
       "      <td>0.001396</td>\n",
       "      <td>0.049722</td>\n",
       "      <td>trained</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.0</td>\n",
       "      <td>5476.0</td>\n",
       "      <td>0.634405</td>\n",
       "      <td>0.500106</td>\n",
       "      <td>0.001396</td>\n",
       "      <td>0.049722</td>\n",
       "      <td>trained</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>19808536.0</td>\n",
       "      <td>0.659590</td>\n",
       "      <td>0.499122</td>\n",
       "      <td>0.000552</td>\n",
       "      <td>0.074696</td>\n",
       "      <td>trained</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>9244190.0</td>\n",
       "      <td>0.694988</td>\n",
       "      <td>0.498570</td>\n",
       "      <td>0.000552</td>\n",
       "      <td>0.074696</td>\n",
       "      <td>trained</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>2682974.0</td>\n",
       "      <td>0.734286</td>\n",
       "      <td>0.498652</td>\n",
       "      <td>0.000552</td>\n",
       "      <td>0.074696</td>\n",
       "      <td>trained</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>7285764.0</td>\n",
       "      <td>0.667452</td>\n",
       "      <td>0.498710</td>\n",
       "      <td>0.004979</td>\n",
       "      <td>0.102343</td>\n",
       "      <td>trained</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2772.0</td>\n",
       "      <td>0.701659</td>\n",
       "      <td>0.499521</td>\n",
       "      <td>0.004979</td>\n",
       "      <td>0.102343</td>\n",
       "      <td>trained</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>752.0</td>\n",
       "      <td>0.675532</td>\n",
       "      <td>0.495127</td>\n",
       "      <td>0.004979</td>\n",
       "      <td>0.102343</td>\n",
       "      <td>trained</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>6586912.0</td>\n",
       "      <td>0.650677</td>\n",
       "      <td>0.499349</td>\n",
       "      <td>0.004979</td>\n",
       "      <td>0.102343</td>\n",
       "      <td>trained</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4662162.0</td>\n",
       "      <td>0.689251</td>\n",
       "      <td>0.498485</td>\n",
       "      <td>0.004979</td>\n",
       "      <td>0.102343</td>\n",
       "      <td>trained</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1399764.0</td>\n",
       "      <td>0.732061</td>\n",
       "      <td>0.498561</td>\n",
       "      <td>0.004979</td>\n",
       "      <td>0.102343</td>\n",
       "      <td>trained</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2.0</td>\n",
       "      <td>5930384.0</td>\n",
       "      <td>0.659853</td>\n",
       "      <td>0.499418</td>\n",
       "      <td>0.004979</td>\n",
       "      <td>0.102343</td>\n",
       "      <td>trained</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2.0</td>\n",
       "      <td>4579256.0</td>\n",
       "      <td>0.700825</td>\n",
       "      <td>0.498650</td>\n",
       "      <td>0.004979</td>\n",
       "      <td>0.102343</td>\n",
       "      <td>trained</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2.0</td>\n",
       "      <td>1282458.0</td>\n",
       "      <td>0.736748</td>\n",
       "      <td>0.498752</td>\n",
       "      <td>0.004979</td>\n",
       "      <td>0.102343</td>\n",
       "      <td>trained</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>3.0</td>\n",
       "      <td>5476.0</td>\n",
       "      <td>0.634405</td>\n",
       "      <td>0.500106</td>\n",
       "      <td>0.004979</td>\n",
       "      <td>0.102343</td>\n",
       "      <td>trained</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>4100499.0</td>\n",
       "      <td>0.412148</td>\n",
       "      <td>0.497885</td>\n",
       "      <td>0.002624</td>\n",
       "      <td>0.088837</td>\n",
       "      <td>public test</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>7115157.0</td>\n",
       "      <td>0.422706</td>\n",
       "      <td>0.497625</td>\n",
       "      <td>0.002624</td>\n",
       "      <td>0.088837</td>\n",
       "      <td>public test</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.0</td>\n",
       "      <td>6632587.0</td>\n",
       "      <td>0.440845</td>\n",
       "      <td>0.498172</td>\n",
       "      <td>0.002624</td>\n",
       "      <td>0.088837</td>\n",
       "      <td>public test</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.0</td>\n",
       "      <td>3088.0</td>\n",
       "      <td>0.352008</td>\n",
       "      <td>0.500250</td>\n",
       "      <td>0.002624</td>\n",
       "      <td>0.088837</td>\n",
       "      <td>public test</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>11142160.0</td>\n",
       "      <td>0.396879</td>\n",
       "      <td>0.498370</td>\n",
       "      <td>0.000975</td>\n",
       "      <td>0.134796</td>\n",
       "      <td>public test</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>5199999.0</td>\n",
       "      <td>0.461190</td>\n",
       "      <td>0.497396</td>\n",
       "      <td>0.000975</td>\n",
       "      <td>0.134796</td>\n",
       "      <td>public test</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1509172.0</td>\n",
       "      <td>0.531675</td>\n",
       "      <td>0.497625</td>\n",
       "      <td>0.000975</td>\n",
       "      <td>0.134796</td>\n",
       "      <td>public test</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>4098505.0</td>\n",
       "      <td>0.412122</td>\n",
       "      <td>0.497885</td>\n",
       "      <td>0.003689</td>\n",
       "      <td>0.183860</td>\n",
       "      <td>public test</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1562.0</td>\n",
       "      <td>0.471191</td>\n",
       "      <td>0.496560</td>\n",
       "      <td>0.003689</td>\n",
       "      <td>0.183860</td>\n",
       "      <td>public test</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>432.0</td>\n",
       "      <td>0.446759</td>\n",
       "      <td>0.499501</td>\n",
       "      <td>0.003689</td>\n",
       "      <td>0.183860</td>\n",
       "      <td>public test</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>3704762.0</td>\n",
       "      <td>0.380264</td>\n",
       "      <td>0.498531</td>\n",
       "      <td>0.003689</td>\n",
       "      <td>0.183860</td>\n",
       "      <td>public test</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2622726.0</td>\n",
       "      <td>0.451071</td>\n",
       "      <td>0.497015</td>\n",
       "      <td>0.003689</td>\n",
       "      <td>0.183860</td>\n",
       "      <td>public test</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.0</td>\n",
       "      <td>787669.0</td>\n",
       "      <td>0.527883</td>\n",
       "      <td>0.497354</td>\n",
       "      <td>0.003689</td>\n",
       "      <td>0.183860</td>\n",
       "      <td>public test</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2.0</td>\n",
       "      <td>3335805.0</td>\n",
       "      <td>0.396644</td>\n",
       "      <td>0.498869</td>\n",
       "      <td>0.003689</td>\n",
       "      <td>0.183860</td>\n",
       "      <td>public test</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2575711.0</td>\n",
       "      <td>0.471488</td>\n",
       "      <td>0.497749</td>\n",
       "      <td>0.003689</td>\n",
       "      <td>0.183860</td>\n",
       "      <td>public test</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2.0</td>\n",
       "      <td>721071.0</td>\n",
       "      <td>0.535868</td>\n",
       "      <td>0.497908</td>\n",
       "      <td>0.003689</td>\n",
       "      <td>0.183860</td>\n",
       "      <td>public test</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>3.0</td>\n",
       "      <td>3088.0</td>\n",
       "      <td>0.352008</td>\n",
       "      <td>0.500250</td>\n",
       "      <td>0.003689</td>\n",
       "      <td>0.183860</td>\n",
       "      <td>public test</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>4100826.0</td>\n",
       "      <td>0.411247</td>\n",
       "      <td>0.496208</td>\n",
       "      <td>0.004292</td>\n",
       "      <td>0.088490</td>\n",
       "      <td>private test</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>7114588.0</td>\n",
       "      <td>0.422191</td>\n",
       "      <td>0.496802</td>\n",
       "      <td>0.004292</td>\n",
       "      <td>0.088490</td>\n",
       "      <td>private test</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.0</td>\n",
       "      <td>6632832.0</td>\n",
       "      <td>0.440402</td>\n",
       "      <td>0.497472</td>\n",
       "      <td>0.004292</td>\n",
       "      <td>0.088490</td>\n",
       "      <td>private test</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.0</td>\n",
       "      <td>3086.0</td>\n",
       "      <td>0.351912</td>\n",
       "      <td>0.500500</td>\n",
       "      <td>0.004292</td>\n",
       "      <td>0.088490</td>\n",
       "      <td>private test</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>11142454.0</td>\n",
       "      <td>0.396445</td>\n",
       "      <td>0.497372</td>\n",
       "      <td>0.001061</td>\n",
       "      <td>0.134681</td>\n",
       "      <td>private test</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>5199856.0</td>\n",
       "      <td>0.460305</td>\n",
       "      <td>0.496310</td>\n",
       "      <td>0.001061</td>\n",
       "      <td>0.134681</td>\n",
       "      <td>private test</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1509022.0</td>\n",
       "      <td>0.531125</td>\n",
       "      <td>0.497231</td>\n",
       "      <td>0.001061</td>\n",
       "      <td>0.134681</td>\n",
       "      <td>private test</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>4098849.0</td>\n",
       "      <td>0.411223</td>\n",
       "      <td>0.496207</td>\n",
       "      <td>0.004672</td>\n",
       "      <td>0.183859</td>\n",
       "      <td>private test</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1553.0</td>\n",
       "      <td>0.469414</td>\n",
       "      <td>0.497729</td>\n",
       "      <td>0.004672</td>\n",
       "      <td>0.183859</td>\n",
       "      <td>private test</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>424.0</td>\n",
       "      <td>0.433962</td>\n",
       "      <td>0.496687</td>\n",
       "      <td>0.004672</td>\n",
       "      <td>0.183859</td>\n",
       "      <td>private test</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>3704792.0</td>\n",
       "      <td>0.380132</td>\n",
       "      <td>0.497944</td>\n",
       "      <td>0.004672</td>\n",
       "      <td>0.183859</td>\n",
       "      <td>private test</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2622721.0</td>\n",
       "      <td>0.450175</td>\n",
       "      <td>0.495828</td>\n",
       "      <td>0.004672</td>\n",
       "      <td>0.183859</td>\n",
       "      <td>private test</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.0</td>\n",
       "      <td>787075.0</td>\n",
       "      <td>0.526919</td>\n",
       "      <td>0.496841</td>\n",
       "      <td>0.004672</td>\n",
       "      <td>0.183859</td>\n",
       "      <td>private test</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2.0</td>\n",
       "      <td>3335727.0</td>\n",
       "      <td>0.396445</td>\n",
       "      <td>0.498350</td>\n",
       "      <td>0.004672</td>\n",
       "      <td>0.183859</td>\n",
       "      <td>private test</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2575582.0</td>\n",
       "      <td>0.470615</td>\n",
       "      <td>0.496758</td>\n",
       "      <td>0.004672</td>\n",
       "      <td>0.183859</td>\n",
       "      <td>private test</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2.0</td>\n",
       "      <td>721523.0</td>\n",
       "      <td>0.535771</td>\n",
       "      <td>0.497643</td>\n",
       "      <td>0.004672</td>\n",
       "      <td>0.183859</td>\n",
       "      <td>private test</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>3.0</td>\n",
       "      <td>3086.0</td>\n",
       "      <td>0.351912</td>\n",
       "      <td>0.500500</td>\n",
       "      <td>0.004672</td>\n",
       "      <td>0.183859</td>\n",
       "      <td>private test</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Gender           n  accuracy       auc  auc_max-min  accuracy_max-min  \\\n",
       "0     0.0   7289288.0  0.667466  0.498710     0.001396          0.049722   \n",
       "1     1.0  12648838.0  0.673901  0.498861     0.001396          0.049722   \n",
       "2     2.0  11792098.0  0.684127  0.498971     0.001396          0.049722   \n",
       "3     3.0      5476.0  0.634405  0.500106     0.001396          0.049722   \n",
       "0     NaN  19808536.0  0.659590  0.499122     0.000552          0.074696   \n",
       "1     NaN   9244190.0  0.694988  0.498570     0.000552          0.074696   \n",
       "2     NaN   2682974.0  0.734286  0.498652     0.000552          0.074696   \n",
       "0     0.0   7285764.0  0.667452  0.498710     0.004979          0.102343   \n",
       "1     0.0      2772.0  0.701659  0.499521     0.004979          0.102343   \n",
       "2     0.0       752.0  0.675532  0.495127     0.004979          0.102343   \n",
       "3     1.0   6586912.0  0.650677  0.499349     0.004979          0.102343   \n",
       "4     1.0   4662162.0  0.689251  0.498485     0.004979          0.102343   \n",
       "5     1.0   1399764.0  0.732061  0.498561     0.004979          0.102343   \n",
       "6     2.0   5930384.0  0.659853  0.499418     0.004979          0.102343   \n",
       "7     2.0   4579256.0  0.700825  0.498650     0.004979          0.102343   \n",
       "8     2.0   1282458.0  0.736748  0.498752     0.004979          0.102343   \n",
       "9     3.0      5476.0  0.634405  0.500106     0.004979          0.102343   \n",
       "0     0.0   4100499.0  0.412148  0.497885     0.002624          0.088837   \n",
       "1     1.0   7115157.0  0.422706  0.497625     0.002624          0.088837   \n",
       "2     2.0   6632587.0  0.440845  0.498172     0.002624          0.088837   \n",
       "3     3.0      3088.0  0.352008  0.500250     0.002624          0.088837   \n",
       "0     NaN  11142160.0  0.396879  0.498370     0.000975          0.134796   \n",
       "1     NaN   5199999.0  0.461190  0.497396     0.000975          0.134796   \n",
       "2     NaN   1509172.0  0.531675  0.497625     0.000975          0.134796   \n",
       "0     0.0   4098505.0  0.412122  0.497885     0.003689          0.183860   \n",
       "1     0.0      1562.0  0.471191  0.496560     0.003689          0.183860   \n",
       "2     0.0       432.0  0.446759  0.499501     0.003689          0.183860   \n",
       "3     1.0   3704762.0  0.380264  0.498531     0.003689          0.183860   \n",
       "4     1.0   2622726.0  0.451071  0.497015     0.003689          0.183860   \n",
       "5     1.0    787669.0  0.527883  0.497354     0.003689          0.183860   \n",
       "6     2.0   3335805.0  0.396644  0.498869     0.003689          0.183860   \n",
       "7     2.0   2575711.0  0.471488  0.497749     0.003689          0.183860   \n",
       "8     2.0    721071.0  0.535868  0.497908     0.003689          0.183860   \n",
       "9     3.0      3088.0  0.352008  0.500250     0.003689          0.183860   \n",
       "0     0.0   4100826.0  0.411247  0.496208     0.004292          0.088490   \n",
       "1     1.0   7114588.0  0.422191  0.496802     0.004292          0.088490   \n",
       "2     2.0   6632832.0  0.440402  0.497472     0.004292          0.088490   \n",
       "3     3.0      3086.0  0.351912  0.500500     0.004292          0.088490   \n",
       "0     NaN  11142454.0  0.396445  0.497372     0.001061          0.134681   \n",
       "1     NaN   5199856.0  0.460305  0.496310     0.001061          0.134681   \n",
       "2     NaN   1509022.0  0.531125  0.497231     0.001061          0.134681   \n",
       "0     0.0   4098849.0  0.411223  0.496207     0.004672          0.183859   \n",
       "1     0.0      1553.0  0.469414  0.497729     0.004672          0.183859   \n",
       "2     0.0       424.0  0.433962  0.496687     0.004672          0.183859   \n",
       "3     1.0   3704792.0  0.380132  0.497944     0.004672          0.183859   \n",
       "4     1.0   2622721.0  0.450175  0.495828     0.004672          0.183859   \n",
       "5     1.0    787075.0  0.526919  0.496841     0.004672          0.183859   \n",
       "6     2.0   3335727.0  0.396445  0.498350     0.004672          0.183859   \n",
       "7     2.0   2575582.0  0.470615  0.496758     0.004672          0.183859   \n",
       "8     2.0    721523.0  0.535771  0.497643     0.004672          0.183859   \n",
       "9     3.0      3086.0  0.351912  0.500500     0.004672          0.183859   \n",
       "\n",
       "        dataset  PremiumPupil  \n",
       "0       trained           NaN  \n",
       "1       trained           NaN  \n",
       "2       trained           NaN  \n",
       "3       trained           NaN  \n",
       "0       trained          -1.0  \n",
       "1       trained           0.0  \n",
       "2       trained           1.0  \n",
       "0       trained          -1.0  \n",
       "1       trained           0.0  \n",
       "2       trained           1.0  \n",
       "3       trained          -1.0  \n",
       "4       trained           0.0  \n",
       "5       trained           1.0  \n",
       "6       trained          -1.0  \n",
       "7       trained           0.0  \n",
       "8       trained           1.0  \n",
       "9       trained          -1.0  \n",
       "0   public test           NaN  \n",
       "1   public test           NaN  \n",
       "2   public test           NaN  \n",
       "3   public test           NaN  \n",
       "0   public test          -1.0  \n",
       "1   public test           0.0  \n",
       "2   public test           1.0  \n",
       "0   public test          -1.0  \n",
       "1   public test           0.0  \n",
       "2   public test           1.0  \n",
       "3   public test          -1.0  \n",
       "4   public test           0.0  \n",
       "5   public test           1.0  \n",
       "6   public test          -1.0  \n",
       "7   public test           0.0  \n",
       "8   public test           1.0  \n",
       "9   public test          -1.0  \n",
       "0  private test           NaN  \n",
       "1  private test           NaN  \n",
       "2  private test           NaN  \n",
       "3  private test           NaN  \n",
       "0  private test          -1.0  \n",
       "1  private test           0.0  \n",
       "2  private test           1.0  \n",
       "0  private test          -1.0  \n",
       "1  private test           0.0  \n",
       "2  private test           1.0  \n",
       "3  private test          -1.0  \n",
       "4  private test           0.0  \n",
       "5  private test           1.0  \n",
       "6  private test          -1.0  \n",
       "7  private test           0.0  \n",
       "8  private test           1.0  \n",
       "9  private test          -1.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import accuracy_score, roc_auc_score\n",
    "\n",
    "# Read in student characteristics\n",
    "student_df=pd.read_csv(\"./data/metadata/student_metadata_task_1_2.csv\")\n",
    "print(f\"Replacing missing premium pupil data NaNs with -1.0\")\n",
    "student_df['PremiumPupil']=student_df['PremiumPupil'].replace(np.NaN,-1.0)\n",
    "\n",
    "evaluation_data=[\n",
    "    \"trained\",\n",
    "    \"public test\",\n",
    "    \"private test\"\n",
    "]\n",
    "\n",
    "dataframes=[]\n",
    "for item in evaluation_data:\n",
    "    # Read in predictions\n",
    "    results_df=pd.read_csv(f\"{item}.results.csv\")\n",
    "\n",
    "    # Binarize predictions\n",
    "    cutoff=0.5\n",
    "    results_df['predictions']=results_df['predictions'].apply(lambda x: 1.0 if x > cutoff else 0.0)\n",
    "\n",
    "    # Merge datasets\n",
    "    df=results_df.merge(student_df,left_on=\"userid\",right_on=\"UserId\",how=\"left\")\n",
    "\n",
    "    # creates metrics on a per-group basis\n",
    "    def group_metrics(frame):\n",
    "\n",
    "        return pd.Series({\n",
    "            \"n\":len(frame),\n",
    "            \"accuracy\":accuracy_score(frame['labels'],frame['predictions']),\n",
    "            \"auc\":roc_auc_score(frame['labels'],frame['predictions'])\n",
    "        })\n",
    "\n",
    "    groupings=[\"Gender\",\"PremiumPupil\",[\"Gender\",\"PremiumPupil\"]]\n",
    "    \n",
    "    for group in groupings:\n",
    "        summary_df=df.groupby(group).apply(group_metrics).reset_index()\n",
    "        # add in metrics across the whole group\n",
    "        summary_df['auc_max-min']=max(summary_df['auc'])-min(summary_df['auc'])\n",
    "        summary_df['accuracy_max-min']=max(summary_df['accuracy'])-min(summary_df['accuracy'])\n",
    "        summary_df['dataset']=item\n",
    "        dataframes.append(summary_df)\n",
    "    \n",
    "display(pd.concat(dataframes))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}