pressureprediction-lstm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f0dbec88",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2021-11-01T22:50:51.108347Z",
     "iopub.status.busy": "2021-11-01T22:50:51.106837Z",
     "iopub.status.idle": "2021-11-01T22:50:56.776396Z",
     "shell.execute_reply": "2021-11-01T22:50:56.776932Z",
     "shell.execute_reply.started": "2021-10-28T18:07:59.529018Z"
    },
    "papermill": {
     "duration": 5.685262,
     "end_time": "2021-11-01T22:50:56.777219",
     "exception": false,
     "start_time": "2021-11-01T22:50:51.091957",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/ventilator-pressure-prediction/sample_submission.csv\n",
      "/kaggle/input/ventilator-pressure-prediction/train.csv\n",
      "/kaggle/input/ventilator-pressure-prediction/test.csv\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import math\n",
    "\n",
    "# Input data files are available in the read-only \"../input/\" directory\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import gc\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n",
    "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "import random\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "import joblib\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import KFold, StratifiedKFold, GroupKFold\n",
    "\n",
    "from sklearn.metrics import accuracy_score, make_scorer\n",
    "from sklearn.metrics import roc_curve, auc, accuracy_score, cohen_kappa_score\n",
    "from sklearn.metrics import mean_absolute_error, mean_squared_error, f1_score, confusion_matrix\n",
    "\n",
    "from sklearn.preprocessing import RobustScaler\n",
    "\n",
    "\n",
    "import tensorflow as tf\n",
    "import tensorflow.keras as keras\n",
    "\n",
    "from tensorflow.keras.models import Sequential, Model\n",
    "\n",
    "#from tensorflow.keras.layers import InputLayer\n",
    "from tensorflow.keras.layers import LSTM, Bidirectional, add, concatenate, GlobalMaxPooling1D, GlobalAveragePooling1D\n",
    "from tensorflow.keras.layers import Conv1D, Conv2D, MaxPooling1D, MaxPooling2D, Conv2DTranspose, AveragePooling1D, UpSampling1D\n",
    "from tensorflow.keras.layers import Dense, Input, Dropout, BatchNormalization, Activation, TimeDistributed\n",
    "from tensorflow.keras.layers import Multiply, Add, Concatenate, Flatten, Average, Lambda\n",
    "\n",
    "from tensorflow.keras.optimizers import Adam, SGD\n",
    "from tensorflow.keras.callbacks import EarlyStopping, Callback, ModelCheckpoint, ReduceLROnPlateau\n",
    "from tensorflow.keras.constraints import unit_norm, max_norm\n",
    "\n",
    "from tensorflow.keras import backend as K\n",
    "from tensorflow_addons.optimizers import CyclicalLearningRate"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f180327",
   "metadata": {
    "papermill": {
     "duration": 0.011256,
     "end_time": "2021-11-01T22:50:56.799670",
     "exception": false,
     "start_time": "2021-11-01T22:50:56.788414",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Google Brain - Ventilator Pressure Prediction\n",
    "\n",
    "https://www.kaggle.com/c/ventilator-pressure-prediction\n",
    "\n",
    "**Columns:**\n",
    "\n",
    "**id** - globally-unique time step identifier across an entire file\n",
    "\n",
    "**breath_id** - globally-unique time step for breaths\n",
    "\n",
    "**R** - lung attribute indicating how restricted the airway is (in cmH2O/L/S). Physically, this is the change in pressure per change in flow (air volume per time). Intuitively, one can imagine blowing up a balloon through a straw. We can change R by changing the diameter of the straw, with higher R being harder to blow.\n",
    "\n",
    "**C** - lung attribute indicating how compliant the lung is (in mL/cmH2O). Physically, this is the change in volume per change in pressure. Intuitively, one can imagine the same balloon example. We can change C by changing the thickness of the balloon’s latex, with higher C having thinner latex and easier to blow.\n",
    "\n",
    "**time_step** - the actual time stamp.\n",
    "\n",
    "**u_in** - the control input for the inspiratory solenoid valve. Ranges from 0 to 100.\n",
    "\n",
    "**u_out** - the control input for the exploratory solenoid valve. Either 0 or 1.\n",
    "\n",
    "**pressure** - the airway pressure measured in the respiratory circuit, measured in cmH2O.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d82d0a6d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-01T22:50:56.848192Z",
     "iopub.status.busy": "2021-11-01T22:50:56.847399Z",
     "iopub.status.idle": "2021-11-01T22:50:56.849490Z",
     "shell.execute_reply": "2021-11-01T22:50:56.850356Z",
     "shell.execute_reply.started": "2021-10-28T18:07:59.549747Z"
    },
    "papermill": {
     "duration": 0.04047,
     "end_time": "2021-11-01T22:50:56.850506",
     "exception": false,
     "start_time": "2021-11-01T22:50:56.810036",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def reduce_mem_usage(df, verbose=True):\n",
    "    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n",
    "    start_mem = df.memory_usage().sum() / 1024**2\n",
    "    for col in df.columns:\n",
    "        if col != 'time':\n",
    "            col_type = df[col].dtypes\n",
    "            if col_type in numerics:\n",
    "                c_min = df[col].min()\n",
    "                c_max = df[col].max()\n",
    "                if str(col_type)[:3] == 'int':\n",
    "                    if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                        df[col] = df[col].astype(np.int8)\n",
    "                    elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                        df[col] = df[col].astype(np.int16)\n",
    "                    elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                        df[col] = df[col].astype(np.int32)\n",
    "                    elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                        df[col] = df[col].astype(np.int64)  \n",
    "                else:\n",
    "                    if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                        df[col] = df[col].astype(np.float16)\n",
    "                    elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                        df[col] = df[col].astype(np.float32)\n",
    "                    else:\n",
    "                        df[col] = df[col].astype(np.float64)    \n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n",
    "    return df\n",
    "\n",
    "def get_stats(df):\n",
    "    stats = pd.DataFrame(index=df.columns, columns=['na_count', 'n_unique', 'type', 'memory_usage'])\n",
    "    for col in df.columns:\n",
    "        stats.loc[col] = [df[col].isna().sum(), df[col].nunique(dropna=False), df[col].dtypes, df[col].memory_usage(deep=True, index=False) / 1024**2]\n",
    "    stats.loc['Overall'] = [stats['na_count'].sum(), stats['n_unique'].sum(), None, df.memory_usage(deep=True).sum() / 1024**2]\n",
    "    return stats\n",
    "\n",
    "def print_header():\n",
    "    print('col         conversion        dtype    na    uniq  size')\n",
    "    print()\n",
    "    \n",
    "def print_values(name, conversion, col):\n",
    "    template = '{:10}  {:16}  {:>7}  {:2}  {:6}  {:1.2f}MB'\n",
    "    print(template.format(name, conversion, str(col.dtypes), col.isna().sum(), col.nunique(dropna=False), col.memory_usage(deep=True, index=False) / 1024 ** 2))\n",
    "    \n",
    "def seed_everything(seed):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    #tf.random.set_seed(seed)    \n",
    "    \n",
    "def draw_sequence( df, start, end, filter_out=None):\n",
    "    f, (ax1, ax2, ax3) = plt.subplots(3, 1, figsize=(16, 8))\n",
    "    for i in range(start, end):\n",
    "        if filter_out is not None and i in filter_out:\n",
    "            continue\n",
    "        df1 = df[df.breath_id == i]\n",
    "        sns.lineplot( data = df1[['u_in']], ax=ax1)\n",
    "        sns.lineplot( data = df1[['u_out']], ax=ax2)    \n",
    "        sns.lineplot( data = df1[['pressure']], ax=ax3)    \n",
    "        \n",
    "def draw_in_out_pressure( df, breath_id1, breath_id2 ):\n",
    "    f, (ax1, ax2) = plt.subplots(2, 1, figsize=(16, 8))\n",
    "    df1 = df[df.breath_id == breath_id1]\n",
    "    sns.lineplot( data = df1[['u_in', 'u_out', 'pressure']], ax=ax1)\n",
    "    df2 = df[df.breath_id == breath_id2]\n",
    "    sns.lineplot( data = df2[['u_in', 'u_out', 'pressure']], ax=ax2)\n",
    "        \n",
    "def display_missing(df):\n",
    "    total = df.isnull().sum().sort_values(ascending=False)\n",
    "    percent = (df.isnull().sum()/df.isnull().count()).sort_values(ascending=False)\n",
    "    missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\n",
    "    missing_data.head()\n",
    "    return missing_data       \n",
    "\n",
    "def select_train_data(df, features, train_size=1000, target_col='pressure'):\n",
    "\n",
    "    data_points = SAMPLE_SIZE*train_size\n",
    "    y = df[target_col][:data_points]\n",
    "    X = df[features][:data_points]\n",
    "    \n",
    "    groups = df['breath_id'][:data_points]\n",
    "    print(f'Original sizes: train: {X.shape}, y_train: {y.shape}' )\n",
    "    return X, y, groups\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c004ac24",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-01T22:50:56.876207Z",
     "iopub.status.busy": "2021-11-01T22:50:56.875485Z",
     "iopub.status.idle": "2021-11-01T22:51:10.969668Z",
     "shell.execute_reply": "2021-11-01T22:51:10.968788Z",
     "shell.execute_reply.started": "2021-10-28T18:07:59.578802Z"
    },
    "papermill": {
     "duration": 14.10904,
     "end_time": "2021-11-01T22:51:10.969833",
     "exception": false,
     "start_time": "2021-11-01T22:50:56.860793",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "RANDOM_SEED = 42\n",
    "SAMPLE_SIZE = 80\n",
    "DEBUG = False\n",
    "HYPER_TUNING = False\n",
    "\n",
    "seed_everything(RANDOM_SEED)\n",
    "\n",
    "train = pd.read_csv('/kaggle/input/ventilator-pressure-prediction/train.csv')\n",
    "test = pd.read_csv('/kaggle/input/ventilator-pressure-prediction/test.csv')\n",
    "submission = pd.read_csv('/kaggle/input/ventilator-pressure-prediction/sample_submission.csv')\n",
    "\n",
    "if DEBUG:\n",
    "    train = train[:SAMPLE_SIZE*10000]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "399b4139",
   "metadata": {
    "papermill": {
     "duration": 0.010128,
     "end_time": "2021-11-01T22:51:10.990808",
     "exception": false,
     "start_time": "2021-11-01T22:51:10.980680",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Feature generation\n",
    "\n",
    "https://www.kaggle.com/cdeotte/lstm-feature-importance\n",
    "\n",
    "https://www.kaggle.com/c/ventilator-pressure-prediction/discussion/280471\n",
    "\n",
    "https://www.kaggle.com/swaralipibose/interesting-feature-importance-by-lstm-gradients\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cb2248e6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-01T22:51:11.046074Z",
     "iopub.status.busy": "2021-11-01T22:51:11.045047Z",
     "iopub.status.idle": "2021-11-01T22:52:22.936467Z",
     "shell.execute_reply": "2021-11-01T22:52:22.935989Z",
     "shell.execute_reply.started": "2021-10-28T18:08:13.67726Z"
    },
    "papermill": {
     "duration": 71.935201,
     "end_time": "2021-11-01T22:52:22.936656",
     "exception": false,
     "start_time": "2021-11-01T22:51:11.001455",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mem. usage decreased to 564.13 Mb (70.7% reduction)\n",
      "Mem. usage decreased to 368.41 Mb (70.6% reduction)\n"
     ]
    }
   ],
   "source": [
    "def gen_features(df):    \n",
    "    df['cross']= df['u_in'] * df['u_out']\n",
    "    df['cross2']= df['time_step'] * df['u_out']\n",
    "    df['area'] = df['time_step'] * df['u_in']\n",
    "    df['area'] = df.groupby('breath_id')['area'].cumsum()\n",
    "    df['time_step_cumsum'] = df.groupby(['breath_id'])['time_step'].cumsum()\n",
    "    df['u_in_cumsum'] = (df['u_in']).groupby(df['breath_id']).cumsum()    \n",
    "    \n",
    "    df['u_in_lag1'] = df.groupby('breath_id')['u_in'].shift(1)\n",
    "    df['u_out_lag1'] = df.groupby('breath_id')['u_out'].shift(1)\n",
    "    df['u_in_lag_back1'] = df.groupby('breath_id')['u_in'].shift(-1)\n",
    "    df['u_out_lag_back1'] = df.groupby('breath_id')['u_out'].shift(-1)\n",
    "    df['u_in_lag2'] = df.groupby('breath_id')['u_in'].shift(2)\n",
    "    df['u_out_lag2'] = df.groupby('breath_id')['u_out'].shift(2)\n",
    "    df['u_in_lag_back2'] = df.groupby('breath_id')['u_in'].shift(-2)\n",
    "\n",
    "    df['u_in_lag3'] = df.groupby('breath_id')['u_in'].shift(3)\n",
    "    df['u_out_lag3'] = df.groupby('breath_id')['u_out'].shift(3)\n",
    "    df['u_in_lag_back3'] = df.groupby('breath_id')['u_in'].shift(-3)\n",
    "    df['u_out_lag_back3'] = df.groupby('breath_id')['u_out'].shift(-3)        \n",
    "\n",
    "    df['u_in_lag4'] = df.groupby('breath_id')['u_in'].shift(4)\n",
    "    df['u_out_lag4'] = df.groupby('breath_id')['u_out'].shift(4)\n",
    "    df['u_in_lag_back4'] = df.groupby('breath_id')['u_in'].shift(-4)\n",
    "    df['u_out_lag_back4'] = df.groupby('breath_id')['u_out'].shift(-4)        \n",
    "    \n",
    "    df['u_in_diff1'] = df['u_in'] - df['u_in_lag1']\n",
    "    df['u_out_diff1'] = df['u_out'] - df['u_out_lag1']\n",
    "    df['u_in_diff2'] = df['u_in'] - df['u_in_lag2']\n",
    "    df['u_out_diff2'] = df['u_out'] - df['u_out_lag2']    \n",
    "    df['u_in_diff3'] = df['u_in'] - df['u_in_lag3']\n",
    "    df['u_out_diff3'] = df['u_out'] - df['u_out_lag3']\n",
    "    df['u_in_diff4'] = df['u_in'] - df['u_in_lag4']\n",
    "    df['u_out_diff4'] = df['u_out'] - df['u_out_lag4']\n",
    "    \n",
    "    df['breath_id__u_in__max'] = df.groupby(['breath_id'])['u_in'].transform('max')\n",
    "    df['breath_id__u_in__mean'] = df.groupby(['breath_id'])['u_in'].transform('mean')\n",
    "    df['breath_id__u_in__diffmax'] = df.groupby(['breath_id'])['u_in'].transform('max') - df['u_in']\n",
    "    df['breath_id__u_in__diffmean'] = df.groupby(['breath_id'])['u_in'].transform('mean') - df['u_in']\n",
    "    \n",
    "    df['time_gap'] = df['time_step'] - df['time_step'].shift(1).fillna(0)\n",
    "    u_in_gap = df['u_in'] - df['u_in'].shift(1).fillna(0)\n",
    "    df['u_in_rate'] = u_in_gap / df['time_gap']\n",
    "\n",
    "    df['R'] = df['R'].astype(str)\n",
    "    df['C'] = df['C'].astype(str)\n",
    "    df['R__C'] = df[\"R\"].astype(str) + '__' + df[\"C\"].astype(str)\n",
    "    df = pd.get_dummies(df)   \n",
    "    \n",
    "    df = df.replace([np.inf, -np.inf], np.nan)    \n",
    "    df.fillna(0, inplace=True)\n",
    "    df = reduce_mem_usage(df)\n",
    "    gc.collect()    \n",
    "    \n",
    "    return df\n",
    "\n",
    "train = gen_features(train)\n",
    "test = gen_features(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3adeb3d4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-01T22:52:22.968658Z",
     "iopub.status.busy": "2021-11-01T22:52:22.967472Z",
     "iopub.status.idle": "2021-11-01T22:52:52.354661Z",
     "shell.execute_reply": "2021-11-01T22:52:52.354249Z",
     "shell.execute_reply.started": "2021-10-28T18:08:45.75951Z"
    },
    "papermill": {
     "duration": 29.406251,
     "end_time": "2021-11-01T22:52:52.354823",
     "exception": false,
     "start_time": "2021-11-01T22:52:22.948572",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6036000, 52)\n",
      "(4024000, 52)\n",
      "(75450, 80, 52)\n",
      "(50300, 80, 52)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ALL_FEATURES = ALL_FEATURES = [c for c in train.columns if c not in ['id', 'pressure', 'breath_id']]\n",
    "\n",
    "targets = train[['pressure']].to_numpy().reshape(-1, SAMPLE_SIZE)\n",
    "train.drop(['pressure', 'id', 'breath_id'], axis=1, inplace=True)\n",
    "test = test.drop(['id', 'breath_id'], axis=1)\n",
    "\n",
    "#if DEBUG:\n",
    "#    print(train.columns)\n",
    "#    print(test.columns)\n",
    "    \n",
    "scaler = RobustScaler()\n",
    "train = scaler.fit_transform(train)\n",
    "test = scaler.transform(test)\n",
    "\n",
    "print(train.shape)\n",
    "print(test.shape)\n",
    "\n",
    "train = train.reshape(-1, SAMPLE_SIZE, train.shape[-1])\n",
    "test = test.reshape(-1, SAMPLE_SIZE, test.shape[-1])\n",
    "\n",
    "print(train.shape)\n",
    "print(test.shape)\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6394b3c",
   "metadata": {
    "papermill": {
     "duration": 0.010636,
     "end_time": "2021-11-01T22:52:52.376619",
     "exception": false,
     "start_time": "2021-11-01T22:52:52.365983",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Train\n",
    "\n",
    "https://www.kaggle.com/marutama/eda-about-lstm-feature-importance\n",
    "\n",
    "https://www.kaggle.com/marutama/finetune-of-tensorflow-bi-lstm-eda-about\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "eb8b5461",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-01T22:52:52.410602Z",
     "iopub.status.busy": "2021-11-01T22:52:52.409080Z",
     "iopub.status.idle": "2021-11-01T22:52:52.411214Z",
     "shell.execute_reply": "2021-11-01T22:52:52.411603Z",
     "shell.execute_reply.started": "2021-10-28T18:08:55.090717Z"
    },
    "papermill": {
     "duration": 0.024283,
     "end_time": "2021-11-01T22:52:52.411758",
     "exception": false,
     "start_time": "2021-11-01T22:52:52.387475",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def build_model( X, units = 64, optimizer='adam'):\n",
    "    #shape = (X.shape[-2], X.shape[-1])\n",
    "    model = Sequential([\n",
    "        Input(shape=X.shape[-2:]),\n",
    "        Bidirectional(LSTM(512, return_sequences=True)),\n",
    "        Bidirectional(LSTM(256, return_sequences=True)),\n",
    "        Bidirectional(LSTM(128, return_sequences=True)),\n",
    "        Dense(128, activation='selu'),\n",
    "        Dense(1),\n",
    "    ])    \n",
    "\n",
    "    model.compile(optimizer=optimizer, loss=\"mae\")\n",
    "    return model\n",
    "\n",
    "\n",
    "def display_feature_importance( model, X_valid, cols):\n",
    "    results = []\n",
    "    print(' Computing LSTM feature importance...')\n",
    "\n",
    "    for k in tqdm(range(len(cols))):\n",
    "        if k>0: \n",
    "            save_col = X_valid[:,:,k-1].copy()\n",
    "            np.random.shuffle(X_valid[:,:,k-1])\n",
    "\n",
    "        oof_preds = model.predict(X_valid, verbose=0).squeeze() \n",
    "        mae = np.mean(np.abs( oof_preds-y_valid ))\n",
    "        results.append({'feature':cols[k],'mae':mae})\n",
    "\n",
    "        if k>0: \n",
    "            X_valid[:,:,k-1] = save_col\n",
    "\n",
    "    # DISPLAY LSTM FEATURE IMPORTANCE\n",
    "    print()\n",
    "    df = pd.DataFrame(results)\n",
    "    df = df.sort_values('mae')\n",
    "    plt.figure(figsize=(10,20))\n",
    "    plt.barh(np.arange(len(cols)),df.mae)\n",
    "    plt.yticks(np.arange(len(cols)),df.feature.values)\n",
    "    plt.title('LSTM Feature Importance',size=16)\n",
    "    plt.ylim((-1,len(cols)))\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5491fb1d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-01T22:52:52.445381Z",
     "iopub.status.busy": "2021-11-01T22:52:52.444836Z",
     "iopub.status.idle": "2021-11-02T02:34:43.719529Z",
     "shell.execute_reply": "2021-11-02T02:34:43.719067Z",
     "shell.execute_reply.started": "2021-10-28T18:24:43.481698Z"
    },
    "papermill": {
     "duration": 13311.296893,
     "end_time": "2021-11-02T02:34:43.719662",
     "exception": false,
     "start_time": "2021-11-01T22:52:52.422769",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- > Fold 1 < ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-01 22:52:52.855778: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2021-11-01 22:52:52.944132: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2021-11-01 22:52:52.944915: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2021-11-01 22:52:52.946650: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2021-11-01 22:52:52.947542: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2021-11-01 22:52:52.948211: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2021-11-01 22:52:52.948819: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2021-11-01 22:52:54.792482: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2021-11-01 22:52:54.794039: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2021-11-01 22:52:54.795258: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2021-11-01 22:52:54.796362: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 15403 MB memory:  -> device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0\n",
      "2021-11-01 22:52:57.577969: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 941607680 exceeds 10% of free system memory.\n",
      "2021-11-01 22:52:58.559445: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 941607680 exceeds 10% of free system memory.\n",
      "2021-11-01 22:52:59.262897: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-01 22:53:06.525501: I tensorflow/stream_executor/cuda/cuda_dnn.cc:369] Loaded cuDNN version 8005\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "56/56 [==============================] - 44s 630ms/step - loss: 2.5660 - val_loss: 1.4675\n",
      "Epoch 2/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.8953 - val_loss: 0.6674\n",
      "Epoch 3/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.6657 - val_loss: 0.6004\n",
      "Epoch 4/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.5592 - val_loss: 0.4986\n",
      "Epoch 5/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.4977 - val_loss: 0.4823\n",
      "Epoch 6/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.4620 - val_loss: 0.4388\n",
      "Epoch 7/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.4437 - val_loss: 0.4577\n",
      "Epoch 8/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.4154 - val_loss: 0.4179\n",
      "Epoch 9/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.4037 - val_loss: 0.3947\n",
      "Epoch 10/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.3944 - val_loss: 0.4443\n",
      "Epoch 11/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.3819 - val_loss: 0.3605\n",
      "Epoch 12/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3631 - val_loss: 0.3624\n",
      "Epoch 13/100\n",
      "56/56 [==============================] - 33s 585ms/step - loss: 0.3651 - val_loss: 0.3624\n",
      "Epoch 14/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.3601 - val_loss: 0.4270\n",
      "Epoch 15/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.3505 - val_loss: 0.3364\n",
      "Epoch 16/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3282 - val_loss: 0.3410\n",
      "Epoch 17/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.3405 - val_loss: 0.4352\n",
      "Epoch 18/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3550 - val_loss: 0.3345\n",
      "Epoch 19/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3265 - val_loss: 0.3349\n",
      "Epoch 20/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.3137 - val_loss: 0.3162\n",
      "Epoch 21/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.3082 - val_loss: 0.3212\n",
      "Epoch 22/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.3049 - val_loss: 0.3056\n",
      "Epoch 23/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2970 - val_loss: 0.3392\n",
      "Epoch 24/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.3115 - val_loss: 0.3471\n",
      "Epoch 25/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.3033 - val_loss: 0.3154\n",
      "Epoch 26/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2846 - val_loss: 0.3085\n",
      "Epoch 27/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3123 - val_loss: 0.3524\n",
      "Epoch 28/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2964 - val_loss: 0.2925\n",
      "Epoch 29/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2804 - val_loss: 0.3071\n",
      "Epoch 30/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2781 - val_loss: 0.3256\n",
      "Epoch 31/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2732 - val_loss: 0.2995\n",
      "Epoch 32/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2852 - val_loss: 0.2814\n",
      "Epoch 33/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2675 - val_loss: 0.2774\n",
      "Epoch 34/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3223 - val_loss: 0.3494\n",
      "Epoch 35/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3067 - val_loss: 0.3273\n",
      "Epoch 36/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2820 - val_loss: 0.2810\n",
      "Epoch 37/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2696 - val_loss: 0.2897\n",
      "Epoch 38/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2593 - val_loss: 0.2638\n",
      "Epoch 39/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2641 - val_loss: 0.3069\n",
      "Epoch 40/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2642 - val_loss: 0.2688\n",
      "Epoch 41/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2634 - val_loss: 0.3450\n",
      "Epoch 42/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2567 - val_loss: 0.2532\n",
      "Epoch 43/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2532 - val_loss: 0.3170\n",
      "Epoch 44/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2462 - val_loss: 0.2876\n",
      "Epoch 45/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2385 - val_loss: 0.2472\n",
      "Epoch 46/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2418 - val_loss: 0.2690\n",
      "Epoch 47/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2520 - val_loss: 0.2849\n",
      "Epoch 48/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2290 - val_loss: 0.2406\n",
      "Epoch 49/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2305 - val_loss: 0.2664\n",
      "Epoch 50/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2448 - val_loss: 0.2478\n",
      "Epoch 51/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2699 - val_loss: 0.2560\n",
      "Epoch 52/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2288 - val_loss: 0.2528\n",
      "Epoch 53/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2832 - val_loss: 0.2833\n",
      "Epoch 54/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2320 - val_loss: 0.2528\n",
      "Epoch 55/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2288 - val_loss: 0.2580\n",
      "Epoch 56/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2427 - val_loss: 0.2767\n",
      "Epoch 57/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2339 - val_loss: 0.2745\n",
      "Epoch 58/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2410 - val_loss: 0.2745\n",
      "\n",
      "Epoch 00058: ReduceLROnPlateau reducing learning rate to 0.0007849999819882214.\n",
      "Epoch 59/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2163 - val_loss: 0.2288\n",
      "Epoch 60/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1998 - val_loss: 0.2309\n",
      "Epoch 61/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.1995 - val_loss: 0.2338\n",
      "Epoch 62/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1982 - val_loss: 0.2277\n",
      "Epoch 63/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1915 - val_loss: 0.2242\n",
      "Epoch 64/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1883 - val_loss: 0.2180\n",
      "Epoch 65/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1865 - val_loss: 0.2139\n",
      "Epoch 66/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1831 - val_loss: 0.2160\n",
      "Epoch 67/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1814 - val_loss: 0.2161\n",
      "Epoch 68/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1794 - val_loss: 0.2220\n",
      "Epoch 69/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1834 - val_loss: 0.2092\n",
      "Epoch 70/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1798 - val_loss: 0.2131\n",
      "Epoch 71/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1833 - val_loss: 0.2211\n",
      "Epoch 72/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1905 - val_loss: 0.2185\n",
      "Epoch 73/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1826 - val_loss: 0.2093\n",
      "Epoch 74/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1749 - val_loss: 0.2069\n",
      "Epoch 75/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1734 - val_loss: 0.2083\n",
      "Epoch 76/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1753 - val_loss: 0.2106\n",
      "Epoch 77/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1839 - val_loss: 0.2691\n",
      "Epoch 78/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1930 - val_loss: 0.2117\n",
      "Epoch 79/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1785 - val_loss: 0.2084\n",
      "Epoch 80/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1741 - val_loss: 0.2235\n",
      "Epoch 81/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1732 - val_loss: 0.2414\n",
      "Epoch 82/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1840 - val_loss: 0.2112\n",
      "Epoch 83/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1900 - val_loss: 0.2180\n",
      "Epoch 84/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1850 - val_loss: 0.2149\n",
      "\n",
      "Epoch 00084: ReduceLROnPlateau reducing learning rate to 0.0003924999909941107.\n",
      "Epoch 85/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1690 - val_loss: 0.2024\n",
      "Epoch 86/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1625 - val_loss: 0.1996\n",
      "Epoch 87/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1589 - val_loss: 0.1984\n",
      "Epoch 88/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1591 - val_loss: 0.2019\n",
      "Epoch 89/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1569 - val_loss: 0.1977\n",
      "Epoch 90/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1563 - val_loss: 0.1972\n",
      "Epoch 91/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1558 - val_loss: 0.1991\n",
      "Epoch 92/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1554 - val_loss: 0.2093\n",
      "Epoch 93/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1595 - val_loss: 0.1992\n",
      "Epoch 94/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1532 - val_loss: 0.1974\n",
      "Epoch 95/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1542 - val_loss: 0.1999\n",
      "Epoch 96/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1527 - val_loss: 0.1982\n",
      "Epoch 97/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1520 - val_loss: 0.1975\n",
      "Epoch 98/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1519 - val_loss: 0.1955\n",
      "Epoch 99/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1532 - val_loss: 0.2057\n",
      "Epoch 100/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1544 - val_loss: 0.1983\n",
      "MAE: 0.19828739762306213\n",
      "--------------- > Fold 2 < ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-01 23:48:54.138226: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 941607680 exceeds 10% of free system memory.\n",
      "2021-11-01 23:48:55.088738: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 941607680 exceeds 10% of free system memory.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "56/56 [==============================] - 41s 629ms/step - loss: 2.4275 - val_loss: 1.0627\n",
      "Epoch 2/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.8862 - val_loss: 0.7402\n",
      "Epoch 3/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.6464 - val_loss: 0.6421\n",
      "Epoch 4/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.5746 - val_loss: 0.5367\n",
      "Epoch 5/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.5054 - val_loss: 0.5637\n",
      "Epoch 6/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.4720 - val_loss: 0.4583\n",
      "Epoch 7/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.4464 - val_loss: 0.4269\n",
      "Epoch 8/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.4229 - val_loss: 0.4421\n",
      "Epoch 9/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.4186 - val_loss: 0.3841\n",
      "Epoch 10/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.3982 - val_loss: 0.3966\n",
      "Epoch 11/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3822 - val_loss: 0.3798\n",
      "Epoch 12/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3938 - val_loss: 0.4312\n",
      "Epoch 13/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3779 - val_loss: 0.3890\n",
      "Epoch 14/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3610 - val_loss: 0.3528\n",
      "Epoch 15/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.3430 - val_loss: 0.3601\n",
      "Epoch 16/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3376 - val_loss: 0.3392\n",
      "Epoch 17/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3374 - val_loss: 0.3598\n",
      "Epoch 18/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3244 - val_loss: 0.3450\n",
      "Epoch 19/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3183 - val_loss: 0.3235\n",
      "Epoch 20/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.3075 - val_loss: 0.4084\n",
      "Epoch 21/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3392 - val_loss: 0.3712\n",
      "Epoch 22/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3079 - val_loss: 0.3256\n",
      "Epoch 23/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3002 - val_loss: 0.3136\n",
      "Epoch 24/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3018 - val_loss: 0.3428\n",
      "Epoch 25/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3111 - val_loss: 0.4157\n",
      "Epoch 26/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3131 - val_loss: 0.3493\n",
      "Epoch 27/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2842 - val_loss: 0.2989\n",
      "Epoch 28/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2834 - val_loss: 0.2892\n",
      "Epoch 29/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3007 - val_loss: 0.3014\n",
      "Epoch 30/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2846 - val_loss: 0.3574\n",
      "Epoch 31/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2911 - val_loss: 0.3246\n",
      "Epoch 32/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2868 - val_loss: 0.3012\n",
      "Epoch 33/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2865 - val_loss: 0.3061\n",
      "Epoch 34/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2764 - val_loss: 0.2835\n",
      "Epoch 35/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2715 - val_loss: 0.2899\n",
      "Epoch 36/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2770 - val_loss: 0.2844\n",
      "Epoch 37/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2743 - val_loss: 0.3094\n",
      "Epoch 38/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2865 - val_loss: 0.3135\n",
      "Epoch 39/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2779 - val_loss: 0.2795\n",
      "Epoch 40/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2809 - val_loss: 0.2668\n",
      "Epoch 41/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2732 - val_loss: 0.2928\n",
      "Epoch 42/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2526 - val_loss: 0.2608\n",
      "Epoch 43/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2384 - val_loss: 0.2532\n",
      "Epoch 44/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2486 - val_loss: 0.2824\n",
      "Epoch 45/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2447 - val_loss: 0.2650\n",
      "Epoch 46/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2450 - val_loss: 0.2920\n",
      "Epoch 47/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2497 - val_loss: 0.2699\n",
      "Epoch 48/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2312 - val_loss: 0.2521\n",
      "Epoch 49/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2324 - val_loss: 0.2514\n",
      "Epoch 50/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2451 - val_loss: 0.3217\n",
      "Epoch 51/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2643 - val_loss: 0.3337\n",
      "Epoch 52/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2425 - val_loss: 0.2534\n",
      "Epoch 53/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2371 - val_loss: 0.2750\n",
      "Epoch 54/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2331 - val_loss: 0.2680\n",
      "Epoch 55/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2406 - val_loss: 0.2720\n",
      "Epoch 56/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2351 - val_loss: 0.2385\n",
      "Epoch 57/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2332 - val_loss: 0.3125\n",
      "Epoch 58/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2564 - val_loss: 0.2662\n",
      "Epoch 59/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2251 - val_loss: 0.2469\n",
      "Epoch 60/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2174 - val_loss: 0.2302\n",
      "Epoch 61/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2084 - val_loss: 0.2372\n",
      "Epoch 62/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2182 - val_loss: 0.2480\n",
      "Epoch 63/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2129 - val_loss: 0.2402\n",
      "Epoch 64/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2011 - val_loss: 0.2311\n",
      "Epoch 65/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2051 - val_loss: 0.2489\n",
      "Epoch 66/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2118 - val_loss: 0.2358\n",
      "Epoch 67/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2217 - val_loss: 0.2640\n",
      "Epoch 68/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2468 - val_loss: 0.2715\n",
      "Epoch 69/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2201 - val_loss: 0.2380\n",
      "Epoch 70/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2235 - val_loss: 0.2438\n",
      "\n",
      "Epoch 00070: ReduceLROnPlateau reducing learning rate to 0.0007849999819882214.\n",
      "Epoch 71/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.1972 - val_loss: 0.2205\n",
      "Epoch 72/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1896 - val_loss: 0.2454\n",
      "Epoch 73/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1980 - val_loss: 0.2181\n",
      "Epoch 74/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1855 - val_loss: 0.2169\n",
      "Epoch 75/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1832 - val_loss: 0.2159\n",
      "Epoch 76/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1843 - val_loss: 0.2288\n",
      "Epoch 77/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.1902 - val_loss: 0.2157\n",
      "Epoch 78/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.1787 - val_loss: 0.2162\n",
      "Epoch 79/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.1739 - val_loss: 0.2183\n",
      "Epoch 80/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.1737 - val_loss: 0.2111\n",
      "Epoch 81/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.1780 - val_loss: 0.2323\n",
      "Epoch 82/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.1862 - val_loss: 0.2133\n",
      "Epoch 83/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.1729 - val_loss: 0.2087\n",
      "Epoch 84/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.1832 - val_loss: 0.2277\n",
      "Epoch 85/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1762 - val_loss: 0.2120\n",
      "Epoch 86/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1686 - val_loss: 0.2099\n",
      "Epoch 87/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.1845 - val_loss: 0.2126\n",
      "Epoch 88/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1670 - val_loss: 0.2179\n",
      "Epoch 89/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.1654 - val_loss: 0.2181\n",
      "Epoch 90/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.1806 - val_loss: 0.2091\n",
      "Epoch 91/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.1637 - val_loss: 0.2134\n",
      "Epoch 92/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1655 - val_loss: 0.2143\n",
      "Epoch 93/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1705 - val_loss: 0.2551\n",
      "\n",
      "Epoch 00093: ReduceLROnPlateau reducing learning rate to 0.0003924999909941107.\n",
      "Epoch 94/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.1728 - val_loss: 0.2044\n",
      "Epoch 95/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1600 - val_loss: 0.2038\n",
      "Epoch 96/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1561 - val_loss: 0.2016\n",
      "Epoch 97/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1534 - val_loss: 0.2004\n",
      "Epoch 98/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1514 - val_loss: 0.1995\n",
      "Epoch 99/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1505 - val_loss: 0.2002\n",
      "Epoch 100/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.1497 - val_loss: 0.1987\n",
      "MAE: 0.19873812794685364\n",
      "--------------- > Fold 3 < ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-02 00:43:50.715343: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 941624320 exceeds 10% of free system memory.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "56/56 [==============================] - 41s 624ms/step - loss: 2.6119 - val_loss: 1.1516\n",
      "Epoch 2/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.8981 - val_loss: 0.7900\n",
      "Epoch 3/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.6645 - val_loss: 0.6654\n",
      "Epoch 4/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.5632 - val_loss: 0.5274\n",
      "Epoch 5/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.5027 - val_loss: 0.5151\n",
      "Epoch 6/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.4704 - val_loss: 0.4766\n",
      "Epoch 7/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.4332 - val_loss: 0.4603\n",
      "Epoch 8/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.4435 - val_loss: 0.4157\n",
      "Epoch 9/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.4040 - val_loss: 0.3977\n",
      "Epoch 10/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3892 - val_loss: 0.4031\n",
      "Epoch 11/100\n",
      "56/56 [==============================] - 33s 585ms/step - loss: 0.4048 - val_loss: 0.3864\n",
      "Epoch 12/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3738 - val_loss: 0.3632\n",
      "Epoch 13/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3558 - val_loss: 0.3576\n",
      "Epoch 14/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.3494 - val_loss: 0.3818\n",
      "Epoch 15/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3500 - val_loss: 0.3690\n",
      "Epoch 16/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3425 - val_loss: 0.4215\n",
      "Epoch 17/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3482 - val_loss: 0.3370\n",
      "Epoch 18/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3289 - val_loss: 0.3308\n",
      "Epoch 19/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.3188 - val_loss: 0.3377\n",
      "Epoch 20/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3168 - val_loss: 0.3308\n",
      "Epoch 21/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3140 - val_loss: 0.3401\n",
      "Epoch 22/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3208 - val_loss: 0.3217\n",
      "Epoch 23/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.3014 - val_loss: 0.3327\n",
      "Epoch 24/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.3115 - val_loss: 0.3312\n",
      "Epoch 25/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3013 - val_loss: 0.3275\n",
      "Epoch 26/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2939 - val_loss: 0.3112\n",
      "Epoch 27/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.3165 - val_loss: 0.3124\n",
      "Epoch 28/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2857 - val_loss: 0.2886\n",
      "Epoch 29/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2815 - val_loss: 0.3076\n",
      "Epoch 30/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2773 - val_loss: 0.2977\n",
      "Epoch 31/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2897 - val_loss: 0.3181\n",
      "Epoch 32/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2806 - val_loss: 0.2854\n",
      "Epoch 33/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2844 - val_loss: 0.2908\n",
      "Epoch 34/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2637 - val_loss: 0.2738\n",
      "Epoch 35/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2592 - val_loss: 0.2758\n",
      "Epoch 36/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2608 - val_loss: 0.2685\n",
      "Epoch 37/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2696 - val_loss: 0.2787\n",
      "Epoch 38/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2752 - val_loss: 0.2959\n",
      "Epoch 39/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2693 - val_loss: 0.2754\n",
      "Epoch 40/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2598 - val_loss: 0.2816\n",
      "Epoch 41/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2589 - val_loss: 0.3390\n",
      "Epoch 42/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2621 - val_loss: 0.2589\n",
      "Epoch 43/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2423 - val_loss: 0.2546\n",
      "Epoch 44/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2429 - val_loss: 0.2735\n",
      "Epoch 45/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2430 - val_loss: 0.2534\n",
      "Epoch 46/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2367 - val_loss: 0.2630\n",
      "Epoch 47/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2434 - val_loss: 0.2502\n",
      "Epoch 48/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2314 - val_loss: 0.2553\n",
      "Epoch 49/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2411 - val_loss: 0.2711\n",
      "Epoch 50/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2421 - val_loss: 0.2517\n",
      "Epoch 51/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2347 - val_loss: 0.2419\n",
      "Epoch 52/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2206 - val_loss: 0.2526\n",
      "Epoch 53/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2261 - val_loss: 0.2393\n",
      "Epoch 54/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2281 - val_loss: 0.2711\n",
      "Epoch 55/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2289 - val_loss: 0.2536\n",
      "Epoch 56/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2237 - val_loss: 0.2562\n",
      "Epoch 57/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2303 - val_loss: 0.2913\n",
      "Epoch 58/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2310 - val_loss: 0.2545\n",
      "Epoch 59/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2189 - val_loss: 0.2388\n",
      "Epoch 60/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2133 - val_loss: 0.2409\n",
      "Epoch 61/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2179 - val_loss: 0.2644\n",
      "Epoch 62/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2297 - val_loss: 0.2423\n",
      "Epoch 63/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2175 - val_loss: 0.2444\n",
      "Epoch 64/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2225 - val_loss: 0.2338\n",
      "Epoch 65/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2098 - val_loss: 0.3013\n",
      "Epoch 66/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2260 - val_loss: 0.2364\n",
      "Epoch 67/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2062 - val_loss: 0.2239\n",
      "Epoch 68/100\n",
      "56/56 [==============================] - 33s 585ms/step - loss: 0.2073 - val_loss: 0.2405\n",
      "Epoch 69/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2439 - val_loss: 0.2443\n",
      "Epoch 70/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2258 - val_loss: 0.2535\n",
      "Epoch 71/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2270 - val_loss: 0.2494\n",
      "Epoch 72/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2152 - val_loss: 0.2443\n",
      "Epoch 73/100\n",
      "56/56 [==============================] - 33s 585ms/step - loss: 0.2119 - val_loss: 0.2292\n",
      "Epoch 74/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2076 - val_loss: 0.2321\n",
      "Epoch 75/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1976 - val_loss: 0.2488\n",
      "Epoch 76/100\n",
      "56/56 [==============================] - 33s 585ms/step - loss: 0.2063 - val_loss: 0.2391\n",
      "Epoch 77/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1975 - val_loss: 0.2324\n",
      "\n",
      "Epoch 00077: ReduceLROnPlateau reducing learning rate to 0.0007849999819882214.\n",
      "Epoch 78/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1827 - val_loss: 0.2240\n",
      "Epoch 79/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1825 - val_loss: 0.2148\n",
      "Epoch 80/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1846 - val_loss: 0.2122\n",
      "Epoch 81/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1745 - val_loss: 0.2085\n",
      "Epoch 82/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1691 - val_loss: 0.2052\n",
      "Epoch 83/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1639 - val_loss: 0.2050\n",
      "Epoch 84/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1632 - val_loss: 0.2043\n",
      "Epoch 85/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1619 - val_loss: 0.2048\n",
      "Epoch 86/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1612 - val_loss: 0.2026\n",
      "Epoch 87/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1583 - val_loss: 0.2029\n",
      "Epoch 88/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1588 - val_loss: 0.2072\n",
      "Epoch 89/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1591 - val_loss: 0.2130\n",
      "Epoch 90/100\n",
      "56/56 [==============================] - 33s 585ms/step - loss: 0.1706 - val_loss: 0.2401\n",
      "Epoch 91/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.1677 - val_loss: 0.2099\n",
      "Epoch 92/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1642 - val_loss: 0.2090\n",
      "Epoch 93/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1623 - val_loss: 0.2066\n",
      "Epoch 94/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1684 - val_loss: 0.2363\n",
      "Epoch 95/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1750 - val_loss: 0.2057\n",
      "Epoch 96/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1579 - val_loss: 0.1999\n",
      "Epoch 97/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1529 - val_loss: 0.1984\n",
      "Epoch 98/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1546 - val_loss: 0.2105\n",
      "Epoch 99/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1710 - val_loss: 0.2139\n",
      "Epoch 100/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1837 - val_loss: 0.2161\n",
      "MAE: 0.21613845229148865\n",
      "--------------- > Fold 4 < ---------------\n",
      "Epoch 1/100\n",
      "56/56 [==============================] - 42s 633ms/step - loss: 2.4025 - val_loss: 1.0734\n",
      "Epoch 2/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.8497 - val_loss: 0.8173\n",
      "Epoch 3/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.6420 - val_loss: 0.5650\n",
      "Epoch 4/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.5515 - val_loss: 0.5202\n",
      "Epoch 5/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.4926 - val_loss: 0.5479\n",
      "Epoch 6/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.4706 - val_loss: 0.4544\n",
      "Epoch 7/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.4664 - val_loss: 0.4365\n",
      "Epoch 8/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.4251 - val_loss: 0.4695\n",
      "Epoch 9/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3989 - val_loss: 0.4058\n",
      "Epoch 10/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3827 - val_loss: 0.4024\n",
      "Epoch 11/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3960 - val_loss: 0.3832\n",
      "Epoch 12/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.3626 - val_loss: 0.3706\n",
      "Epoch 13/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3671 - val_loss: 0.3670\n",
      "Epoch 14/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3592 - val_loss: 0.3870\n",
      "Epoch 15/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3463 - val_loss: 0.3441\n",
      "Epoch 16/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.3378 - val_loss: 0.3492\n",
      "Epoch 17/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.3285 - val_loss: 0.3480\n",
      "Epoch 18/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3287 - val_loss: 0.3262\n",
      "Epoch 19/100\n",
      "56/56 [==============================] - 33s 585ms/step - loss: 0.3088 - val_loss: 0.3172\n",
      "Epoch 20/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3074 - val_loss: 0.3269\n",
      "Epoch 21/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.3011 - val_loss: 0.3133\n",
      "Epoch 22/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3191 - val_loss: 0.3147\n",
      "Epoch 23/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.3230 - val_loss: 0.3190\n",
      "Epoch 24/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3043 - val_loss: 0.3239\n",
      "Epoch 25/100\n",
      "56/56 [==============================] - 33s 585ms/step - loss: 0.3037 - val_loss: 0.3162\n",
      "Epoch 26/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2842 - val_loss: 0.3268\n",
      "Epoch 27/100\n",
      "56/56 [==============================] - 33s 586ms/step - loss: 0.2969 - val_loss: 0.3084\n",
      "Epoch 28/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.3031 - val_loss: 0.3105\n",
      "Epoch 29/100\n",
      "56/56 [==============================] - 33s 585ms/step - loss: 0.2777 - val_loss: 0.2936\n",
      "Epoch 30/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2738 - val_loss: 0.2977\n",
      "Epoch 31/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2682 - val_loss: 0.2770\n",
      "Epoch 32/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2594 - val_loss: 0.2985\n",
      "Epoch 33/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2646 - val_loss: 0.2849\n",
      "Epoch 34/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2741 - val_loss: 0.3057\n",
      "Epoch 35/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2610 - val_loss: 0.2685\n",
      "Epoch 36/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2668 - val_loss: 0.2776\n",
      "Epoch 37/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2574 - val_loss: 0.2862\n",
      "Epoch 38/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2545 - val_loss: 0.2950\n",
      "Epoch 39/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2725 - val_loss: 0.3025\n",
      "Epoch 40/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2612 - val_loss: 0.2728\n",
      "Epoch 41/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2530 - val_loss: 0.2602\n",
      "Epoch 42/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2427 - val_loss: 0.2547\n",
      "Epoch 43/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2368 - val_loss: 0.2573\n",
      "Epoch 44/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2409 - val_loss: 0.2678\n",
      "Epoch 45/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2348 - val_loss: 0.2677\n",
      "Epoch 46/100\n",
      "56/56 [==============================] - 33s 585ms/step - loss: 0.2373 - val_loss: 0.2770\n",
      "Epoch 47/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2605 - val_loss: 0.2946\n",
      "Epoch 48/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2592 - val_loss: 0.2601\n",
      "Epoch 49/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2255 - val_loss: 0.2363\n",
      "Epoch 50/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2203 - val_loss: 0.2473\n",
      "Epoch 51/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2282 - val_loss: 0.2581\n",
      "Epoch 52/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2291 - val_loss: 0.2534\n",
      "Epoch 53/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2267 - val_loss: 0.2454\n",
      "Epoch 54/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2198 - val_loss: 0.2502\n",
      "Epoch 55/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2263 - val_loss: 0.2599\n",
      "Epoch 56/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2236 - val_loss: 0.2349\n",
      "Epoch 57/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2138 - val_loss: 0.2275\n",
      "Epoch 58/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2096 - val_loss: 0.2329\n",
      "Epoch 59/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2292 - val_loss: 0.2712\n",
      "Epoch 60/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2217 - val_loss: 0.2340\n",
      "Epoch 61/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2121 - val_loss: 0.2370\n",
      "Epoch 62/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2188 - val_loss: 0.2322\n",
      "Epoch 63/100\n",
      "56/56 [==============================] - 33s 585ms/step - loss: 0.2122 - val_loss: 0.2281\n",
      "Epoch 64/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.2112 - val_loss: 0.2693\n",
      "Epoch 65/100\n",
      "56/56 [==============================] - 33s 585ms/step - loss: 0.2219 - val_loss: 0.2610\n",
      "Epoch 66/100\n",
      "56/56 [==============================] - 33s 585ms/step - loss: 0.2332 - val_loss: 0.2599\n",
      "Epoch 67/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2214 - val_loss: 0.2255\n",
      "Epoch 68/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2058 - val_loss: 0.2247\n",
      "Epoch 69/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1993 - val_loss: 0.2423\n",
      "Epoch 70/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2123 - val_loss: 0.2332\n",
      "Epoch 71/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2081 - val_loss: 0.2334\n",
      "Epoch 72/100\n",
      "56/56 [==============================] - 33s 585ms/step - loss: 0.1955 - val_loss: 0.2245\n",
      "Epoch 73/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1912 - val_loss: 0.2170\n",
      "Epoch 74/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1967 - val_loss: 0.2239\n",
      "Epoch 75/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1918 - val_loss: 0.2130\n",
      "Epoch 76/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1856 - val_loss: 0.2384\n",
      "Epoch 77/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.1987 - val_loss: 0.2530\n",
      "Epoch 78/100\n",
      "56/56 [==============================] - 33s 585ms/step - loss: 0.2071 - val_loss: 0.2445\n",
      "Epoch 79/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1976 - val_loss: 0.2312\n",
      "Epoch 80/100\n",
      "56/56 [==============================] - 33s 585ms/step - loss: 0.1952 - val_loss: 0.2247\n",
      "Epoch 81/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1985 - val_loss: 0.2230\n",
      "Epoch 82/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2218 - val_loss: 0.2470\n",
      "Epoch 83/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.2241 - val_loss: 0.2878\n",
      "Epoch 84/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.2349 - val_loss: 0.2510\n",
      "Epoch 85/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1981 - val_loss: 0.2248\n",
      "\n",
      "Epoch 00085: ReduceLROnPlateau reducing learning rate to 0.0007849999819882214.\n",
      "Epoch 86/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1804 - val_loss: 0.2058\n",
      "Epoch 87/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1716 - val_loss: 0.2041\n",
      "Epoch 88/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1672 - val_loss: 0.2060\n",
      "Epoch 89/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1646 - val_loss: 0.2013\n",
      "Epoch 90/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1615 - val_loss: 0.2029\n",
      "Epoch 91/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1622 - val_loss: 0.2039\n",
      "Epoch 92/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.1595 - val_loss: 0.2033\n",
      "Epoch 93/100\n",
      "56/56 [==============================] - 33s 585ms/step - loss: 0.1571 - val_loss: 0.2014\n",
      "Epoch 94/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.1550 - val_loss: 0.2006\n",
      "Epoch 95/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1538 - val_loss: 0.1982\n",
      "Epoch 96/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1555 - val_loss: 0.2100\n",
      "Epoch 97/100\n",
      "56/56 [==============================] - 33s 582ms/step - loss: 0.1611 - val_loss: 0.2210\n",
      "Epoch 98/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1581 - val_loss: 0.2085\n",
      "Epoch 99/100\n",
      "56/56 [==============================] - 33s 583ms/step - loss: 0.1618 - val_loss: 0.2197\n",
      "Epoch 100/100\n",
      "56/56 [==============================] - 33s 584ms/step - loss: 0.1725 - val_loss: 0.2346\n",
      "MAE: 0.2346438467502594\n",
      "Mean score: 0.2120, std: 0.0149.\n"
     ]
    }
   ],
   "source": [
    "EPOCH = 100\n",
    "BATCH_SIZE = 1024\n",
    "TPU = False\n",
    "TOTAL_SPLITS = 4\n",
    "LEARNING_RATE = 0.00157\n",
    "\n",
    "if DEBUG:    \n",
    "    EPOCH = 20\n",
    "    TOTAL_SPLITS = 2    \n",
    "\n",
    "lr = ReduceLROnPlateau(monitor=\"val_loss\", factor=0.5, patience=10, verbose=1)\n",
    "es = EarlyStopping(monitor=\"val_loss\", patience=60, verbose=1, mode=\"min\", restore_best_weights=True)\n",
    "cyclical_learning_rate = CyclicalLearningRate( initial_learning_rate=3e-7, maximal_learning_rate=3e-5, step_size=2360, \n",
    "                                              scale_fn=lambda x: 1 / (2.0 ** (x - 1)), scale_mode='cycle')\n",
    "\n",
    "if TPU:\n",
    "    # detect and init the TPU\n",
    "    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect()\n",
    "    ## instantiate a distribution strategy\n",
    "    xpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)\n",
    "else:\n",
    "    # GET GPU STRATEGY\n",
    "    xpu_strategy = tf.distribute.get_strategy()\n",
    "\n",
    "models = []\n",
    "scores = []\n",
    "with xpu_strategy.scope():\n",
    "\n",
    "    folds = KFold(n_splits=TOTAL_SPLITS, shuffle=True, random_state=RANDOM_SEED)\n",
    "    for fold_n, (train_index, valid_index) in enumerate(folds.split(train, targets)):\n",
    "        print('-'*15, '>', f'Fold {fold_n+1}', '<', '-'*15)\n",
    "        X_train, X_valid = train[train_index], train[valid_index]\n",
    "        y_train, y_valid = targets[train_index], targets[valid_index]    \n",
    "\n",
    "        model = build_model(X_train, optimizer=Adam(learning_rate=LEARNING_RATE))\n",
    "        #tf.train.Checkpoint( model = model, optimizer = Adam(learning_rate=cyclical_learning_rate))\n",
    "        history = model.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=EPOCH, batch_size=BATCH_SIZE, \n",
    "                            callbacks=[lr, es], \n",
    "                            shuffle=False, workers=8, use_multiprocessing=True)   \n",
    "        #display_feature_importance( model, X_valid, ALL_FEATURES)\n",
    "        #break # only one fold\n",
    "        \n",
    "        oof_preds = model.predict(X_valid, verbose=0).squeeze() \n",
    "        mae = mean_absolute_error(y_valid, oof_preds)   \n",
    "        scores.append(mae)\n",
    "        models.append(model)\n",
    "        print(f'MAE: {mae}')\n",
    "\n",
    "print('Mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "026c23b2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-02T02:34:55.979284Z",
     "iopub.status.busy": "2021-11-02T02:34:55.978658Z",
     "iopub.status.idle": "2021-11-02T02:37:45.290185Z",
     "shell.execute_reply": "2021-11-02T02:37:45.289632Z",
     "shell.execute_reply.started": "2021-10-28T17:18:15.831197Z"
    },
    "papermill": {
     "duration": 175.374834,
     "end_time": "2021-11-02T02:37:45.290326",
     "exception": false,
     "start_time": "2021-11-02T02:34:49.915492",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "predicted = []\n",
    "for model in models:\n",
    "    yhat = model.predict(test, verbose=0).squeeze()\n",
    "    predicted.append(yhat)\n",
    "    \n",
    "mean_pred = np.median(predicted, axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f5daf54",
   "metadata": {
    "papermill": {
     "duration": 6.173887,
     "end_time": "2021-11-02T02:37:58.055606",
     "exception": false,
     "start_time": "2021-11-02T02:37:51.881719",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4c64616c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-02T02:38:10.298264Z",
     "iopub.status.busy": "2021-11-02T02:38:10.297173Z",
     "iopub.status.idle": "2021-11-02T02:38:26.601968Z",
     "shell.execute_reply": "2021-11-02T02:38:26.601417Z",
     "shell.execute_reply.started": "2021-10-28T17:18:15.83278Z"
    },
    "papermill": {
     "duration": 22.26716,
     "end_time": "2021-11-02T02:38:26.602106",
     "exception": false,
     "start_time": "2021-11-02T02:38:04.334946",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>pressure</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>6.283592</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>6.288374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>7.507535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>8.051300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>9.465290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>10.674587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>11.902813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>13.135768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>14.349846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>15.443434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11</td>\n",
       "      <td>16.573406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12</td>\n",
       "      <td>16.968323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>13</td>\n",
       "      <td>18.064379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14</td>\n",
       "      <td>18.490173</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>15</td>\n",
       "      <td>19.121590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>16</td>\n",
       "      <td>19.924500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>17</td>\n",
       "      <td>20.624039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>18</td>\n",
       "      <td>21.284876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>19</td>\n",
       "      <td>21.866997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>20</td>\n",
       "      <td>22.426262</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    id   pressure\n",
       "0    1   6.283592\n",
       "1    2   6.288374\n",
       "2    3   7.507535\n",
       "3    4   8.051300\n",
       "4    5   9.465290\n",
       "5    6  10.674587\n",
       "6    7  11.902813\n",
       "7    8  13.135768\n",
       "8    9  14.349846\n",
       "9   10  15.443434\n",
       "10  11  16.573406\n",
       "11  12  16.968323\n",
       "12  13  18.064379\n",
       "13  14  18.490173\n",
       "14  15  19.121590\n",
       "15  16  19.924500\n",
       "16  17  20.624039\n",
       "17  18  21.284876\n",
       "18  19  21.866997\n",
       "19  20  22.426262"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission['pressure'] = mean_pred.ravel()\n",
    "submission.to_csv('submission.csv', index=False, float_format='%.6f')\n",
    "submission.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "586f8fd8",
   "metadata": {
    "papermill": {
     "duration": 6.151479,
     "end_time": "2021-11-02T02:38:38.784593",
     "exception": false,
     "start_time": "2021-11-02T02:38:32.633114",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 13684.996781,
   "end_time": "2021-11-02T02:38:48.683659",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-11-01T22:50:43.686878",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}