notebooks/model_training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import torch\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "from torch import nn, optim\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder, StandardScaler\n",
    "from sklearn.metrics import mean_squared_error\n",
    "import pickle\n",
    "import xgboost as xgb\n",
    "from pathlib import Path"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('../data/prepared_data_v3.csv')\n",
    "data.drop(columns=['name_track', 'name_artist', 'release_date', 'genres'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "le_track = LabelEncoder()\n",
    "le_artist = LabelEncoder()\n",
    "\n",
    "data['id_track'] = le_track.fit_transform(data['id_track'])\n",
    "data['id_artist'] = le_artist.fit_transform(data['id_artist'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../artifacts/preprocessing/le_track.pkl', 'wb') as f:\n",
    "    pickle.dump(le_track, f)\n",
    "with open('../artifacts/preprocessing/le_artist.pkl', 'wb') as f:\n",
    "    pickle.dump(le_artist, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "no_weeks = data['week_number'].max()\n",
    "for week in range(no_weeks):\n",
    "    data.loc[data['week_number'] == week, 'next_week_plays'] = data.loc[data['week_number'] == week+1, 'track_plays'].values\n",
    "\n",
    "data['next_week_plays'] = data['next_week_plays'].fillna(0).astype(int)\n",
    "data = data.drop(data[data['week_number'] == no_weeks].index)\n",
    "data.drop(columns=['week_number'], inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_track</th>\n",
       "      <th>popularity</th>\n",
       "      <th>duration_ms</th>\n",
       "      <th>explicit</th>\n",
       "      <th>id_artist</th>\n",
       "      <th>danceability</th>\n",
       "      <th>energy</th>\n",
       "      <th>key</th>\n",
       "      <th>loudness</th>\n",
       "      <th>speechiness</th>\n",
       "      <th>acousticness</th>\n",
       "      <th>instrumentalness</th>\n",
       "      <th>liveness</th>\n",
       "      <th>valence</th>\n",
       "      <th>tempo</th>\n",
       "      <th>artist_popularity</th>\n",
       "      <th>track_plays</th>\n",
       "      <th>artist_plays</th>\n",
       "      <th>next_week_plays</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1262</td>\n",
       "      <td>55</td>\n",
       "      <td>201467</td>\n",
       "      <td>0</td>\n",
       "      <td>200</td>\n",
       "      <td>0.673</td>\n",
       "      <td>0.3770</td>\n",
       "      <td>0</td>\n",
       "      <td>-14.141</td>\n",
       "      <td>0.0697</td>\n",
       "      <td>0.586</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.332</td>\n",
       "      <td>0.713</td>\n",
       "      <td>88.973</td>\n",
       "      <td>61.090909</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3316</td>\n",
       "      <td>58</td>\n",
       "      <td>179867</td>\n",
       "      <td>0</td>\n",
       "      <td>200</td>\n",
       "      <td>0.448</td>\n",
       "      <td>0.1200</td>\n",
       "      <td>0</td>\n",
       "      <td>-14.089</td>\n",
       "      <td>0.0355</td>\n",
       "      <td>0.877</td>\n",
       "      <td>0.013500</td>\n",
       "      <td>0.100</td>\n",
       "      <td>0.261</td>\n",
       "      <td>86.407</td>\n",
       "      <td>61.090909</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>17586</td>\n",
       "      <td>53</td>\n",
       "      <td>147000</td>\n",
       "      <td>0</td>\n",
       "      <td>200</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.4050</td>\n",
       "      <td>0</td>\n",
       "      <td>-9.935</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.842</td>\n",
       "      <td>0.001140</td>\n",
       "      <td>0.198</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>61.090909</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5396</td>\n",
       "      <td>74</td>\n",
       "      <td>137520</td>\n",
       "      <td>0</td>\n",
       "      <td>200</td>\n",
       "      <td>0.399</td>\n",
       "      <td>0.2580</td>\n",
       "      <td>5</td>\n",
       "      <td>-16.028</td>\n",
       "      <td>0.0330</td>\n",
       "      <td>0.792</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.128</td>\n",
       "      <td>0.192</td>\n",
       "      <td>108.174</td>\n",
       "      <td>61.090909</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4393</td>\n",
       "      <td>66</td>\n",
       "      <td>204400</td>\n",
       "      <td>0</td>\n",
       "      <td>200</td>\n",
       "      <td>0.507</td>\n",
       "      <td>0.0779</td>\n",
       "      <td>0</td>\n",
       "      <td>-12.099</td>\n",
       "      <td>0.0544</td>\n",
       "      <td>0.866</td>\n",
       "      <td>0.002750</td>\n",
       "      <td>0.108</td>\n",
       "      <td>0.326</td>\n",
       "      <td>70.808</td>\n",
       "      <td>61.090909</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id_track  popularity  duration_ms  explicit  id_artist  danceability  \\\n",
       "0      1262          55       201467         0        200         0.673   \n",
       "1      3316          58       179867         0        200         0.448   \n",
       "2     17586          53       147000         0        200         0.000   \n",
       "3      5396          74       137520         0        200         0.399   \n",
       "4      4393          66       204400         0        200         0.507   \n",
       "\n",
       "   energy  key  loudness  speechiness  acousticness  instrumentalness  \\\n",
       "0  0.3770    0   -14.141       0.0697         0.586          0.000000   \n",
       "1  0.1200    0   -14.089       0.0355         0.877          0.013500   \n",
       "2  0.4050    0    -9.935       0.0000         0.842          0.001140   \n",
       "3  0.2580    5   -16.028       0.0330         0.792          0.000002   \n",
       "4  0.0779    0   -12.099       0.0544         0.866          0.002750   \n",
       "\n",
       "   liveness  valence    tempo  artist_popularity  track_plays  artist_plays  \\\n",
       "0     0.332    0.713   88.973          61.090909            0             0   \n",
       "1     0.100    0.261   86.407          61.090909            0             0   \n",
       "2     0.198    0.000    0.000          61.090909            0             0   \n",
       "3     0.128    0.192  108.174          61.090909            0             0   \n",
       "4     0.108    0.326   70.808          61.090909            0             0   \n",
       "\n",
       "   next_week_plays  \n",
       "0                0  \n",
       "1                0  \n",
       "2                0  \n",
       "3                0  \n",
       "4                0  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_to_normalize = ['popularity', 'duration_ms', 'danceability', 'energy', 'key', 'loudness', 'speechiness', \n",
    "                         'acousticness', 'instrumentalness', 'liveness', 'valence', 'tempo', 'artist_popularity', \n",
    "                         'track_plays', 'artist_plays']\n",
    "\n",
    "scaler = StandardScaler()\n",
    "\n",
    "data[features_to_normalize] = scaler.fit_transform(data[features_to_normalize])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../artifacts/preprocessing/scaler.pkl', 'wb') as f:\n",
    "    pickle.dump(scaler, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = data['next_week_plays'].values\n",
    "X = data.drop('next_week_plays', axis=1).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = TensorDataset(torch.from_numpy(X_train).float(), torch.from_numpy(y_train).float())\n",
    "val_dataset = TensorDataset(torch.from_numpy(X_val).float(), torch.from_numpy(y_val).float())\n",
    "test_dataset = TensorDataset(torch.from_numpy(X_test).float(), torch.from_numpy(y_test).float())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)\n",
    "val_loader = DataLoader(val_dataset, batch_size=32, shuffle=True)\n",
    "test_loader = DataLoader(test_dataset, batch_size=32, shuffle=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Modeling"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Linear Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LinearRegressionPredictor(nn.Module):\n",
    "    def __init__(self, num_features: int) -> None:\n",
    "        super(LinearRegressionPredictor, self).__init__()\n",
    "        self.layer = nn.Linear(num_features, 1)\n",
    "\n",
    "        nn.init.uniform_(self.layer.weight, -0.1, 0.1)\n",
    "        nn.init.constant_(self.layer.bias, 0.0)\n",
    "\n",
    "    def forward(self, x: torch.Tensor) -> torch.Tensor:\n",
    "        return self.layer(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LinearRegressionPredictor(X_train.shape[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.MSELoss()\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.0001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 loss: nan\n",
      "Epoch 1 loss: nan\n",
      "Epoch 2 loss: nan\n",
      "Epoch 3 loss: nan\n",
      "Epoch 4 loss: nan\n",
      "Epoch 5 loss: nan\n",
      "Epoch 6 loss: nan\n",
      "Epoch 7 loss: nan\n",
      "Epoch 8 loss: nan\n",
      "Epoch 9 loss: nan\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(10):\n",
    "    epoch_loss = 0.0\n",
    "    for i, (inputs, labels) in enumerate(train_loader):\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        outputs = model(inputs)\n",
    "        \n",
    "        loss = criterion(outputs, labels.unsqueeze(1))\n",
    "        epoch_loss += loss.item()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "    print(f'Epoch {epoch} loss: {epoch_loss / len(train_loader)}')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NNPredictor(nn.Module):\n",
    "    def __init__(self, num_features, num_hidden) -> None:\n",
    "        super(NNPredictor, self).__init__()\n",
    "        self.input_layer = nn.Linear(num_features, num_hidden)\n",
    "        self.hidden_layer_1 = nn.Linear(num_hidden, num_hidden)\n",
    "        self.hidden_layer_2 = nn.Linear(num_hidden, num_hidden)\n",
    "        self.output_layer = nn.Linear(num_hidden, 1)\n",
    "\n",
    "        self.activation = nn.ReLU()\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.activation(self.input_layer(x))\n",
    "        x = self.activation(self.hidden_layer_1(x))\n",
    "        x = self.activation(self.hidden_layer_2(x))\n",
    "        return self.output_layer(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn_model = NNPredictor(X_train.shape[1], 128).float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.MSELoss()\n",
    "optimizer = optim.Adam(nn_model.parameters(), lr=0.0001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 loss: 3.9870061107341686\n",
      "Epoch 1 loss: 1.5895029223942008\n",
      "Epoch 2 loss: 1.0789377526425072\n",
      "Epoch 3 loss: 1.2232236589304983\n",
      "Epoch 4 loss: 0.9372224525354215\n",
      "Epoch 5 loss: 0.834023672474661\n",
      "Epoch 6 loss: 0.7608101785094771\n",
      "Epoch 7 loss: 0.7696647046671796\n",
      "Epoch 8 loss: 0.6910719545250038\n",
      "Epoch 9 loss: 0.6803542912090125\n",
      "Epoch 10 loss: 0.6343899124637077\n",
      "Epoch 11 loss: 0.658352235959992\n",
      "Epoch 12 loss: 0.6300027976520965\n",
      "Epoch 13 loss: 0.6160819699892356\n",
      "Epoch 14 loss: 0.6379005396627971\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(15):\n",
    "    epoch_loss = 0.0\n",
    "    for i, (inputs, labels) in enumerate(train_loader):\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        outputs = nn_model(inputs)\n",
    "        \n",
    "        loss = criterion(outputs, labels.unsqueeze(1))\n",
    "        epoch_loss += loss.item()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "    print(f'Epoch {epoch} loss: {epoch_loss / len(train_loader)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation RMSE:  2.378784665023856\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "nn_model.eval()\n",
    "y_pred = np.array([])\n",
    "with torch.no_grad():\n",
    "    for i, (inputs, labels) in enumerate(val_loader):\n",
    "        outputs = nn_model(inputs)\n",
    "        y_pred = np.append(y_pred, outputs.squeeze(1).numpy())\n",
    "\n",
    "\n",
    "print('Validation RMSE: ', mean_squared_error(y_val, y_pred, squared=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(nn_model.state_dict(), '../artifacts/models/nn_regressor.pth')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_model = xgb.XGBRegressor(objective='reg:squarederror', n_estimators=1000, max_depth=8, learning_rate=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/kklassa/Code/Uni/IUM/venv/lib/python3.11/site-packages/xgboost/sklearn.py:835: UserWarning: `early_stopping_rounds` in `fit` method is deprecated for better compatibility with scikit-learn, use `early_stopping_rounds` in constructor or`set_params` instead.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:1.80652\n",
      "[1]\tvalidation_0-rmse:1.79051\n",
      "[2]\tvalidation_0-rmse:1.77469\n",
      "[3]\tvalidation_0-rmse:1.75903\n",
      "[4]\tvalidation_0-rmse:1.74355\n",
      "[5]\tvalidation_0-rmse:1.72825\n",
      "[6]\tvalidation_0-rmse:1.71311\n",
      "[7]\tvalidation_0-rmse:1.69815\n",
      "[8]\tvalidation_0-rmse:1.68335\n",
      "[9]\tvalidation_0-rmse:1.66872\n",
      "[10]\tvalidation_0-rmse:1.65425\n",
      "[11]\tvalidation_0-rmse:1.63994\n",
      "[12]\tvalidation_0-rmse:1.62580\n",
      "[13]\tvalidation_0-rmse:1.61181\n",
      "[14]\tvalidation_0-rmse:1.59799\n",
      "[15]\tvalidation_0-rmse:1.58431\n",
      "[16]\tvalidation_0-rmse:1.57080\n",
      "[17]\tvalidation_0-rmse:1.55744\n",
      "[18]\tvalidation_0-rmse:1.54423\n",
      "[19]\tvalidation_0-rmse:1.53117\n",
      "[20]\tvalidation_0-rmse:1.51826\n",
      "[21]\tvalidation_0-rmse:1.50550\n",
      "[22]\tvalidation_0-rmse:1.49289\n",
      "[23]\tvalidation_0-rmse:1.48042\n",
      "[24]\tvalidation_0-rmse:1.46810\n",
      "[25]\tvalidation_0-rmse:1.45591\n",
      "[26]\tvalidation_0-rmse:1.44388\n",
      "[27]\tvalidation_0-rmse:1.43198\n",
      "[28]\tvalidation_0-rmse:1.42022\n",
      "[29]\tvalidation_0-rmse:1.40859\n",
      "[30]\tvalidation_0-rmse:1.39710\n",
      "[31]\tvalidation_0-rmse:1.38575\n",
      "[32]\tvalidation_0-rmse:1.37453\n",
      "[33]\tvalidation_0-rmse:1.36344\n",
      "[34]\tvalidation_0-rmse:1.35249\n",
      "[35]\tvalidation_0-rmse:1.34166\n",
      "[36]\tvalidation_0-rmse:1.33097\n",
      "[37]\tvalidation_0-rmse:1.32039\n",
      "[38]\tvalidation_0-rmse:1.30995\n",
      "[39]\tvalidation_0-rmse:1.29963\n",
      "[40]\tvalidation_0-rmse:1.28944\n",
      "[41]\tvalidation_0-rmse:1.27937\n",
      "[42]\tvalidation_0-rmse:1.26943\n",
      "[43]\tvalidation_0-rmse:1.25962\n",
      "[44]\tvalidation_0-rmse:1.24993\n",
      "[45]\tvalidation_0-rmse:1.24035\n",
      "[46]\tvalidation_0-rmse:1.23089\n",
      "[47]\tvalidation_0-rmse:1.22154\n",
      "[48]\tvalidation_0-rmse:1.21231\n",
      "[49]\tvalidation_0-rmse:1.20320\n",
      "[50]\tvalidation_0-rmse:1.19419\n",
      "[51]\tvalidation_0-rmse:1.18529\n",
      "[52]\tvalidation_0-rmse:1.17651\n",
      "[53]\tvalidation_0-rmse:1.16784\n",
      "[54]\tvalidation_0-rmse:1.15931\n",
      "[55]\tvalidation_0-rmse:1.15085\n",
      "[56]\tvalidation_0-rmse:1.14252\n",
      "[57]\tvalidation_0-rmse:1.13430\n",
      "[58]\tvalidation_0-rmse:1.12618\n",
      "[59]\tvalidation_0-rmse:1.11816\n",
      "[60]\tvalidation_0-rmse:1.11025\n",
      "[61]\tvalidation_0-rmse:1.10243\n",
      "[62]\tvalidation_0-rmse:1.09471\n",
      "[63]\tvalidation_0-rmse:1.08710\n",
      "[64]\tvalidation_0-rmse:1.07959\n",
      "[65]\tvalidation_0-rmse:1.07221\n",
      "[66]\tvalidation_0-rmse:1.06489\n",
      "[67]\tvalidation_0-rmse:1.05766\n",
      "[68]\tvalidation_0-rmse:1.05052\n",
      "[69]\tvalidation_0-rmse:1.04349\n",
      "[70]\tvalidation_0-rmse:1.03655\n",
      "[71]\tvalidation_0-rmse:1.02971\n",
      "[72]\tvalidation_0-rmse:1.02295\n",
      "[73]\tvalidation_0-rmse:1.01632\n",
      "[74]\tvalidation_0-rmse:1.00976\n",
      "[75]\tvalidation_0-rmse:1.00329\n",
      "[76]\tvalidation_0-rmse:0.99687\n",
      "[77]\tvalidation_0-rmse:0.99058\n",
      "[78]\tvalidation_0-rmse:0.98437\n",
      "[79]\tvalidation_0-rmse:0.97823\n",
      "[80]\tvalidation_0-rmse:0.97215\n",
      "[81]\tvalidation_0-rmse:0.96619\n",
      "[82]\tvalidation_0-rmse:0.96031\n",
      "[83]\tvalidation_0-rmse:0.95450\n",
      "[84]\tvalidation_0-rmse:0.94877\n",
      "[85]\tvalidation_0-rmse:0.94313\n",
      "[86]\tvalidation_0-rmse:0.93756\n",
      "[87]\tvalidation_0-rmse:0.93205\n",
      "[88]\tvalidation_0-rmse:0.92666\n",
      "[89]\tvalidation_0-rmse:0.92131\n",
      "[90]\tvalidation_0-rmse:0.91608\n",
      "[91]\tvalidation_0-rmse:0.91093\n",
      "[92]\tvalidation_0-rmse:0.90583\n",
      "[93]\tvalidation_0-rmse:0.90079\n",
      "[94]\tvalidation_0-rmse:0.89583\n",
      "[95]\tvalidation_0-rmse:0.89095\n",
      "[96]\tvalidation_0-rmse:0.88618\n",
      "[97]\tvalidation_0-rmse:0.88149\n",
      "[98]\tvalidation_0-rmse:0.87681\n",
      "[99]\tvalidation_0-rmse:0.87221\n",
      "[100]\tvalidation_0-rmse:0.86766\n",
      "[101]\tvalidation_0-rmse:0.86320\n",
      "[102]\tvalidation_0-rmse:0.85878\n",
      "[103]\tvalidation_0-rmse:0.85444\n",
      "[104]\tvalidation_0-rmse:0.85015\n",
      "[105]\tvalidation_0-rmse:0.84593\n",
      "[106]\tvalidation_0-rmse:0.84176\n",
      "[107]\tvalidation_0-rmse:0.83765\n",
      "[108]\tvalidation_0-rmse:0.83361\n",
      "[109]\tvalidation_0-rmse:0.82962\n",
      "[110]\tvalidation_0-rmse:0.82570\n",
      "[111]\tvalidation_0-rmse:0.82181\n",
      "[112]\tvalidation_0-rmse:0.81801\n",
      "[113]\tvalidation_0-rmse:0.81427\n",
      "[114]\tvalidation_0-rmse:0.81056\n",
      "[115]\tvalidation_0-rmse:0.80694\n",
      "[116]\tvalidation_0-rmse:0.80337\n",
      "[117]\tvalidation_0-rmse:0.79983\n",
      "[118]\tvalidation_0-rmse:0.79636\n",
      "[119]\tvalidation_0-rmse:0.79299\n",
      "[120]\tvalidation_0-rmse:0.78963\n",
      "[121]\tvalidation_0-rmse:0.78636\n",
      "[122]\tvalidation_0-rmse:0.78310\n",
      "[123]\tvalidation_0-rmse:0.77994\n",
      "[124]\tvalidation_0-rmse:0.77678\n",
      "[125]\tvalidation_0-rmse:0.77365\n",
      "[126]\tvalidation_0-rmse:0.77058\n",
      "[127]\tvalidation_0-rmse:0.76761\n",
      "[128]\tvalidation_0-rmse:0.76463\n",
      "[129]\tvalidation_0-rmse:0.76170\n",
      "[130]\tvalidation_0-rmse:0.75887\n",
      "[131]\tvalidation_0-rmse:0.75603\n",
      "[132]\tvalidation_0-rmse:0.75325\n",
      "[133]\tvalidation_0-rmse:0.75050\n",
      "[134]\tvalidation_0-rmse:0.74783\n",
      "[135]\tvalidation_0-rmse:0.74513\n",
      "[136]\tvalidation_0-rmse:0.74253\n",
      "[137]\tvalidation_0-rmse:0.73998\n",
      "[138]\tvalidation_0-rmse:0.73744\n",
      "[139]\tvalidation_0-rmse:0.73494\n",
      "[140]\tvalidation_0-rmse:0.73250\n",
      "[141]\tvalidation_0-rmse:0.73017\n",
      "[142]\tvalidation_0-rmse:0.72778\n",
      "[143]\tvalidation_0-rmse:0.72544\n",
      "[144]\tvalidation_0-rmse:0.72313\n",
      "[145]\tvalidation_0-rmse:0.72086\n",
      "[146]\tvalidation_0-rmse:0.71871\n",
      "[147]\tvalidation_0-rmse:0.71650\n",
      "[148]\tvalidation_0-rmse:0.71436\n",
      "[149]\tvalidation_0-rmse:0.71223\n",
      "[150]\tvalidation_0-rmse:0.71016\n",
      "[151]\tvalidation_0-rmse:0.70812\n",
      "[152]\tvalidation_0-rmse:0.70610\n",
      "[153]\tvalidation_0-rmse:0.70411\n",
      "[154]\tvalidation_0-rmse:0.70223\n",
      "[155]\tvalidation_0-rmse:0.70030\n",
      "[156]\tvalidation_0-rmse:0.69842\n",
      "[157]\tvalidation_0-rmse:0.69659\n",
      "[158]\tvalidation_0-rmse:0.69476\n",
      "[159]\tvalidation_0-rmse:0.69297\n",
      "[160]\tvalidation_0-rmse:0.69122\n",
      "[161]\tvalidation_0-rmse:0.68951\n",
      "[162]\tvalidation_0-rmse:0.68781\n",
      "[163]\tvalidation_0-rmse:0.68614\n",
      "[164]\tvalidation_0-rmse:0.68445\n",
      "[165]\tvalidation_0-rmse:0.68281\n",
      "[166]\tvalidation_0-rmse:0.68123\n",
      "[167]\tvalidation_0-rmse:0.67965\n",
      "[168]\tvalidation_0-rmse:0.67812\n",
      "[169]\tvalidation_0-rmse:0.67660\n",
      "[170]\tvalidation_0-rmse:0.67511\n",
      "[171]\tvalidation_0-rmse:0.67368\n",
      "[172]\tvalidation_0-rmse:0.67225\n",
      "[173]\tvalidation_0-rmse:0.67082\n",
      "[174]\tvalidation_0-rmse:0.66946\n",
      "[175]\tvalidation_0-rmse:0.66808\n",
      "[176]\tvalidation_0-rmse:0.66673\n",
      "[177]\tvalidation_0-rmse:0.66544\n",
      "[178]\tvalidation_0-rmse:0.66414\n",
      "[179]\tvalidation_0-rmse:0.66285\n",
      "[180]\tvalidation_0-rmse:0.66162\n",
      "[181]\tvalidation_0-rmse:0.66042\n",
      "[182]\tvalidation_0-rmse:0.65925\n",
      "[183]\tvalidation_0-rmse:0.65806\n",
      "[184]\tvalidation_0-rmse:0.65693\n",
      "[185]\tvalidation_0-rmse:0.65578\n",
      "[186]\tvalidation_0-rmse:0.65468\n",
      "[187]\tvalidation_0-rmse:0.65358\n",
      "[188]\tvalidation_0-rmse:0.65247\n",
      "[189]\tvalidation_0-rmse:0.65140\n",
      "[190]\tvalidation_0-rmse:0.65039\n",
      "[191]\tvalidation_0-rmse:0.64938\n",
      "[192]\tvalidation_0-rmse:0.64838\n",
      "[193]\tvalidation_0-rmse:0.64739\n",
      "[194]\tvalidation_0-rmse:0.64643\n",
      "[195]\tvalidation_0-rmse:0.64547\n",
      "[196]\tvalidation_0-rmse:0.64455\n",
      "[197]\tvalidation_0-rmse:0.64364\n",
      "[198]\tvalidation_0-rmse:0.64274\n",
      "[199]\tvalidation_0-rmse:0.64187\n",
      "[200]\tvalidation_0-rmse:0.64099\n",
      "[201]\tvalidation_0-rmse:0.64015\n",
      "[202]\tvalidation_0-rmse:0.63929\n",
      "[203]\tvalidation_0-rmse:0.63848\n",
      "[204]\tvalidation_0-rmse:0.63766\n",
      "[205]\tvalidation_0-rmse:0.63686\n",
      "[206]\tvalidation_0-rmse:0.63607\n",
      "[207]\tvalidation_0-rmse:0.63531\n",
      "[208]\tvalidation_0-rmse:0.63454\n",
      "[209]\tvalidation_0-rmse:0.63379\n",
      "[210]\tvalidation_0-rmse:0.63310\n",
      "[211]\tvalidation_0-rmse:0.63237\n",
      "[212]\tvalidation_0-rmse:0.63165\n",
      "[213]\tvalidation_0-rmse:0.63097\n",
      "[214]\tvalidation_0-rmse:0.63028\n",
      "[215]\tvalidation_0-rmse:0.62962\n",
      "[216]\tvalidation_0-rmse:0.62895\n",
      "[217]\tvalidation_0-rmse:0.62831\n",
      "[218]\tvalidation_0-rmse:0.62767\n",
      "[219]\tvalidation_0-rmse:0.62705\n",
      "[220]\tvalidation_0-rmse:0.62645\n",
      "[221]\tvalidation_0-rmse:0.62586\n",
      "[222]\tvalidation_0-rmse:0.62527\n",
      "[223]\tvalidation_0-rmse:0.62472\n",
      "[224]\tvalidation_0-rmse:0.62415\n",
      "[225]\tvalidation_0-rmse:0.62360\n",
      "[226]\tvalidation_0-rmse:0.62306\n",
      "[227]\tvalidation_0-rmse:0.62253\n",
      "[228]\tvalidation_0-rmse:0.62200\n",
      "[229]\tvalidation_0-rmse:0.62148\n",
      "[230]\tvalidation_0-rmse:0.62097\n",
      "[231]\tvalidation_0-rmse:0.62047\n",
      "[232]\tvalidation_0-rmse:0.62001\n",
      "[233]\tvalidation_0-rmse:0.61952\n",
      "[234]\tvalidation_0-rmse:0.61907\n",
      "[235]\tvalidation_0-rmse:0.61861\n",
      "[236]\tvalidation_0-rmse:0.61815\n",
      "[237]\tvalidation_0-rmse:0.61773\n",
      "[238]\tvalidation_0-rmse:0.61728\n",
      "[239]\tvalidation_0-rmse:0.61686\n",
      "[240]\tvalidation_0-rmse:0.61644\n",
      "[241]\tvalidation_0-rmse:0.61602\n",
      "[242]\tvalidation_0-rmse:0.61558\n",
      "[243]\tvalidation_0-rmse:0.61520\n",
      "[244]\tvalidation_0-rmse:0.61482\n",
      "[245]\tvalidation_0-rmse:0.61442\n",
      "[246]\tvalidation_0-rmse:0.61401\n",
      "[247]\tvalidation_0-rmse:0.61366\n",
      "[248]\tvalidation_0-rmse:0.61330\n",
      "[249]\tvalidation_0-rmse:0.61293\n",
      "[250]\tvalidation_0-rmse:0.61259\n",
      "[251]\tvalidation_0-rmse:0.61223\n",
      "[252]\tvalidation_0-rmse:0.61187\n",
      "[253]\tvalidation_0-rmse:0.61156\n",
      "[254]\tvalidation_0-rmse:0.61126\n",
      "[255]\tvalidation_0-rmse:0.61096\n",
      "[256]\tvalidation_0-rmse:0.61066\n",
      "[257]\tvalidation_0-rmse:0.61035\n",
      "[258]\tvalidation_0-rmse:0.61003\n",
      "[259]\tvalidation_0-rmse:0.60977\n",
      "[260]\tvalidation_0-rmse:0.60950\n",
      "[261]\tvalidation_0-rmse:0.60924\n",
      "[262]\tvalidation_0-rmse:0.60893\n",
      "[263]\tvalidation_0-rmse:0.60864\n",
      "[264]\tvalidation_0-rmse:0.60835\n",
      "[265]\tvalidation_0-rmse:0.60807\n",
      "[266]\tvalidation_0-rmse:0.60784\n",
      "[267]\tvalidation_0-rmse:0.60761\n",
      "[268]\tvalidation_0-rmse:0.60734\n",
      "[269]\tvalidation_0-rmse:0.60712\n",
      "[270]\tvalidation_0-rmse:0.60689\n",
      "[271]\tvalidation_0-rmse:0.60664\n",
      "[272]\tvalidation_0-rmse:0.60639\n",
      "[273]\tvalidation_0-rmse:0.60614\n",
      "[274]\tvalidation_0-rmse:0.60593\n",
      "[275]\tvalidation_0-rmse:0.60570\n",
      "[276]\tvalidation_0-rmse:0.60550\n",
      "[277]\tvalidation_0-rmse:0.60532\n",
      "[278]\tvalidation_0-rmse:0.60514\n",
      "[279]\tvalidation_0-rmse:0.60497\n",
      "[280]\tvalidation_0-rmse:0.60478\n",
      "[281]\tvalidation_0-rmse:0.60457\n",
      "[282]\tvalidation_0-rmse:0.60437\n",
      "[283]\tvalidation_0-rmse:0.60417\n",
      "[284]\tvalidation_0-rmse:0.60398\n",
      "[285]\tvalidation_0-rmse:0.60381\n",
      "[286]\tvalidation_0-rmse:0.60363\n",
      "[287]\tvalidation_0-rmse:0.60347\n",
      "[288]\tvalidation_0-rmse:0.60331\n",
      "[289]\tvalidation_0-rmse:0.60319\n",
      "[290]\tvalidation_0-rmse:0.60303\n",
      "[291]\tvalidation_0-rmse:0.60291\n",
      "[292]\tvalidation_0-rmse:0.60276\n",
      "[293]\tvalidation_0-rmse:0.60263\n",
      "[294]\tvalidation_0-rmse:0.60251\n",
      "[295]\tvalidation_0-rmse:0.60234\n",
      "[296]\tvalidation_0-rmse:0.60222\n",
      "[297]\tvalidation_0-rmse:0.60206\n",
      "[298]\tvalidation_0-rmse:0.60195\n",
      "[299]\tvalidation_0-rmse:0.60182\n",
      "[300]\tvalidation_0-rmse:0.60167\n",
      "[301]\tvalidation_0-rmse:0.60156\n",
      "[302]\tvalidation_0-rmse:0.60146\n",
      "[303]\tvalidation_0-rmse:0.60136\n",
      "[304]\tvalidation_0-rmse:0.60125\n",
      "[305]\tvalidation_0-rmse:0.60114\n",
      "[306]\tvalidation_0-rmse:0.60102\n",
      "[307]\tvalidation_0-rmse:0.60091\n",
      "[308]\tvalidation_0-rmse:0.60078\n",
      "[309]\tvalidation_0-rmse:0.60066\n",
      "[310]\tvalidation_0-rmse:0.60055\n",
      "[311]\tvalidation_0-rmse:0.60046\n",
      "[312]\tvalidation_0-rmse:0.60034\n",
      "[313]\tvalidation_0-rmse:0.60023\n",
      "[314]\tvalidation_0-rmse:0.60013\n",
      "[315]\tvalidation_0-rmse:0.60004\n",
      "[316]\tvalidation_0-rmse:0.59994\n",
      "[317]\tvalidation_0-rmse:0.59986\n",
      "[318]\tvalidation_0-rmse:0.59980\n",
      "[319]\tvalidation_0-rmse:0.59970\n",
      "[320]\tvalidation_0-rmse:0.59961\n",
      "[321]\tvalidation_0-rmse:0.59952\n",
      "[322]\tvalidation_0-rmse:0.59946\n",
      "[323]\tvalidation_0-rmse:0.59939\n",
      "[324]\tvalidation_0-rmse:0.59934\n",
      "[325]\tvalidation_0-rmse:0.59925\n",
      "[326]\tvalidation_0-rmse:0.59917\n",
      "[327]\tvalidation_0-rmse:0.59910\n",
      "[328]\tvalidation_0-rmse:0.59903\n",
      "[329]\tvalidation_0-rmse:0.59896\n",
      "[330]\tvalidation_0-rmse:0.59888\n",
      "[331]\tvalidation_0-rmse:0.59881\n",
      "[332]\tvalidation_0-rmse:0.59878\n",
      "[333]\tvalidation_0-rmse:0.59871\n",
      "[334]\tvalidation_0-rmse:0.59864\n",
      "[335]\tvalidation_0-rmse:0.59859\n",
      "[336]\tvalidation_0-rmse:0.59855\n",
      "[337]\tvalidation_0-rmse:0.59848\n",
      "[338]\tvalidation_0-rmse:0.59844\n",
      "[339]\tvalidation_0-rmse:0.59838\n",
      "[340]\tvalidation_0-rmse:0.59832\n",
      "[341]\tvalidation_0-rmse:0.59831\n",
      "[342]\tvalidation_0-rmse:0.59826\n",
      "[343]\tvalidation_0-rmse:0.59824\n",
      "[344]\tvalidation_0-rmse:0.59821\n",
      "[345]\tvalidation_0-rmse:0.59814\n",
      "[346]\tvalidation_0-rmse:0.59810\n",
      "[347]\tvalidation_0-rmse:0.59806\n",
      "[348]\tvalidation_0-rmse:0.59802\n",
      "[349]\tvalidation_0-rmse:0.59800\n",
      "[350]\tvalidation_0-rmse:0.59800\n",
      "[351]\tvalidation_0-rmse:0.59795\n",
      "[352]\tvalidation_0-rmse:0.59792\n",
      "[353]\tvalidation_0-rmse:0.59788\n",
      "[354]\tvalidation_0-rmse:0.59785\n",
      "[355]\tvalidation_0-rmse:0.59784\n",
      "[356]\tvalidation_0-rmse:0.59780\n",
      "[357]\tvalidation_0-rmse:0.59778\n",
      "[358]\tvalidation_0-rmse:0.59778\n",
      "[359]\tvalidation_0-rmse:0.59775\n",
      "[360]\tvalidation_0-rmse:0.59771\n",
      "[361]\tvalidation_0-rmse:0.59768\n",
      "[362]\tvalidation_0-rmse:0.59767\n",
      "[363]\tvalidation_0-rmse:0.59767\n",
      "[364]\tvalidation_0-rmse:0.59762\n",
      "[365]\tvalidation_0-rmse:0.59762\n",
      "[366]\tvalidation_0-rmse:0.59761\n",
      "[367]\tvalidation_0-rmse:0.59760\n",
      "[368]\tvalidation_0-rmse:0.59759\n",
      "[369]\tvalidation_0-rmse:0.59755\n",
      "[370]\tvalidation_0-rmse:0.59752\n",
      "[371]\tvalidation_0-rmse:0.59752\n",
      "[372]\tvalidation_0-rmse:0.59752\n",
      "[373]\tvalidation_0-rmse:0.59749\n",
      "[374]\tvalidation_0-rmse:0.59749\n",
      "[375]\tvalidation_0-rmse:0.59746\n",
      "[376]\tvalidation_0-rmse:0.59746\n",
      "[377]\tvalidation_0-rmse:0.59743\n",
      "[378]\tvalidation_0-rmse:0.59741\n",
      "[379]\tvalidation_0-rmse:0.59738\n",
      "[380]\tvalidation_0-rmse:0.59739\n",
      "[381]\tvalidation_0-rmse:0.59736\n",
      "[382]\tvalidation_0-rmse:0.59733\n",
      "[383]\tvalidation_0-rmse:0.59730\n",
      "[384]\tvalidation_0-rmse:0.59727\n",
      "[385]\tvalidation_0-rmse:0.59727\n",
      "[386]\tvalidation_0-rmse:0.59724\n",
      "[387]\tvalidation_0-rmse:0.59722\n",
      "[388]\tvalidation_0-rmse:0.59719\n",
      "[389]\tvalidation_0-rmse:0.59719\n",
      "[390]\tvalidation_0-rmse:0.59717\n",
      "[391]\tvalidation_0-rmse:0.59713\n",
      "[392]\tvalidation_0-rmse:0.59710\n",
      "[393]\tvalidation_0-rmse:0.59708\n",
      "[394]\tvalidation_0-rmse:0.59709\n",
      "[395]\tvalidation_0-rmse:0.59710\n",
      "[396]\tvalidation_0-rmse:0.59712\n",
      "[397]\tvalidation_0-rmse:0.59712\n",
      "[398]\tvalidation_0-rmse:0.59710\n",
      "[399]\tvalidation_0-rmse:0.59707\n",
      "[400]\tvalidation_0-rmse:0.59708\n",
      "[401]\tvalidation_0-rmse:0.59709\n",
      "[402]\tvalidation_0-rmse:0.59706\n",
      "[403]\tvalidation_0-rmse:0.59705\n",
      "[404]\tvalidation_0-rmse:0.59707\n",
      "[405]\tvalidation_0-rmse:0.59708\n",
      "[406]\tvalidation_0-rmse:0.59706\n",
      "[407]\tvalidation_0-rmse:0.59709\n",
      "[408]\tvalidation_0-rmse:0.59707\n",
      "[409]\tvalidation_0-rmse:0.59706\n",
      "[410]\tvalidation_0-rmse:0.59705\n",
      "[411]\tvalidation_0-rmse:0.59704\n",
      "[412]\tvalidation_0-rmse:0.59702\n",
      "[413]\tvalidation_0-rmse:0.59701\n",
      "[414]\tvalidation_0-rmse:0.59700\n",
      "[415]\tvalidation_0-rmse:0.59702\n",
      "[416]\tvalidation_0-rmse:0.59700\n",
      "[417]\tvalidation_0-rmse:0.59699\n",
      "[418]\tvalidation_0-rmse:0.59698\n",
      "[419]\tvalidation_0-rmse:0.59699\n",
      "[420]\tvalidation_0-rmse:0.59699\n",
      "[421]\tvalidation_0-rmse:0.59698\n",
      "[422]\tvalidation_0-rmse:0.59698\n",
      "[423]\tvalidation_0-rmse:0.59697\n",
      "[424]\tvalidation_0-rmse:0.59697\n",
      "[425]\tvalidation_0-rmse:0.59696\n",
      "[426]\tvalidation_0-rmse:0.59695\n",
      "[427]\tvalidation_0-rmse:0.59693\n",
      "[428]\tvalidation_0-rmse:0.59694\n",
      "[429]\tvalidation_0-rmse:0.59693\n",
      "[430]\tvalidation_0-rmse:0.59694\n",
      "[431]\tvalidation_0-rmse:0.59694\n",
      "[432]\tvalidation_0-rmse:0.59693\n",
      "[433]\tvalidation_0-rmse:0.59693\n",
      "[434]\tvalidation_0-rmse:0.59693\n",
      "[435]\tvalidation_0-rmse:0.59693\n",
      "[436]\tvalidation_0-rmse:0.59692\n",
      "[437]\tvalidation_0-rmse:0.59693\n",
      "[438]\tvalidation_0-rmse:0.59694\n",
      "[439]\tvalidation_0-rmse:0.59691\n",
      "[440]\tvalidation_0-rmse:0.59694\n",
      "[441]\tvalidation_0-rmse:0.59694\n",
      "[442]\tvalidation_0-rmse:0.59692\n",
      "[443]\tvalidation_0-rmse:0.59691\n",
      "[444]\tvalidation_0-rmse:0.59694\n",
      "[445]\tvalidation_0-rmse:0.59693\n",
      "[446]\tvalidation_0-rmse:0.59692\n",
      "[447]\tvalidation_0-rmse:0.59692\n",
      "[448]\tvalidation_0-rmse:0.59691\n",
      "[449]\tvalidation_0-rmse:0.59690\n",
      "[450]\tvalidation_0-rmse:0.59689\n",
      "[451]\tvalidation_0-rmse:0.59689\n",
      "[452]\tvalidation_0-rmse:0.59689\n",
      "[453]\tvalidation_0-rmse:0.59689\n",
      "[454]\tvalidation_0-rmse:0.59690\n",
      "[455]\tvalidation_0-rmse:0.59690\n",
      "[456]\tvalidation_0-rmse:0.59690\n",
      "[457]\tvalidation_0-rmse:0.59690\n",
      "[458]\tvalidation_0-rmse:0.59689\n",
      "[459]\tvalidation_0-rmse:0.59691\n",
      "[460]\tvalidation_0-rmse:0.59692\n",
      "[461]\tvalidation_0-rmse:0.59694\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>XGBRegressor(base_score=None, booster=None, callbacks=None,\n",
       "             colsample_bylevel=None, colsample_bynode=None,\n",
       "             colsample_bytree=None, early_stopping_rounds=None,\n",
       "             enable_categorical=False, eval_metric=None, feature_types=None,\n",
       "             gamma=None, gpu_id=None, grow_policy=None, importance_type=None,\n",
       "             interaction_constraints=None, learning_rate=0.01, max_bin=None,\n",
       "             max_cat_threshold=None, max_cat_to_onehot=None,\n",
       "             max_delta_step=None, max_depth=8, max_leaves=None,\n",
       "             min_child_weight=None, missing=nan, monotone_constraints=None,\n",
       "             n_estimators=1000, n_jobs=None, num_parallel_tree=None,\n",
       "             predictor=None, random_state=None, ...)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">XGBRegressor</label><div class=\"sk-toggleable__content\"><pre>XGBRegressor(base_score=None, booster=None, callbacks=None,\n",
       "             colsample_bylevel=None, colsample_bynode=None,\n",
       "             colsample_bytree=None, early_stopping_rounds=None,\n",
       "             enable_categorical=False, eval_metric=None, feature_types=None,\n",
       "             gamma=None, gpu_id=None, grow_policy=None, importance_type=None,\n",
       "             interaction_constraints=None, learning_rate=0.01, max_bin=None,\n",
       "             max_cat_threshold=None, max_cat_to_onehot=None,\n",
       "             max_delta_step=None, max_depth=8, max_leaves=None,\n",
       "             min_child_weight=None, missing=nan, monotone_constraints=None,\n",
       "             n_estimators=1000, n_jobs=None, num_parallel_tree=None,\n",
       "             predictor=None, random_state=None, ...)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "XGBRegressor(base_score=None, booster=None, callbacks=None,\n",
       "             colsample_bylevel=None, colsample_bynode=None,\n",
       "             colsample_bytree=None, early_stopping_rounds=None,\n",
       "             enable_categorical=False, eval_metric=None, feature_types=None,\n",
       "             gamma=None, gpu_id=None, grow_policy=None, importance_type=None,\n",
       "             interaction_constraints=None, learning_rate=0.01, max_bin=None,\n",
       "             max_cat_threshold=None, max_cat_to_onehot=None,\n",
       "             max_delta_step=None, max_depth=8, max_leaves=None,\n",
       "             min_child_weight=None, missing=nan, monotone_constraints=None,\n",
       "             n_estimators=1000, n_jobs=None, num_parallel_tree=None,\n",
       "             predictor=None, random_state=None, ...)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb_model.fit(X_train, y_train, eval_set=[(X_val, y_val)], early_stopping_rounds=10, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation RMSE:  0.5968891546239947\n"
     ]
    }
   ],
   "source": [
    "y_pred = xgb_model.predict(X_val)\n",
    "\n",
    "print('Validation RMSE: ', mean_squared_error(y_val, y_pred, squared=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = Path('../artifacts/models/xgb_regressor.model')\n",
    "xgb_model.save_model(filename)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Neural Network Test RMSE:  2.4448367441098537\n"
     ]
    }
   ],
   "source": [
    "nn_model.eval()\n",
    "y_pred = np.array([])\n",
    "with torch.no_grad():\n",
    "    for i, (inputs, labels) in enumerate(test_loader):\n",
    "        outputs = nn_model(inputs)\n",
    "        y_pred = np.append(y_pred, outputs.squeeze(1).numpy())\n",
    "\n",
    "\n",
    "print('Neural Network Test RMSE: ', mean_squared_error(y_test, y_pred, squared=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGBoostRegressor Validation RMSE:  0.6175303550758209\n"
     ]
    }
   ],
   "source": [
    "y_pred = xgb_model.predict(X_test)\n",
    "\n",
    "print('XGBoostRegressor Validation RMSE: ', mean_squared_error(y_test, y_pred, squared=False))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}