forecast_est_monthly3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "01988880",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 30907,
     "status": "ok",
     "timestamp": 1655385599862,
     "user": {
      "displayName": "NDOUMBE Emile Herve",
      "userId": "09643065439420591964"
     },
     "user_tz": 240
    },
    "id": "01988880",
    "outputId": "e8c8fbc0-6a9c-4e72-bec9-1f338ec43a54"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
      "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (1.3.5)\n",
      "Requirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.7/dist-packages (from pandas) (1.21.6)\n",
      "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas) (2022.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas) (2.8.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas) (1.15.0)\n",
      "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (1.21.6)\n",
      "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
      "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (3.2.2)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (0.11.0)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (2.8.2)\n",
      "Requirement already satisfied: numpy>=1.11 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (1.21.6)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (3.0.9)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (1.4.2)\n",
      "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from kiwisolver>=1.0.1->matplotlib) (4.2.0)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.1->matplotlib) (1.15.0)\n",
      "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
      "Collecting functools\n",
      "  Downloading functools-0.5.tar.gz (4.9 kB)\n",
      "Building wheels for collected packages: functools\n",
      "  Building wheel for functools (setup.py) ... \u001b[?25lerror\n",
      "\u001b[31m  ERROR: Failed building wheel for functools\u001b[0m\n",
      "\u001b[?25h  Running setup.py clean for functools\n",
      "Failed to build functools\n",
      "Installing collected packages: functools\n",
      "    Running setup.py install for functools ... \u001b[?25l\u001b[?25herror\n",
      "\u001b[31mERROR: Command errored out with exit status 1: /usr/bin/python3 -u -c 'import io, os, sys, setuptools, tokenize; sys.argv[0] = '\"'\"'/tmp/pip-install-8zzjn7wx/functools_9b99894b6f2a43daa5e606da2121897a/setup.py'\"'\"'; __file__='\"'\"'/tmp/pip-install-8zzjn7wx/functools_9b99894b6f2a43daa5e606da2121897a/setup.py'\"'\"';f = getattr(tokenize, '\"'\"'open'\"'\"', open)(__file__) if os.path.exists(__file__) else io.StringIO('\"'\"'from setuptools import setup; setup()'\"'\"');code = f.read().replace('\"'\"'\\r\\n'\"'\"', '\"'\"'\\n'\"'\"');f.close();exec(compile(code, __file__, '\"'\"'exec'\"'\"'))' install --record /tmp/pip-record-ibk9fq0t/install-record.txt --single-version-externally-managed --compile --install-headers /usr/local/include/python3.7/functools Check the logs for full command output.\u001b[0m\n",
      "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
      "Collecting varname\n",
      "  Downloading varname-0.8.3-py3-none-any.whl (21 kB)\n",
      "Collecting pure_eval<1.0.0\n",
      "  Downloading pure_eval-0.2.2-py3-none-any.whl (11 kB)\n",
      "Collecting executing<0.9.0,>=0.8.3\n",
      "  Downloading executing-0.8.3-py2.py3-none-any.whl (16 kB)\n",
      "Collecting asttokens<3.0.0,>=2.0.0\n",
      "  Downloading asttokens-2.0.5-py2.py3-none-any.whl (20 kB)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from asttokens<3.0.0,>=2.0.0->varname) (1.15.0)\n",
      "Installing collected packages: pure-eval, executing, asttokens, varname\n",
      "Successfully installed asttokens-2.0.5 executing-0.8.3 pure-eval-0.2.2 varname-0.8.3\n",
      "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
      "Requirement already satisfied: statsmodels in /usr/local/lib/python3.7/dist-packages (0.10.2)\n",
      "Requirement already satisfied: scipy>=0.18 in /usr/local/lib/python3.7/dist-packages (from statsmodels) (1.4.1)\n",
      "Requirement already satisfied: numpy>=1.11 in /usr/local/lib/python3.7/dist-packages (from statsmodels) (1.21.6)\n",
      "Requirement already satisfied: pandas>=0.19 in /usr/local/lib/python3.7/dist-packages (from statsmodels) (1.3.5)\n",
      "Requirement already satisfied: patsy>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from statsmodels) (0.5.2)\n",
      "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.19->statsmodels) (2022.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.19->statsmodels) (2.8.2)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from patsy>=0.4.0->statsmodels) (1.15.0)\n",
      "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
      "Requirement already satisfied: sklearn in /usr/local/lib/python3.7/dist-packages (0.0)\n",
      "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from sklearn) (1.0.2)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->sklearn) (3.1.0)\n",
      "Requirement already satisfied: numpy>=1.14.6 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->sklearn) (1.21.6)\n",
      "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->sklearn) (1.1.0)\n",
      "Requirement already satisfied: scipy>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->sklearn) (1.4.1)\n",
      "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
      "Requirement already satisfied: seaborn in /usr/local/lib/python3.7/dist-packages (0.11.2)\n",
      "Requirement already satisfied: pandas>=0.23 in /usr/local/lib/python3.7/dist-packages (from seaborn) (1.3.5)\n",
      "Requirement already satisfied: matplotlib>=2.2 in /usr/local/lib/python3.7/dist-packages (from seaborn) (3.2.2)\n",
      "Requirement already satisfied: scipy>=1.0 in /usr/local/lib/python3.7/dist-packages (from seaborn) (1.4.1)\n",
      "Requirement already satisfied: numpy>=1.15 in /usr/local/lib/python3.7/dist-packages (from seaborn) (1.21.6)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=2.2->seaborn) (1.4.2)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=2.2->seaborn) (2.8.2)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=2.2->seaborn) (3.0.9)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=2.2->seaborn) (0.11.0)\n",
      "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from kiwisolver>=1.0.1->matplotlib>=2.2->seaborn) (4.2.0)\n",
      "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.23->seaborn) (2022.1)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.1->matplotlib>=2.2->seaborn) (1.15.0)\n",
      "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
      "Requirement already satisfied: xgboost in /usr/local/lib/python3.7/dist-packages (0.90)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from xgboost) (1.21.6)\n",
      "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from xgboost) (1.4.1)\n",
      "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
      "Collecting scikit-optimize\n",
      "  Downloading scikit_optimize-0.9.0-py2.py3-none-any.whl (100 kB)\n",
      "\u001b[K     |████████████████████████████████| 100 kB 5.7 MB/s \n",
      "\u001b[?25hRequirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.7/dist-packages (from scikit-optimize) (1.0.2)\n",
      "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.7/dist-packages (from scikit-optimize) (1.21.6)\n",
      "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-optimize) (1.1.0)\n",
      "Requirement already satisfied: scipy>=0.19.1 in /usr/local/lib/python3.7/dist-packages (from scikit-optimize) (1.4.1)\n",
      "Collecting pyaml>=16.9\n",
      "  Downloading pyaml-21.10.1-py2.py3-none-any.whl (24 kB)\n",
      "Requirement already satisfied: PyYAML in /usr/local/lib/python3.7/dist-packages (from pyaml>=16.9->scikit-optimize) (3.13)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.20.0->scikit-optimize) (3.1.0)\n",
      "Installing collected packages: pyaml, scikit-optimize\n",
      "Successfully installed pyaml-21.10.1 scikit-optimize-0.9.0\n",
      "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
      "Requirement already satisfied: tensorflow in /usr/local/lib/python3.7/dist-packages (2.8.2+zzzcolab20220527125636)\n",
      "Requirement already satisfied: tensorboard<2.9,>=2.8 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (2.8.0)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (3.3.0)\n",
      "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (3.1.0)\n",
      "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (0.2.0)\n",
      "Requirement already satisfied: flatbuffers>=1.12 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (2.0)\n",
      "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.15.0)\n",
      "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.14.1)\n",
      "Requirement already satisfied: tensorflow-estimator<2.9,>=2.8 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (2.8.0)\n",
      "Requirement already satisfied: keras-preprocessing>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.1.2)\n",
      "Requirement already satisfied: absl-py>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.1.0)\n",
      "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (4.2.0)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from tensorflow) (57.4.0)\n",
      "Requirement already satisfied: protobuf<3.20,>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (3.17.3)\n",
      "Requirement already satisfied: keras<2.9,>=2.8.0rc0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (2.8.0)\n",
      "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.6.3)\n",
      "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (0.26.0)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.1.0)\n",
      "Requirement already satisfied: gast>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (0.5.3)\n",
      "Requirement already satisfied: libclang>=9.0.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (14.0.1)\n",
      "Requirement already satisfied: numpy>=1.20 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.21.6)\n",
      "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.46.3)\n",
      "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.7/dist-packages (from astunparse>=1.6.0->tensorflow) (0.37.1)\n",
      "Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py>=2.9.0->tensorflow) (1.5.2)\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow) (1.35.0)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow) (1.0.1)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow) (2.23.0)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow) (0.4.6)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow) (3.3.7)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow) (1.8.1)\n",
      "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow) (0.6.1)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow) (4.8)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow) (4.2.4)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow) (0.2.8)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.9,>=2.8->tensorflow) (1.3.1)\n",
      "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard<2.9,>=2.8->tensorflow) (4.11.4)\n",
      "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.9,>=2.8->tensorflow) (3.8.0)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow) (0.4.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow) (2022.5.18.1)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow) (1.24.3)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow) (2.10)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow) (3.0.4)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.9,>=2.8->tensorflow) (3.2.0)\n"
     ]
    }
   ],
   "source": [
    "#================== Out-of-sample forecasting excess industry-level return based on machine learning methods===================================================================================================#\n",
    "\n",
    "\n",
    "\n",
    "!pip install pandas\n",
    "!pip install numpy\n",
    "!pip install matplotlib\n",
    "!pip install functools\n",
    "!pip install varname\n",
    "!pip install statsmodels\n",
    "!pip install sklearn\n",
    "!pip install seaborn\n",
    "!pip install xgboost \n",
    "!pip install scikit-optimize\n",
    "!pip install tensorflow\n",
    "\n",
    "!pip install dask\n",
    "import dask as dd\n",
    "import glob\n",
    "\n",
    "!pip install arch\n",
    "from arch import arch_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a96e58ab",
   "metadata": {
    "executionInfo": {
     "elapsed": 3553,
     "status": "ok",
     "timestamp": 1655385642852,
     "user": {
      "displayName": "NDOUMBE Emile Herve",
      "userId": "09643065439420591964"
     },
     "user_tz": 240
    },
    "id": "a96e58ab"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from functools import reduce\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import sys\n",
    "import math\n",
    "from varname import nameof\n",
    "import statsmodels.formula.api as smf\n",
    "import statsmodels.api as sm\n",
    "from statsmodels import robust\n",
    "from statsmodels.tsa.api import VAR\n",
    "from sklearn.metrics import r2_score\n",
    "import seaborn as sns\n",
    "\n",
    "# import a Lasso module for time series analysis\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.linear_model import LassoCV, RidgeCV\n",
    "from sklearn.model_selection import TimeSeriesSplit\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error\n",
    "\n",
    "# import Random Forest module\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "# import XGBoost module\n",
    "from xgboost import XGBRegressor\n",
    "from skopt import gp_minimize\n",
    "from skopt.space import Real, Integer\n",
    "from functools import partial\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Activation, Bidirectional, LSTM, Dropout, SimpleRNN\n",
    "from tensorflow.keras import backend as K\n",
    "from tensorflow.keras.backend import clear_session\n",
    "from tensorflow.keras import regularizers\n",
    "from tensorflow.keras.layers import Flatten\n",
    "from tensorflow.keras.layers import Conv1D\n",
    "from tensorflow.keras.layers import MaxPooling1D\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.cross_decomposition import PLSRegression, PLSSVD\n",
    "from sklearn.preprocessing import scale\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "import warnings\n",
    "#import scipy.optimize as spop\n",
    "from scipy import stats\n",
    "from statsmodels.tsa.vector_ar.var_model import forecast\n",
    "from scipy.optimize import minimize\n",
    "import multiprocessing as multi\n",
    "warnings.filterwarnings(action='ignore',category=RuntimeWarning)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5271caf3",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 26376,
     "status": "ok",
     "timestamp": 1655385677375,
     "user": {
      "displayName": "NDOUMBE Emile Herve",
      "userId": "09643065439420591964"
     },
     "user_tz": 240
    },
    "id": "5271caf3",
    "outputId": "90f6254a-b90e-4d1a-c514-c50b0e4beeac"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/gdrive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/gdrive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "936e3e42",
   "metadata": {
    "id": "936e3e42"
   },
   "outputs": [],
   "source": [
    "##### Set the current working directory\n",
    "\n",
    "path1 =  '/content/gdrive/MyDrive/garch_month'                   \n",
    "all_files = glob.glob(path1 + \"/*.csv\")\n",
    "\n",
    "path2='/content/gdrive/MyDrive/filemonth'\n",
    "files=glob.glob(path2 +\"/*.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "Qoqd1ct8YEuk",
   "metadata": {
    "id": "Qoqd1ct8YEuk"
   },
   "outputs": [],
   "source": [
    "#### Implement our model: sDOC + Garch(1,1) function\n",
    "\n",
    "def garch_mle(params):\n",
    "    mu_0=params[0]\n",
    "    mu_1=params[1]\n",
    "    mu_2=params[2]\n",
    "    mu_3=params[3]\n",
    "    mu_4=params[4]\n",
    "    mu_5=params[5]\n",
    "    mu_6=params[6]\n",
    "    mu_7=params[7]\n",
    "    mu_8=params[8]\n",
    "    mu_9=params[9]\n",
    "    mu_10=params[10]\n",
    "    omega=params[11]\n",
    "    alpha=params[12]\n",
    "    beta=params[13]\n",
    "    \n",
    "    \n",
    "    global R,X\n",
    "    assert (R.shape[0] == X.shape[0]), \"numbers of rows not match!\"\n",
    "    T = X.shape[0]\n",
    "    dim = X.shape[1]\n",
    "    # R = R.flatten()\n",
    "    # X = X.flatten() # flatten arrays\n",
    "\n",
    "    R1 = np.empty( shape = (0, 1) )\n",
    "    X1 = np.empty( shape = (0, dim) )\n",
    "    for t in np.arange(0, T-tau):\n",
    "        R1 = np.append(R1, R[t+tau].reshape(1, 1), axis = 0)\n",
    "        X1 = np.append(X1, X[t, :].reshape(1, dim), axis = 0)\n",
    "\n",
    "    a = np.mat(R1)\n",
    "    R1=a.T\n",
    "    X1 = np.mat(X1)\n",
    "    mean=np.average(R1)\n",
    "    \n",
    "    # Long_run volatility\n",
    "    long_run=omega/(1-alpha-beta)\n",
    "    \n",
    "    #calculating realised and conditional volatility\n",
    "    resid=R1-mu_0-X1[:,0]*mu_1-X1[:,1]*mu_2-X1[:,2]*mu_3-X1[:,3]*mu_4-X1[:,4]*mu_5-X1[:,5]*mu_6-X1[:,6]*mu_7-X1[:,7]*mu_8-X1[:,8]*mu_9-X1[:,9]*mu_10\n",
    "    \n",
    "    realised=abs(resid)\n",
    "    c=np.array(realised)\n",
    "    conditional=np.zeros(len(R1))\n",
    "    conditional[0]=long_run\n",
    "    for t in range(1,len(R1)):\n",
    "        conditional[t]=omega+alpha*resid[t-1]**2+beta*conditional[t-1]\n",
    "        \n",
    "    #calculating log_likelihood\n",
    "    likelihood=(1/((2*np.pi)**(1/2)*conditional**(1/2)))*np.exp(-c**2/(2*conditional))\n",
    "    log_likelihood=np.sum(np.log(likelihood))\n",
    "    \n",
    "    return -log_likelihood\n",
    "\n",
    "\n",
    "#### Calculate the out-of-sample forecasts \n",
    "\n",
    "def multivar_ML(R, X, tau):\n",
    "    \n",
    "    assert (R.shape[0] == X.shape[0]), \"numbers of rows not match!\"\n",
    "    T = X.shape[0]\n",
    "    dim = X.shape[1]\n",
    "    \n",
    "\n",
    "    R1 = np.empty( shape = (0, 1) )\n",
    "    X1 = np.empty( shape = (0, dim) )\n",
    "    for t in np.arange(0, T-tau):\n",
    "        R1 = np.append(R1, R[t+tau].reshape(1, 1), axis = 0)\n",
    "        X1 = np.append(X1, X[t, :].reshape(1, dim), axis = 0)\n",
    "\n",
    "        \n",
    "    \n",
    "    # Maximizing log_likelihood\n",
    "    params=[0.05,0.6,0.6,0.6,0.6,0.6,0.6,0.01,0.5,0.4,0.5,0.1,0.05,0.92]\n",
    "    res=spop.minimize(garch_mle,params,method=\"Nelder-Mead\",bounds=((-1,1),(-1,1),(-1,1),(-1,1),(-1,1),(-1,1),(-1,1),(-1,1),(-1,1),(-1,1),(-1,1),(0.001,1),(0.001,1),(0.001,1)))\n",
    "    params=res.x\n",
    "    mu_0=res.x[0]\n",
    "    mu=np.mat(res.x[1:11])\n",
    "     \n",
    "        \n",
    "    # compute residuals and residual variance\n",
    "    epsilon = R1 - mu_0 - (X1 @ mu.T)\n",
    "    var_res=np.var(epsilon)\n",
    "    # compute forecast\n",
    "    forecast_tau = mu_0 + ( mu @ X[T-1, :].reshape(dim, 1) )\n",
    "    return forecast_tau,var_res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "91f9e797",
   "metadata": {
    "executionInfo": {
     "elapsed": 1296,
     "status": "ok",
     "timestamp": 1655402693293,
     "user": {
      "displayName": "NDOUMBE Emile Herve",
      "userId": "09643065439420591964"
     },
     "user_tz": 240
    },
    "id": "91f9e797"
   },
   "outputs": [],
   "source": [
    "##### Implement the principal component regression model\n",
    "\n",
    "def multivar_lhOLSPC(R, X, num_PCs, tau):\n",
    "    assert (R.shape[0] == X.shape[0]), \"numbers of rows not match!\"\n",
    "    T = X.shape[0]\n",
    "    dim = X.shape[1]\n",
    "    # R = R.flatten()\n",
    "    # X = X.flatten() # flatten arrays\n",
    "    \n",
    "    R1 = np.empty( shape = (0, 1) )\n",
    "    X1 = np.empty( shape = (0, dim) )\n",
    "    for t in np.arange(0, T):\n",
    "        if t < T-tau:\n",
    "            R1 = np.append(R1, R[t+tau].reshape(1, 1), axis = 0)\n",
    "        else:\n",
    "            R1 = np.append(R1, np.array([0]).reshape(1, 1), axis = 0)\n",
    "        X1 = np.append(X1, X[t, :].reshape(1, dim), axis = 0)\n",
    "\n",
    "    # split the numpy array into train and test data\n",
    "    if tau == 0:\n",
    "        X1_train, X1_test = X1[0:(T-1), :], X1[(T-1):, :]\n",
    "        R1_train, R1_test = R1[0:(T-1), :], R1[(T-1):, :]\n",
    "    else:\n",
    "        X1_train, X1_test = X1[0:(T-tau), :], X1[(T-tau):, :]\n",
    "        R1_train, R1_test = R1[0:(T-tau)], R1[(T-tau):]\n",
    "\n",
    "    \n",
    "     # standardizing the features\n",
    "    X11 = StandardScaler().fit_transform(X1_train)\n",
    "    X12= StandardScaler().fit_transform(X1_test)\n",
    "    \n",
    "    pca = PCA(n_components = num_PCs)\n",
    "    X_train = pca.fit_transform(X11)\n",
    "    X_test= pca.transform(X12)\n",
    "    \n",
    "    # do time series regressions\n",
    "    PCs = np.mat(X_train)\n",
    "    PCs1 = sm.add_constant(PCs)\n",
    "    PC=np.mat(X_test)\n",
    "    R1 = np.mat(R1_train)\n",
    "    ts_res = sm.OLS(R1, PCs1).fit()\n",
    "    alpha = ts_res.params[0]\n",
    "    beta = np.mat(ts_res.params[1:])\n",
    "    \n",
    "    \n",
    "    \n",
    "    # compute residuals\n",
    "    epsilon = R1 - alpha - (PCs @ beta.T)\n",
    "    am=arch_model(epsilon, vol=\"Garch\", p=1, o=0, q=1, dist=\"Normal\")\n",
    "    res=am.fit(update_freq=5)\n",
    "    forecasts=res.forecast(reindex=False)\n",
    "    g=forecasts.variance.iloc[-3:]\n",
    "\n",
    "    \n",
    "    # forecast for the test data\n",
    "    forecast_tau=alpha + (beta@PC.T)\n",
    "    \n",
    "    return forecast_tau,g.values\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "sraMKEi07Xxj",
   "metadata": {
    "executionInfo": {
     "elapsed": 433,
     "status": "ok",
     "timestamp": 1655385701434,
     "user": {
      "displayName": "NDOUMBE Emile Herve",
      "userId": "09643065439420591964"
     },
     "user_tz": 240
    },
    "id": "sraMKEi07Xxj"
   },
   "outputs": [],
   "source": [
    "##### Implement Artificial neural network with 3 layers\n",
    "\n",
    "def ANNf(R, X, tau, batch_size: int, num_epochs: int):\n",
    "    assert (R.shape[0] == X.shape[0]), \"numbers of rows not match!\"\n",
    "    T = X.shape[0]\n",
    "    dim = X.shape[1]\n",
    "    # R = R.flatten()\n",
    "    # X = X.flatten() # flatten arrays\n",
    "\n",
    "    R1 = np.empty( shape = (0, 1) )\n",
    "    X1 = np.empty( shape = (0, dim) )\n",
    "    for t in np.arange(0, T):\n",
    "        if t < T-tau:\n",
    "            R1 = np.append(R1, R[t+tau].reshape(1, 1), axis = 0)\n",
    "        else:\n",
    "            R1 = np.append(R1, np.array([0]).reshape(1, 1), axis = 0)\n",
    "        X1 = np.append(X1, X[t, :].reshape(1, dim), axis = 0)\n",
    "\n",
    "    # studentize data\n",
    "    # X1 = StandardScaler().fit_transform(X1)\n",
    "\n",
    "    # split the numpy array into train and test data\n",
    "    if tau == 0:\n",
    "        X1_train, X1_test = X1[0:(T-1), :], X1[(T-1):, :]\n",
    "        R1_train, R1_test = R1[0:(T-1), :], R1[(T-1):, :]\n",
    "    else:\n",
    "        X1_train, X1_test = X1[0:(T-tau), :], X1[(T-tau):, :]\n",
    "        R1_train, R1_test = R1[0:(T-tau)], R1[(T-tau):]\n",
    "\n",
    "    # Create the model\n",
    "    model = Sequential()\n",
    "\n",
    "    # define the first layer of the ANN\n",
    "    model.add( Dense( 100, input_shape= (dim, ) ) )\n",
    "    model.add(Activation('relu'))\n",
    "    model.add( Dropout(0.5) )\n",
    "\n",
    "    # define the second layer\n",
    "    model.add( Dense(50) )\n",
    "    model.add( Activation('relu') )\n",
    "    model.add( Dropout(0.5) )\n",
    "\n",
    "    # define the third layer\n",
    "    model.add( Dense(10) )\n",
    "    model.add( Activation('relu') )\n",
    "    model.add( Dropout(0.5) )\n",
    "\n",
    "    # define the output layer\n",
    "    model.add( Dense(1) )\n",
    "    model.add( Activation('relu') )\n",
    "\n",
    "\n",
    "    # compile the ANN model\n",
    "    model.compile(loss='mean_squared_error', optimizer='adam')\n",
    "    # model.summary()\n",
    "\n",
    "    # train the model\n",
    "    model.fit(X1_train, R1_train, batch_size=batch_size, shuffle=False, epochs=num_epochs, verbose=0) # Verbosity mode 0: silent\n",
    "\n",
    "    # calculate residuals for the train data\n",
    "    R1_pred = model(X1_train)\n",
    "    # print('R1_pred = \\n', R1_pred)\n",
    "    residuals = R1_train - R1_pred\n",
    "    \n",
    "    var_res=np.var(residuals)\n",
    "\n",
    "\n",
    "    # forecast for the test data\n",
    "    forecasts = model(X1_test)\n",
    "    # print('forecasts = \\n', forecasts)\n",
    "    forecast_tau = float(forecasts[len(forecasts) - 1])\n",
    "    \n",
    "    K.clear_session()\n",
    "    del model # delete the model\n",
    "\n",
    "    # output results\n",
    "    if tau == 0:\n",
    "        return forecast_tau\n",
    "    else:\n",
    "        return forecast_tau,var_res\n",
    "    \n",
    "    \n",
    "def split_sequences(sequences, n_steps):\n",
    "    X, y = list(), list()\n",
    "    for i in range(len(sequences)):\n",
    "        # find the end of this pattern\n",
    "        end_ix = i + n_steps\n",
    "        # check if we are beyond the dataset\n",
    "        if end_ix > len(sequences):\n",
    "            break\n",
    "        # gather input and output parts of the pattern\n",
    "        seq_x, seq_y = sequences[i:end_ix, :-1], sequences[end_ix-1, -1]\n",
    "        X.append(seq_x)\n",
    "        y.append(seq_y)\n",
    "    return np.array(X), np.array(y)\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "LHQfOCb77X9q",
   "metadata": {
    "executionInfo": {
     "elapsed": 156,
     "status": "ok",
     "timestamp": 1655385709803,
     "user": {
      "displayName": "NDOUMBE Emile Herve",
      "userId": "09643065439420591964"
     },
     "user_tz": 240
    },
    "id": "LHQfOCb77X9q"
   },
   "outputs": [],
   "source": [
    "##### Calculate the mean squared error of XGBoost forecasts for validating samples\n",
    "def rmsfe_XGBoost(args, R_train, X_train, R_valid, X_valid, seed):\n",
    "    \"\"\"   seed               : model seed\n",
    "    booster : booster to use (gbtree, gblinear or dart; gbtree and dart use tree based models while gblinear uses linear functions)\n",
    "    n_estimators       : number of boosted trees to fit\n",
    "    max_depth          : maximum tree depth for base learners\n",
    "    learning_rate      : boosting learning rate (xgb’s “eta”)\n",
    "    max_delta_step : maximum step size\n",
    "    min_child_weight   : minimum sum of instance weight(hessian) needed in a child\n",
    "    subsample          : subsample ratio of the training instance\n",
    "    colsample_bytree   : subsample ratio of columns when constructing each tree\n",
    "    colsample_bylevel  : subsample ratio of columns for each split, in each level\n",
    "    gamma              :  regularization hyperparameter \n",
    "    reg_alpha, reg_lambda: regularization parameters \"\"\"\n",
    "\n",
    "    # global models, train_scores, test_scores, curr_model_hyper_params\n",
    "    curr_model_hyper_params = ['colsample_bylevel', 'colsample_bytree', 'gamma', 'learning_rate', 'max_delta_step', 'max_depth', \\\n",
    "                                                        'min_child_weight', 'n_estimators', 'reg_alpha', 'reg_lambda', 'subsample']\n",
    "    params = {curr_model_hyper_params[i]: args[i] for i, j in enumerate(curr_model_hyper_params)}\n",
    "    model = XGBRegressor(booster='gbtree', objective ='reg:squarederror', random_state=42, seed=seed)\n",
    "    model.set_params(**params)\n",
    "    model.fit(X_train, R_train) # fit training samples to model\n",
    "    R_pred = model.predict(X_valid)\n",
    "    msfe = mean_squared_error(R_valid, R_pred)\n",
    "    del model\n",
    "    return msfe\n",
    "\n",
    "##### Find optimal hyperparameters for XGBoost with cross validation\n",
    "def minimize_XGBoost(R_train, X_train, R_valid, X_valid, seed = 100, n_calls = 50):\n",
    "    # defining the space\n",
    "    space = [\n",
    "        Real(0.1, 1, name=\"colsample_bylevel\"),\n",
    "        Real(0.1, 1, name=\"colsample_bytree\"),\n",
    "        Real(0, 1, name=\"gamma\"),\n",
    "        Real(0, 1, name=\"learning_rate\"),\n",
    "        Real(0, 10, name=\"max_delta_step\"),\n",
    "        Integer(1, 15, name=\"max_depth\"),\n",
    "        Real(0.1, 500, name=\"min_child_weight\"),\n",
    "        Integer(10, 100, name=\"n_estimators\"),\n",
    "        Real(0, 0.5, name=\"reg_alpha\"),\n",
    "        Real(0, 0.5, name=\"reg_lambda\"),\n",
    "        Real(0.1, 1, name=\"subsample\"),\n",
    "    ]\n",
    "\n",
    "    objective_function = partial(rmsfe_XGBoost, R_train=R_train, X_train=X_train, R_valid=R_valid, X_valid=X_valid, seed=seed)\n",
    "\n",
    "    # minimize the RMSFE\n",
    "    res = gp_minimize(objective_function, space, base_estimator=None, n_calls=n_calls, n_random_starts=n_calls-1, random_state=42, n_jobs=1)\n",
    "    return res.x\n",
    "\n",
    "##### Implement XGBoost using cross validation\n",
    "def XGBoostf_CV(R, X, tau: int, seed=1234, n_calls = 150):\n",
    "    assert (R.shape[0] == X.shape[0]), \"numbers of rows not match!\"\n",
    "    assert(tau > 0), \"the forecast horizon must be greater than zero!\"\n",
    "    \n",
    "    T = X.shape[0]\n",
    "    dim = X.shape[1]\n",
    "    # R = R.flatten()\n",
    "    # X = X.flatten() # flatten arrays\n",
    "\n",
    "    R1 = np.empty( shape = (0, 1) )\n",
    "    X1 = np.empty( shape = (0, dim) )\n",
    "    for t in np.arange(0, T):\n",
    "        if t < T-tau:\n",
    "            R1 = np.append(R1, R[t+tau].reshape(1, 1), axis = 0)\n",
    "        else:\n",
    "            R1 = np.append(R1, np.array([0]).reshape(1, 1), axis = 0)\n",
    "        X1 = np.append(X1, X[t, :].reshape(1, dim), axis = 0)\n",
    "\n",
    "    # studentize data\n",
    "    X1 = StandardScaler().fit_transform(X1)\n",
    "\n",
    "    # split the numpy array into train, validation, and test data\n",
    "    sratio = 0.8\n",
    "    X1_train, X1_valid, X1_test = X1[0:(math.floor(sratio*T)-tau), :].reshape(-1, dim), X1[(math.floor(sratio*T)-tau):(T-tau), :].reshape(-1, dim), \\\n",
    "                                                                                                                                                                                X1[(T-tau):, :].reshape(-1, dim)\n",
    "    R1_train, R1_valid, R1_test = R1[0:(math.floor(sratio*T)-tau), :].reshape(-1, 1), R1[(math.floor(sratio*T)-tau):(T-tau), :].reshape(-1, 1), \\\n",
    "                                                                                                                                                                                    R1[(T-tau):, :].reshape(-1, 1)\n",
    "\n",
    "    # find optimal hyperparameters for XGBoost using cross validation\n",
    "    x = minimize_XGBoost(R1_train, X1_train, R1_valid, X1_valid, seed = seed, n_calls = n_calls)\n",
    "\n",
    "    # define model with optimal hyperparameters\n",
    "    model = XGBRegressor(booster='gbtree', objective ='reg:squarederror', seed=seed, colsample_bylevel=x[0], colsample_bytree=x[1], gamma=x[2], \\\n",
    "                                            learning_rate=x[3], max_delta_step=x[4], max_depth=x[5], min_child_weight=x[6], n_estimators=x[7], \\\n",
    "                                            reg_alpha=x[8], reg_lambda=x[9], subsample=x[10])\n",
    "\n",
    "    # train the model\n",
    "    X1_train_valid = np.concatenate( (X1_train, X1_valid), axis=0)\n",
    "    R1_train_valid = np.concatenate( (R1_train, R1_valid), axis=0)\n",
    "    model.fit(X1_train_valid, R1_train_valid)\n",
    "\n",
    "    # calculate residuals for the train data\n",
    "    R1_pred = model.predict(X1_train_valid)\n",
    "    residuals = R1_train_valid - R1_pred\n",
    "    resid=residuals[0]\n",
    "    am=arch_model(resid, vol=\"Garch\",p=1,o=0,q=1,dist=\"Normal\" )\n",
    "    res=am.fit(update_freq=5)\n",
    "    forecasts=res.forecast(reindex=False)\n",
    "    g=forecasts.variance.iloc[-3:]\n",
    "   \n",
    "    \n",
    "\n",
    "\n",
    "    # forecast for the test data\n",
    "    fore_casts = model.predict(X1_test)\n",
    "    # print('forecasts = \\n', forecasts)\n",
    "    forecast_tau = float(fore_casts[len(fore_casts) - 1])\n",
    "    \n",
    "    del model # delete the model\n",
    "\n",
    "    # output results\n",
    "    return forecast_tau, g.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f9d22b9c",
   "metadata": {
    "id": "f9d22b9c"
   },
   "outputs": [],
   "source": [
    "##### Implement Random Forest\n",
    "\n",
    "\n",
    "# Create a Random Forest model\n",
    "\n",
    "def create_RF_model(n_estimators=100,max_depth=3,bootstrap=True):\n",
    "\n",
    "  model=RandomForestRegressor(n_estimators=n_estimators, criterion='squared_error',max_depth=max_depth,bootstrap=bootstrap)\n",
    "  return model\n",
    "\n",
    "\n",
    "\n",
    "def randomforestf(R, X, tau: int):\n",
    "    assert (R.shape[0] == X.shape[0]), \"numbers of rows not match!\"\n",
    "    assert(tau > 0), \"the forecast horizon must be greater than zero!\"\n",
    "    \n",
    "    T = X.shape[0]\n",
    "    dim = X.shape[1]\n",
    "    # R = R.flatten()\n",
    "    # X = X.flatten() # flatten arrays\n",
    "\n",
    "    R1 = np.empty( shape = (0, 1) )\n",
    "    X1 = np.empty( shape = (0, dim) )\n",
    "    for t in np.arange(0, T):\n",
    "        if t < T-tau:\n",
    "            R1 = np.append(R1, R[t+tau].reshape(1, 1), axis = 0)\n",
    "        else:\n",
    "            R1 = np.append(R1, np.array([0]).reshape(1, 1), axis = 0)\n",
    "        X1 = np.append(X1, X[t, :].reshape(1, dim), axis = 0)\n",
    "\n",
    "    # studentize data\n",
    "    X1 = StandardScaler().fit_transform(X1)\n",
    "\n",
    "    # split the numpy array into train and test data\n",
    "    X1_train, X1_test = X1[0:(T-tau), :], X1[(T-tau):, :]\n",
    "    R1_train, R1_test = R1[0:(T-tau)].ravel(), R1[(T-tau):].ravel()\n",
    "\n",
    "    # Build a Random Forest model\n",
    "    model= create_RF_model()\n",
    "\n",
    "    # Define the grid search parameters\n",
    "    n_estimators=[200,300]\n",
    "    max_depth=[80,90,100]\n",
    "    max_features=[3,4]\n",
    "    min_samples_leaf=[3,4]\n",
    "    min_samples_split=[8,10]\n",
    "    bootstrap=[True,False]\n",
    "\n",
    "    param_grid=dict(n_estimators=n_estimators,max_depth=max_depth,max_features=max_features,min_samples_leaf=min_samples_leaf,bootstrap=bootstrap)\n",
    "\n",
    "    # Use time-series cross-validation\n",
    "    tscv=TimeSeriesSplit(n_splits=2, test_size=10)\n",
    "\n",
    "    # Perform grid search \n",
    "    model_cv=GridSearchCV(model,param_grid=param_grid,cv=tscv,refit=True,scoring= \"neg_mean_squared_error\", n_jobs=-1)\n",
    "    \n",
    "    # Cross-validate a model by using the grid search\n",
    "    model_cv.fit(X1_train,R1_train)\n",
    "    \n",
    "    # Random forest model with optimal parameters\n",
    "    model1=model_cv.best_estimator_\n",
    "        \n",
    "    # train the model\n",
    "    model1.fit(X1_train, R1_train)\n",
    "       \n",
    "    # calculate residuals for the train data\n",
    "    R1_pred = model1.predict(X1_train)\n",
    "    residuals = R1_train - R1_pred\n",
    "    am= arch_model(residuals, vol=\"Garch\",p=1,o=0,q=1,dist=\"Normal\")\n",
    "    res=am.fit(update_freq=5)\n",
    "    forecasts=res.forecast(reindex=False)\n",
    "    g=forecasts.variance.iloc[-3:]\n",
    "\n",
    "    \n",
    "\n",
    "\n",
    "    # forecast for the test data\n",
    "    fore_casts = model1.predict( X1_test.reshape(-1, dim) )\n",
    "    # print('forecasts = \\n', forecasts)\n",
    "    forecast_tau = float(fore_casts[len(fore_casts) - 1])\n",
    "    \n",
    "    del model # delete the model\n",
    "\n",
    "\n",
    "    # output results\n",
    "    return forecast_tau, g.values\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "274ac1cb",
   "metadata": {
    "id": "274ac1cb"
   },
   "outputs": [],
   "source": [
    "###### Calculate rolling-window OoS R^2 for 'one'-steps ahead forecasts using the sDOC + Garch(1,1) model\n",
    "\n",
    "def OoS_R_sq(sstart: int, df: pd.DataFrame, tau, wsize: int, T1: int,ylag: int, use_model)-> tuple:\n",
    "    \"\"\" 'df': a pandas dataframe, 'freq': a frequency of type 'string', e.g., '3M', '1D', '1Y', or '1Q' etc., 'tau': forecast horizon, 'wsize': window size,\n",
    "            'sstart': sub-sample starting point, 'T1': size of a sub-sample, 'ylag': AR lag of the dependent variable \"\"\"\n",
    "    assert (T1 > wsize+tau), \"size of a subsample must be much greater than the window size!\"\n",
    "    assert (sstart+T1 <= len(df) ), \"end point of the last subsample must be less than or equal to size of the dataframe!\"\n",
    "\n",
    "    df = df.iloc[sstart:(sstart+T1), :].copy()\n",
    "    # get data for the dependant variable and predictors\n",
    "    if ylag > 0:\n",
    "        for i in np.arange(1, ylag+1):\n",
    "            df[f'ylag{i}'] = df.iloc[:, 1].shift(i)\n",
    "        df.dropna(inplace = True)\n",
    "    # print( df.iloc[0, 0] )\n",
    "\n",
    "    R = np.array(df.values[:, 1], dtype='float64')\n",
    "    X = np.array(df.values[:, 2:], dtype='float64')\n",
    "    \n",
    "    dim = X.shape[1]\n",
    "    \n",
    "\n",
    "    rmse = 0\n",
    "    var = 0\n",
    "    mae = 0\n",
    "    #list_err = []\n",
    "    list_forecast=[]\n",
    "    list_rol_var_res=[]\n",
    "    observe_data=[]\n",
    "    list_rol_var=[]\n",
    "    list_rolavg=[]\n",
    "    for s in np.arange(T1-wsize-tau-ylag):\n",
    "        if use_model=='multivar_ML':\n",
    "            forecast,var_res= multivar_ML(R[s:(s+wsize+1)].reshape(-1,1), X[s:(s+wsize+1), :].reshape(-1,dim),tau)\n",
    "            \n",
    "        \n",
    "        else:\n",
    "            print(f'Model {use_model} does not exist!')\n",
    "            sys.exit()   \n",
    "        \n",
    "        \n",
    "        a=np.mean(R[s:(s+wsize)])\n",
    "        list_forecast.append(forecast)\n",
    "        list_rol_var_res.append(var_res)\n",
    "        observe_data.append(R[(s+wsize+1):(s+wsize+2)])\n",
    "        r = R[s+wsize+tau] # actual returns\n",
    "        var1= np.var(R[s:(s+wsize)])\n",
    "        list_rol_var.append(var1)\n",
    "        list_rolavg.append(a)\n",
    "        rmse += pow(r - forecast, 2) / (T1-wsize-tau)\n",
    "        var += pow(r- a , 2) / (T1-wsize-tau)\n",
    "        #mae += abs(r - forecast) / (T1-wsize-tau)\n",
    "    del df # delete this copy of the dataframe\n",
    "    return  float(1 - rmse/(var + 0.00001)),list_forecast,list_rol_var_res,observe_data,list_rol_var,list_rolavg\n",
    "   \n",
    "            \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ee9c06d",
   "metadata": {
    "id": "4ee9c06d"
   },
   "outputs": [],
   "source": [
    "file=pd.read_csv(all_files[1])\n",
    "file.shape\n",
    "\n",
    "#b=OoS_R_sq(sstart=0, df=file, tau=1, wsize=85, T1=167, ylag=0, use_model='multivar_ML')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76c5e802",
   "metadata": {
    "id": "76c5e802"
   },
   "outputs": [],
   "source": [
    "### Export rolling window forecasts, excess industry-level return from testing sample, rolling variance, rolling window average of excess industry-level return, exponential moving average\n",
    "\n",
    "\n",
    "a1=np.array(b[1])\n",
    "c1=np.mat(a1)\n",
    "d1=pd.DataFrame(c1)\n",
    "d1=d1.T\n",
    "d1=d1.fillna(0)\n",
    "\n",
    "a2=np.array(b[2])\n",
    "c2=np.mat(a2)\n",
    "d2=pd.DataFrame(c2)\n",
    "d2=d2.T\n",
    "d2=d2.fillna(0)\n",
    "\n",
    "a3=np.array(b[3])\n",
    "c3=np.mat(a3)\n",
    "d3=pd.DataFrame(c3)\n",
    "d3=d3.fillna(0)\n",
    "\n",
    "d4=pd.DataFrame(b[4])\n",
    "d4=d4.fillna(0)\n",
    "\n",
    "q= file['excess_log_return'].ewm(com=0.40).mean()\n",
    "q=pd.DataFrame(q)\n",
    "q=q.iloc[86:,]\n",
    "q=q.values\n",
    "q=pd.DataFrame(q)\n",
    "\n",
    "a5=np.array(b[5])\n",
    "c5=np.mat(a5)\n",
    "d5=pd.DataFrame(c5)\n",
    "d5=d5.T\n",
    "d5=d5.fillna(0)\n",
    "\n",
    "df=pd.concat([d1,d2,d3,d4,d5,q],axis=1)\n",
    "df.columns=['return_forecast','rol_var_res','observe_return','rolling_var','rol_avg','ewma']\n",
    "df.to_csv(r'C:/Users/endou012/emile/sDOC_result/file_63.csv' )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "9d3591f3",
   "metadata": {
    "executionInfo": {
     "elapsed": 1497,
     "status": "ok",
     "timestamp": 1655402881544,
     "user": {
      "displayName": "NDOUMBE Emile Herve",
      "userId": "09643065439420591964"
     },
     "user_tz": 240
    },
    "id": "9d3591f3"
   },
   "outputs": [],
   "source": [
    "##### Calculate rolling-window OoS R^2 for 'one'-steps ahead forecasts using random forest, neural network with 3 layers and principal components regression\n",
    "\n",
    "\n",
    "def OoS_R_sq(sstart: int, df: pd.DataFrame, tau, wsize: int, T1: int, num_PCs:int, ylag: int, use_model)-> tuple:\n",
    "    \"\"\" 'df': a pandas dataframe, 'freq': a frequency of type 'string', e.g., '3M', '1D', '1Y', or '1Q' etc., 'tau': forecast horizon, 'wsize': window size,\n",
    "            'sstart': sub-sample starting point, 'T1': size of a sub-sample, 'ylag': AR lag of the dependent variable \"\"\"\n",
    "    assert (T1 > wsize+tau), \"size of a subsample must be much greater than the window size!\"\n",
    "    assert (sstart+T1 <= len(df) ), \"end point of the last subsample must be less than or equal to size of the dataframe!\"\n",
    "\n",
    "    df = df.iloc[sstart:(sstart+T1), :].copy()\n",
    "    # get data for the dependant variable and predictors\n",
    "    if ylag > 0:\n",
    "        for i in np.arange(1, ylag+1):\n",
    "            df[f'ylag{i}'] = df.iloc[:, 1].shift(i)\n",
    "        df.dropna(inplace = True)\n",
    "    # print( df.iloc[0, 0] )\n",
    "\n",
    "    R = np.array(df.values[:, 1], dtype='float64')\n",
    "    X = np.array(df.values[:, 2:], dtype='float64')\n",
    "    \n",
    "    dim = X.shape[1]\n",
    "    global forecast\n",
    "    \n",
    "    rmse = 0\n",
    "    var = 0\n",
    "    #mae = 0\n",
    "    list_forecast=[]\n",
    "    list_rol_var_pred=[]\n",
    "    observe_data=[]\n",
    "    list_rol_var=[]\n",
    "    list_rolavg=[]\n",
    "    #list_err = []\n",
    "    for s in np.arange(T1-wsize-tau-ylag):\n",
    "    # estimate a long-run regression model and make a 'tau'-steps ahead forecast\n",
    "        \n",
    "        if use_model == 'multivar_lhOLSPC':\n",
    "          r_forecast, var_pred= multivar_lhOLSPC(R[s:(s+wsize+1)].reshape(-1,1),X[s:(s+wsize+1),:].reshape(-1,dim),num_PCs,tau)\n",
    "                                                                                                            \n",
    "        \n",
    "        elif use_model == 'ANNf':\n",
    "          r_forecast, var_pred= ANNf(R[s:(s+wsize+1)].reshape(-1,1), X[s:(s+wsize+1),:].reshape(-1,dim), tau, batch_size,num_epochs)\n",
    "\n",
    "                                                                                                                 \n",
    "        \n",
    "        elif use_model == 'xgboostf_cv':\n",
    "          r_forecast, var_pred = XGBoostf_CV(R[s:(s+wsize+1)].reshape(-1,1), X[s:(s+wsize+1),:].reshape(-1,dim),tau,n_calls=50)\n",
    "                                                                                                                          \n",
    "        elif use_model == 'randomforestf':\n",
    "            r_forecast, var_pred= randomforestf(R[s:(s+wsize+1)].reshape(-1, 1), X[s:(s+wsize+1), :].reshape(-1, dim), tau)\n",
    "        else:\n",
    "            print(f'Model {use_model} does not exist!')\n",
    "            sys.exit()\n",
    "\n",
    "        a=np.mean(R[s:(s+wsize)])\n",
    "        list_forecast.append(r_forecast)\n",
    "        list_rol_var_pred.append(var_pred)\n",
    "        observe_data.append(R[(s+wsize+1):(s+wsize+2)])\n",
    "        \n",
    "        r = R[s+wsize+tau] # actual returns\n",
    "        var1=np.var(R[s:(s+wsize)])\n",
    "        list_rol_var.append(var1)\n",
    "        list_rolavg.append(a)\n",
    "\n",
    "        rmse +=  pow(r - r_forecast, 2) / (T1-wsize-tau)\n",
    "        var += pow(r - a, 2) / (T1-wsize-tau)\n",
    "    \n",
    "    \n",
    "    del df # delete this copy of the dataframe\n",
    "    return  float(1 - rmse/(var + 0.00001) ),list_forecast,list_rol_var_pred,observe_data,list_rol_var,list_rolavg\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "52ca0692",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2906,
     "status": "ok",
     "timestamp": 1655403316441,
     "user": {
      "displayName": "NDOUMBE Emile Herve",
      "userId": "09643065439420591964"
     },
     "user_tz": 240
    },
    "id": "52ca0692",
    "outputId": "03698292-f4fd-4abc-b9e8-2d0fbc77fd0c"
   },
   "outputs": [],
   "source": [
    "fil=pd.read_csv(files[1])\n",
    "fil.shape\n",
    "#b=OoS_R_sq(sstart=0,df=fil,tau=1,wsize=85,T1=167, num_PCs=10, ylag=1,use_model='multivar_lhOLSPC')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "id": "4a20d8f2",
   "metadata": {
    "executionInfo": {
     "elapsed": 1229,
     "status": "ok",
     "timestamp": 1655403355693,
     "user": {
      "displayName": "NDOUMBE Emile Herve",
      "userId": "09643065439420591964"
     },
     "user_tz": 240
    },
    "id": "4a20d8f2"
   },
   "outputs": [],
   "source": [
    "### Export rolling window forecasts, excess industry-level return from testing sample, rolling variance, rolling window average of excess industry-level return, exponential moving average\n",
    "\n",
    "\n",
    "a1=np.array(b[1])\n",
    "c1=np.mat(a1)\n",
    "d1=pd.DataFrame(c1)\n",
    "d1=d1.T\n",
    "d1=d1.fillna(0)\n",
    "\n",
    "a2=np.array(b[2])\n",
    "c2=np.mat(a2)\n",
    "d2=pd.DataFrame(c2)\n",
    "d2=d2.T\n",
    "d2=d2.fillna(0)\n",
    "\n",
    "a3=np.array(b[3])\n",
    "c3=np.mat(a3)\n",
    "d3=pd.DataFrame(c3)\n",
    "d3=d3.fillna(0)\n",
    "\n",
    "d4=pd.DataFrame(b[4])\n",
    "d4=d4.fillna(0)\n",
    "\n",
    "a5=np.array(b[5])\n",
    "c5=np.mat(a5)\n",
    "d5=pd.DataFrame(c5)\n",
    "d5=d5.T\n",
    "d5=d5.fillna(0)\n",
    "\n",
    "q=fil['excess_log_return'].ewm(com=0.40).mean()\n",
    "q=pd.DataFrame(q)\n",
    "q=q.iloc[86:]\n",
    "q=q.values\n",
    "q=pd.DataFrame(q)\n",
    "mer=fil.loc[:,\"date\"][86:]\n",
    "mi=mer.values\n",
    "dmer=pd.DataFrame(mi)\n",
    "\n",
    "\n",
    "\n",
    "df=pd.concat([dmer,d1,d2,d3,d4,d5,q],axis=1)\n",
    "df.columns=['date','return_forecast','rol_var_pred','observe_return','rolling_var','rol_avg','ewm']\n",
    "df.to_csv('/content/gdrive/MyDrive/PCRresult/file_40.csv')\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "name": "forecast_est_monthly3.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}