Project_code.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "T8-MpNASDEG5",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -yjnius (/home/leczhang/.local/lib/python3.9/site-packages)\u001b[0m\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -ightgbm (/home/leczhang/.local/lib/python3.9/site-packages)\u001b[0m\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -yjnius (/home/leczhang/.local/lib/python3.9/site-packages)\u001b[0m\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -ightgbm (/home/leczhang/.local/lib/python3.9/site-packages)\u001b[0m\n",
      "Collecting python-terrier\n",
      "  Cloning https://github.com/terrier-org/pyterrier.git to /tmp/pip-install-t93923cq/python-terrier_a2998679b250463186ab97291c573a2b\n",
      "  Running command git clone -q https://github.com/terrier-org/pyterrier.git /tmp/pip-install-t93923cq/python-terrier_a2998679b250463186ab97291c573a2b\n",
      "  Resolved https://github.com/terrier-org/pyterrier.git to commit 134d80342a746c438c4deef5d30f1d1462ebab7a\n",
      "Requirement already satisfied: numpy in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from python-terrier) (1.20.3)\n",
      "Requirement already satisfied: pandas in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from python-terrier) (1.3.4)\n",
      "Requirement already satisfied: wget in /home/leczhang/.local/lib/python3.9/site-packages (from python-terrier) (3.2)\n",
      "Requirement already satisfied: tqdm in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from python-terrier) (4.62.3)\n",
      "Requirement already satisfied: pyjnius>=1.4.2 in /home/leczhang/.local/lib/python3.9/site-packages (from python-terrier) (1.4.2)\n",
      "Requirement already satisfied: matchpy in /home/leczhang/.local/lib/python3.9/site-packages (from python-terrier) (0.5.5)\n",
      "Requirement already satisfied: scikit-learn in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from python-terrier) (0.24.2)\n",
      "Requirement already satisfied: deprecated in /home/leczhang/.local/lib/python3.9/site-packages (from python-terrier) (1.2.13)\n",
      "Requirement already satisfied: chest in /home/leczhang/.local/lib/python3.9/site-packages (from python-terrier) (0.2.3)\n",
      "Requirement already satisfied: scipy in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from python-terrier) (1.7.1)\n",
      "Requirement already satisfied: requests in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from python-terrier) (2.26.0)\n",
      "Requirement already satisfied: joblib in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from python-terrier) (1.1.0)\n",
      "Requirement already satisfied: nptyping==1.4.4 in /home/leczhang/.local/lib/python3.9/site-packages (from python-terrier) (1.4.4)\n",
      "Requirement already satisfied: more_itertools in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from python-terrier) (8.10.0)\n",
      "Requirement already satisfied: ir_datasets>=0.3.2 in /home/leczhang/.local/lib/python3.9/site-packages (from python-terrier) (0.5.4)\n",
      "Requirement already satisfied: jinja2 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from python-terrier) (3.1.2)\n",
      "Requirement already satisfied: statsmodels in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from python-terrier) (0.12.2)\n",
      "Requirement already satisfied: ir_measures>=0.3.1 in /home/leczhang/.local/lib/python3.9/site-packages (from python-terrier) (0.3.1)\n",
      "Requirement already satisfied: dill in /home/leczhang/.local/lib/python3.9/site-packages (from python-terrier) (0.3.6)\n",
      "Requirement already satisfied: pytrec_eval_terrier>=0.5.3 in /home/leczhang/.local/lib/python3.9/site-packages (from python-terrier) (0.5.5)\n",
      "Requirement already satisfied: typish>=1.7.0 in /home/leczhang/.local/lib/python3.9/site-packages (from nptyping==1.4.4->python-terrier) (1.9.3)\n",
      "Requirement already satisfied: ijson>=3.1.3 in /home/leczhang/.local/lib/python3.9/site-packages (from ir_datasets>=0.3.2->python-terrier) (3.1.4)\n",
      "Requirement already satisfied: unlzw3>=0.2.1 in /home/leczhang/.local/lib/python3.9/site-packages (from ir_datasets>=0.3.2->python-terrier) (0.2.1)\n",
      "Requirement already satisfied: lz4>=3.1.1 in /home/leczhang/.local/lib/python3.9/site-packages (from ir_datasets>=0.3.2->python-terrier) (4.0.2)\n",
      "Requirement already satisfied: trec-car-tools>=2.5.4 in /home/leczhang/.local/lib/python3.9/site-packages (from ir_datasets>=0.3.2->python-terrier) (2.6)\n",
      "Requirement already satisfied: lxml>=4.5.2 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from ir_datasets>=0.3.2->python-terrier) (4.6.3)\n",
      "Requirement already satisfied: pyyaml>=5.3.1 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from ir_datasets>=0.3.2->python-terrier) (6.0)\n",
      "Requirement already satisfied: beautifulsoup4>=4.4.1 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from ir_datasets>=0.3.2->python-terrier) (4.10.0)\n",
      "Requirement already satisfied: pyautocorpus>=0.1.1 in /home/leczhang/.local/lib/python3.9/site-packages (from ir_datasets>=0.3.2->python-terrier) (0.1.9)\n",
      "Requirement already satisfied: zlib-state>=0.1.3 in /home/leczhang/.local/lib/python3.9/site-packages (from ir_datasets>=0.3.2->python-terrier) (0.1.5)\n",
      "Requirement already satisfied: warc3-wet-clueweb09>=0.2.5 in /home/leczhang/.local/lib/python3.9/site-packages (from ir_datasets>=0.3.2->python-terrier) (0.2.5)\n",
      "Requirement already satisfied: warc3-wet>=0.2.3 in /home/leczhang/.local/lib/python3.9/site-packages (from ir_datasets>=0.3.2->python-terrier) (0.2.3)\n",
      "Requirement already satisfied: soupsieve>1.2 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from beautifulsoup4>=4.4.1->ir_datasets>=0.3.2->python-terrier) (2.2.1)\n",
      "Requirement already satisfied: cwl-eval>=1.0.10 in /home/leczhang/.local/lib/python3.9/site-packages (from ir_measures>=0.3.1->python-terrier) (1.0.12)\n",
      "Requirement already satisfied: six>=1.7.0 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from pyjnius>=1.4.2->python-terrier) (1.16.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from requests->python-terrier) (3.2)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from requests->python-terrier) (1.26.7)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from requests->python-terrier) (2.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from requests->python-terrier) (2021.10.8)\n",
      "Requirement already satisfied: cbor>=1.0.0 in /home/leczhang/.local/lib/python3.9/site-packages (from trec-car-tools>=2.5.4->ir_datasets>=0.3.2->python-terrier) (1.0.0)\n",
      "Requirement already satisfied: heapdict in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from chest->python-terrier) (1.0.1)\n",
      "Requirement already satisfied: wrapt<2,>=1.10 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from deprecated->python-terrier) (1.12.1)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from jinja2->python-terrier) (2.1.1)\n",
      "Requirement already satisfied: multiset<3.0,>=2.0 in /home/leczhang/.local/lib/python3.9/site-packages (from matchpy->python-terrier) (2.1.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from pandas->python-terrier) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2017.3 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from pandas->python-terrier) (2021.3)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from scikit-learn->python-terrier) (2.2.0)\n",
      "Requirement already satisfied: patsy>=0.5 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from statsmodels->python-terrier) (0.5.2)\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -yjnius (/home/leczhang/.local/lib/python3.9/site-packages)\u001b[0m\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -ightgbm (/home/leczhang/.local/lib/python3.9/site-packages)\u001b[0m\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -yjnius (/home/leczhang/.local/lib/python3.9/site-packages)\u001b[0m\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -ightgbm (/home/leczhang/.local/lib/python3.9/site-packages)\u001b[0m\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -yjnius (/home/leczhang/.local/lib/python3.9/site-packages)\u001b[0m\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -ightgbm (/home/leczhang/.local/lib/python3.9/site-packages)\u001b[0m\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -yjnius (/home/leczhang/.local/lib/python3.9/site-packages)\u001b[0m\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -ightgbm (/home/leczhang/.local/lib/python3.9/site-packages)\u001b[0m\n",
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -yjnius (/home/leczhang/.local/lib/python3.9/site-packages)\u001b[0m\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -ightgbm (/home/leczhang/.local/lib/python3.9/site-packages)\u001b[0m\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -yjnius (/home/leczhang/.local/lib/python3.9/site-packages)\u001b[0m\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -ightgbm (/home/leczhang/.local/lib/python3.9/site-packages)\u001b[0m\n",
      "Collecting git+https://github.com/Georgetown-IR-Lab/OpenNIR\n",
      "  Cloning https://github.com/Georgetown-IR-Lab/OpenNIR to /tmp/pip-req-build-0gz4_b41\n",
      "  Running command git clone -q https://github.com/Georgetown-IR-Lab/OpenNIR /tmp/pip-req-build-0gz4_b41\n",
      "  Resolved https://github.com/Georgetown-IR-Lab/OpenNIR to commit 88a4679372f471a04d284a99404ffce2b7a1dc49\n",
      "Requirement already satisfied: torch>=1.3.1 in /home/leczhang/.local/lib/python3.9/site-packages (from OpenNIR==0.1.0) (1.13.1)\n",
      "Requirement already satisfied: pytorch-pretrained-bert==0.6.1 in /home/leczhang/.local/lib/python3.9/site-packages (from OpenNIR==0.1.0) (0.6.1)\n",
      "Requirement already satisfied: pytorch-transformers==1.1.0 in /home/leczhang/.local/lib/python3.9/site-packages (from OpenNIR==0.1.0) (1.1.0)\n",
      "Requirement already satisfied: tokenizers>=0.1.1 in /home/leczhang/.local/lib/python3.9/site-packages (from OpenNIR==0.1.0) (0.13.1)\n",
      "Requirement already satisfied: transformers>=4.3.3 in /home/leczhang/.local/lib/python3.9/site-packages (from OpenNIR==0.1.0) (4.23.1)\n",
      "Requirement already satisfied: tqdm>=4.38.0 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from OpenNIR==0.1.0) (4.62.3)\n",
      "Requirement already satisfied: colorlog==4.0.2 in /home/leczhang/.local/lib/python3.9/site-packages (from OpenNIR==0.1.0) (4.0.2)\n",
      "Requirement already satisfied: terminaltables>=3.1.0 in /home/leczhang/.local/lib/python3.9/site-packages (from OpenNIR==0.1.0) (3.1.10)\n",
      "Requirement already satisfied: matplotlib>=3.0.2 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from OpenNIR==0.1.0) (3.4.3)\n",
      "Requirement already satisfied: seaborn>=0.9.0 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from OpenNIR==0.1.0) (0.11.2)\n",
      "Requirement already satisfied: python-ternary>=1.0.6 in /home/leczhang/.local/lib/python3.9/site-packages (from OpenNIR==0.1.0) (1.0.8)\n",
      "Requirement already satisfied: beautifulsoup4>=4.9.3 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from OpenNIR==0.1.0) (4.10.0)\n",
      "Requirement already satisfied: html5lib>=1.0.1 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from OpenNIR==0.1.0) (1.1)\n",
      "Requirement already satisfied: Unidecode>=1.0.22 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from OpenNIR==0.1.0) (1.2.0)\n",
      "Requirement already satisfied: nltk>=3.4.5 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from OpenNIR==0.1.0) (3.6.5)\n",
      "Requirement already satisfied: sqlitedict>=1.6.0 in /home/leczhang/.local/lib/python3.9/site-packages (from OpenNIR==0.1.0) (2.0.0)\n",
      "Requirement already satisfied: pytrec-eval-terrier>=0.5.1 in /home/leczhang/.local/lib/python3.9/site-packages (from OpenNIR==0.1.0) (0.5.5)\n",
      "Requirement already satisfied: gensim>=3.7.3 in /home/leczhang/.local/lib/python3.9/site-packages (from OpenNIR==0.1.0) (4.2.0)\n",
      "Requirement already satisfied: Cython>=0.29.2 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from OpenNIR==0.1.0) (0.29.24)\n",
      "Requirement already satisfied: pyjnius>=1.2.1 in /home/leczhang/.local/lib/python3.9/site-packages (from OpenNIR==0.1.0) (1.4.2)\n",
      "Requirement already satisfied: ir_datasets>=0.2.0 in /home/leczhang/.local/lib/python3.9/site-packages (from OpenNIR==0.1.0) (0.5.4)\n",
      "Requirement already satisfied: ir_measures>=0.2.1 in /home/leczhang/.local/lib/python3.9/site-packages (from OpenNIR==0.1.0) (0.3.1)\n",
      "Requirement already satisfied: pytools>=2018.5.2 in /home/leczhang/.local/lib/python3.9/site-packages (from OpenNIR==0.1.0) (2022.1.12)\n",
      "Requirement already satisfied: cached-property>=1.5.1 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from OpenNIR==0.1.0) (1.5.2)\n",
      "Requirement already satisfied: regex in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from pytorch-pretrained-bert==0.6.1->OpenNIR==0.1.0) (2021.8.3)\n",
      "Requirement already satisfied: numpy in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from pytorch-pretrained-bert==0.6.1->OpenNIR==0.1.0) (1.20.3)\n",
      "Requirement already satisfied: requests in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from pytorch-pretrained-bert==0.6.1->OpenNIR==0.1.0) (2.26.0)\n",
      "Requirement already satisfied: boto3 in /home/leczhang/.local/lib/python3.9/site-packages (from pytorch-pretrained-bert==0.6.1->OpenNIR==0.1.0) (1.25.4)\n",
      "Requirement already satisfied: sentencepiece in /home/leczhang/.local/lib/python3.9/site-packages (from pytorch-transformers==1.1.0->OpenNIR==0.1.0) (0.1.97)\n",
      "Requirement already satisfied: soupsieve>1.2 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from beautifulsoup4>=4.9.3->OpenNIR==0.1.0) (2.2.1)\n",
      "Requirement already satisfied: smart-open>=1.8.1 in /home/leczhang/.local/lib/python3.9/site-packages (from gensim>=3.7.3->OpenNIR==0.1.0) (6.2.0)\n",
      "Requirement already satisfied: scipy>=0.18.1 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from gensim>=3.7.3->OpenNIR==0.1.0) (1.7.1)\n",
      "Requirement already satisfied: six>=1.9 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from html5lib>=1.0.1->OpenNIR==0.1.0) (1.16.0)\n",
      "Requirement already satisfied: webencodings in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from html5lib>=1.0.1->OpenNIR==0.1.0) (0.5.1)\n",
      "Requirement already satisfied: unlzw3>=0.2.1 in /home/leczhang/.local/lib/python3.9/site-packages (from ir_datasets>=0.2.0->OpenNIR==0.1.0) (0.2.1)\n",
      "Requirement already satisfied: pyautocorpus>=0.1.1 in /home/leczhang/.local/lib/python3.9/site-packages (from ir_datasets>=0.2.0->OpenNIR==0.1.0) (0.1.9)\n",
      "Requirement already satisfied: trec-car-tools>=2.5.4 in /home/leczhang/.local/lib/python3.9/site-packages (from ir_datasets>=0.2.0->OpenNIR==0.1.0) (2.6)\n",
      "Requirement already satisfied: lz4>=3.1.1 in /home/leczhang/.local/lib/python3.9/site-packages (from ir_datasets>=0.2.0->OpenNIR==0.1.0) (4.0.2)\n",
      "Requirement already satisfied: lxml>=4.5.2 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from ir_datasets>=0.2.0->OpenNIR==0.1.0) (4.6.3)\n",
      "Requirement already satisfied: ijson>=3.1.3 in /home/leczhang/.local/lib/python3.9/site-packages (from ir_datasets>=0.2.0->OpenNIR==0.1.0) (3.1.4)\n",
      "Requirement already satisfied: zlib-state>=0.1.3 in /home/leczhang/.local/lib/python3.9/site-packages (from ir_datasets>=0.2.0->OpenNIR==0.1.0) (0.1.5)\n",
      "Requirement already satisfied: warc3-wet>=0.2.3 in /home/leczhang/.local/lib/python3.9/site-packages (from ir_datasets>=0.2.0->OpenNIR==0.1.0) (0.2.3)\n",
      "Requirement already satisfied: pyyaml>=5.3.1 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from ir_datasets>=0.2.0->OpenNIR==0.1.0) (6.0)\n",
      "Requirement already satisfied: warc3-wet-clueweb09>=0.2.5 in /home/leczhang/.local/lib/python3.9/site-packages (from ir_datasets>=0.2.0->OpenNIR==0.1.0) (0.2.5)\n",
      "Requirement already satisfied: cwl-eval>=1.0.10 in /home/leczhang/.local/lib/python3.9/site-packages (from ir_measures>=0.2.1->OpenNIR==0.1.0) (1.0.12)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from matplotlib>=3.0.2->OpenNIR==0.1.0) (1.3.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from matplotlib>=3.0.2->OpenNIR==0.1.0) (0.10.0)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from matplotlib>=3.0.2->OpenNIR==0.1.0) (8.4.0)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from matplotlib>=3.0.2->OpenNIR==0.1.0) (3.0.4)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from matplotlib>=3.0.2->OpenNIR==0.1.0) (2.8.2)\n",
      "Requirement already satisfied: click in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from nltk>=3.4.5->OpenNIR==0.1.0) (8.0.3)\n",
      "Requirement already satisfied: joblib in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from nltk>=3.4.5->OpenNIR==0.1.0) (1.1.0)\n",
      "Requirement already satisfied: typing-extensions>=4.0 in /home/leczhang/.local/lib/python3.9/site-packages (from pytools>=2018.5.2->OpenNIR==0.1.0) (4.4.0)\n",
      "Requirement already satisfied: platformdirs>=2.2.0 in /home/leczhang/.local/lib/python3.9/site-packages (from pytools>=2018.5.2->OpenNIR==0.1.0) (2.5.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from requests->pytorch-pretrained-bert==0.6.1->OpenNIR==0.1.0) (3.2)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from requests->pytorch-pretrained-bert==0.6.1->OpenNIR==0.1.0) (2021.10.8)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from requests->pytorch-pretrained-bert==0.6.1->OpenNIR==0.1.0) (1.26.7)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from requests->pytorch-pretrained-bert==0.6.1->OpenNIR==0.1.0) (2.0.4)\n",
      "Requirement already satisfied: pandas>=0.23 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from seaborn>=0.9.0->OpenNIR==0.1.0) (1.3.4)\n",
      "Requirement already satisfied: pytz>=2017.3 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from pandas>=0.23->seaborn>=0.9.0->OpenNIR==0.1.0) (2021.3)\n",
      "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in /home/leczhang/.local/lib/python3.9/site-packages (from torch>=1.3.1->OpenNIR==0.1.0) (8.5.0.96)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in /home/leczhang/.local/lib/python3.9/site-packages (from torch>=1.3.1->OpenNIR==0.1.0) (11.7.99)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in /home/leczhang/.local/lib/python3.9/site-packages (from torch>=1.3.1->OpenNIR==0.1.0) (11.7.99)\n",
      "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in /home/leczhang/.local/lib/python3.9/site-packages (from torch>=1.3.1->OpenNIR==0.1.0) (11.10.3.66)\n",
      "Requirement already satisfied: setuptools in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.3.1->OpenNIR==0.1.0) (58.0.4)\n",
      "Requirement already satisfied: wheel in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.3.1->OpenNIR==0.1.0) (0.37.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.10.0 in /home/leczhang/.local/lib/python3.9/site-packages (from transformers>=4.3.3->OpenNIR==0.1.0) (0.10.1)\n",
      "Requirement already satisfied: packaging>=20.0 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from transformers>=4.3.3->OpenNIR==0.1.0) (21.0)\n",
      "Requirement already satisfied: filelock in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from transformers>=4.3.3->OpenNIR==0.1.0) (3.3.1)\n",
      "Requirement already satisfied: cbor>=1.0.0 in /home/leczhang/.local/lib/python3.9/site-packages (from trec-car-tools>=2.5.4->ir_datasets>=0.2.0->OpenNIR==0.1.0) (1.0.0)\n",
      "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /home/leczhang/.local/lib/python3.9/site-packages (from boto3->pytorch-pretrained-bert==0.6.1->OpenNIR==0.1.0) (0.6.0)\n",
      "Requirement already satisfied: botocore<1.29.0,>=1.28.4 in /home/leczhang/.local/lib/python3.9/site-packages (from boto3->pytorch-pretrained-bert==0.6.1->OpenNIR==0.1.0) (1.28.4)\n",
      "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /home/leczhang/.local/lib/python3.9/site-packages (from boto3->pytorch-pretrained-bert==0.6.1->OpenNIR==0.1.0) (1.0.1)\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -yjnius (/home/leczhang/.local/lib/python3.9/site-packages)\u001b[0m\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -ightgbm (/home/leczhang/.local/lib/python3.9/site-packages)\u001b[0m\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -yjnius (/home/leczhang/.local/lib/python3.9/site-packages)\u001b[0m\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -ightgbm (/home/leczhang/.local/lib/python3.9/site-packages)\u001b[0m\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -yjnius (/home/leczhang/.local/lib/python3.9/site-packages)\u001b[0m\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -ightgbm (/home/leczhang/.local/lib/python3.9/site-packages)\u001b[0m\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -yjnius (/home/leczhang/.local/lib/python3.9/site-packages)\u001b[0m\n",
      "\u001b[33mWARNING: Ignoring invalid distribution -ightgbm (/home/leczhang/.local/lib/python3.9/site-packages)\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install --upgrade git+https://github.com/terrier-org/pyterrier.git#egg=python-terrier\n",
    "!pip install --upgrade git+https://github.com/Georgetown-IR-Lab/OpenNIR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "PQH-L3ka2oMq"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTerrier 0.9.1 has loaded Terrier 5.7 (built by craigm on 2022-11-10 18:30) and terrier-helper 0.0.7\n",
      "\n",
      "No etc/terrier.properties, using terrier.default.properties for bootstrap configuration.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"JAVA_HOME\"] = \"/sw/pkgs/arc/openjdk/jdk-18.0.1.1\"\n",
    "os.environ['JVM_PATH']='/sw/pkgs/arc/openjdk/jdk-18.0.1.1/lib/server/libjvm.so'\n",
    "\n",
    "\n",
    "import pyterrier as pt\n",
    "if not pt.started():\n",
    "    pt.init(tqdm='notebook')\n",
    "\n",
    "import pandas as pd\n",
    "import onir_pt\n",
    "import numpy as np\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = pd.read_csv('./dataset_full.csv')\n",
    "query = pd.read_csv('./query.csv')\n",
    "qrels = pd.read_csv('./qrels.csv')\n",
    "\n",
    "\n",
    "query['qid']=query['qid'].astype('str')\n",
    "qrels['qid']=qrels['qid'].astype('str')\n",
    "\n",
    "\n",
    "combined_list = []\n",
    "for i in range(len(dataset)):\n",
    "    combined_list.append(dataset['Description'][i] + dataset['pos_review_content'][i])\n",
    "dataset['combined'] = combined_list\n",
    "dataset['revised_title']=dataset['pos_review_title'].apply(lambda row: row[0:int(len(row)/2)])\n",
    "dataset = dataset.fillna('None')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "sWWWT9Jf4c0q"
   },
   "outputs": [],
   "source": [
    "!rm -rf ./proj_file\n",
    "index_path = './proj_file'\n",
    "\n",
    "indexer = pt.DFIndexer(index_path, overwrite=True, blocks=True)\n",
    "index_ref = indexer.index(dataset['combined'], dataset['docno'])\n",
    "index = pt.IndexFactory.of(index_ref)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of documents: 13571\n",
      "Number of terms: 65000\n",
      "Number of postings: 4111997\n",
      "Number of fields: 0\n",
      "Number of tokens: 7330675\n",
      "Field names: []\n",
      "Positions:   true\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(index.getCollectionStatistics().toString())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "QGOqtrauGpQs"
   },
   "source": [
    "# Query Expansion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "5QQ14zxfBOCC"
   },
   "outputs": [],
   "source": [
    "bm25 = pt.BatchRetrieve(index,wmodel=\"BM25\")\n",
    "qe = pt.rewrite.Bo1QueryExpansion(index)\n",
    "sdm = pt.rewrite.SequentialDependence()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>AP(rel=3)</th>\n",
       "      <th>P(rel=3)@50</th>\n",
       "      <th>ndcg</th>\n",
       "      <th>ndcg_cut_5</th>\n",
       "      <th>ndcg_cut_10</th>\n",
       "      <th>mrt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BM25</td>\n",
       "      <td>0.505849</td>\n",
       "      <td>0.570</td>\n",
       "      <td>0.740277</td>\n",
       "      <td>0.667947</td>\n",
       "      <td>0.684460</td>\n",
       "      <td>26.543634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Bo1 QE</td>\n",
       "      <td>0.539878</td>\n",
       "      <td>0.600</td>\n",
       "      <td>0.763332</td>\n",
       "      <td>0.680896</td>\n",
       "      <td>0.695181</td>\n",
       "      <td>60.029948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>SDM</td>\n",
       "      <td>0.512819</td>\n",
       "      <td>0.568</td>\n",
       "      <td>0.742006</td>\n",
       "      <td>0.692826</td>\n",
       "      <td>0.699737</td>\n",
       "      <td>40.336547</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     name  AP(rel=3)  P(rel=3)@50      ndcg  ndcg_cut_5  ndcg_cut_10  \\\n",
       "0    BM25   0.505849        0.570  0.740277    0.667947     0.684460   \n",
       "1  Bo1 QE   0.539878        0.600  0.763332    0.680896     0.695181   \n",
       "2     SDM   0.512819        0.568  0.742006    0.692826     0.699737   \n",
       "\n",
       "         mrt  \n",
       "0  26.543634  \n",
       "1  60.029948  \n",
       "2  40.336547  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pyterrier.measures import *\n",
    "pt.Experiment(\n",
    "    [bm25%50, (bm25>>qe>>bm25)%50, (sdm>>bm25)%50],\n",
    "    query,\n",
    "    qrels,\n",
    "    names=['BM25', 'Bo1 QE','SDM'],\n",
    "    eval_metrics=[AP(rel=3), P(rel=3)@50, \"ndcg\", \"ndcg_cut_5\", \"ndcg_cut_10\", 'mrt']\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Learning to rank\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since 60% train data, 20% valid data, 20% test data is reasonable. The total topics is 40.\n",
    "Therefore, we set Train set to be 24,  Test and Valid to be 8."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "SEED=13\n",
    "from sklearn.model_selection import train_test_split\n",
    "tr_va_data,test_data=train_test_split(query,test_size=8,random_state=SEED)\n",
    "train_data,valid_data=train_test_split(tr_va_data,test_size=8,random_state=SEED)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 1 Pair-wise BERT reranker (Store into checkpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gc, torch\n",
    "gc.collect()\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "config file not found: config\n"
     ]
    }
   ],
   "source": [
    "VBERT_CACHE_NAME = 'trained_bert55.tar.gz'\n",
    "\n",
    "if not os.path.exists(VBERT_CACHE_NAME):\n",
    "    vbert = onir_pt.reranker('vanilla_transformer', 'bert', text_field='revised_title', vocab_config={'train': True})\n",
    "    pipeline1 = (pt.BatchRetrieve(index, wmodel=\"BM25\") %55) >> pt.apply.generic(lambda df: df.merge(dataset,on='docno')) >> vbert\n",
    "    pipeline1.fit(train_data, qrels, valid_data, qrels)\n",
    "    onir_pt.reranker.to_checkpoint(vbert, VBERT_CACHE_NAME) #Save trained BERT into file\n",
    "\n",
    "else:\n",
    "    vbert=onir_pt.reranker.from_checkpoint(VBERT_CACHE_NAME, text_field='revised_title')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-12-17 21:24:33,374][onir_pt][DEBUG] using GPU (deterministic)\n",
      "[2022-12-17 21:24:43,612][onir_pt][DEBUG] [starting] batches\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "batches:   0%|          | 0/100 [17ms<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-12-17 21:25:21,910][onir_pt][DEBUG] [finished] batches: [38.30s] [100it] [ 2.61it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>ndcg</th>\n",
       "      <th>ndcg_cut_5</th>\n",
       "      <th>ndcg_cut_10</th>\n",
       "      <th>AP(rel=3)</th>\n",
       "      <th>P(rel=3)@10</th>\n",
       "      <th>mrt</th>\n",
       "      <th>ndcg +</th>\n",
       "      <th>ndcg -</th>\n",
       "      <th>ndcg p-value</th>\n",
       "      <th>...</th>\n",
       "      <th>ndcg_cut_5 p-value</th>\n",
       "      <th>ndcg_cut_10 +</th>\n",
       "      <th>ndcg_cut_10 -</th>\n",
       "      <th>ndcg_cut_10 p-value</th>\n",
       "      <th>AP(rel=3) +</th>\n",
       "      <th>AP(rel=3) -</th>\n",
       "      <th>AP(rel=3) p-value</th>\n",
       "      <th>P(rel=3)@10 +</th>\n",
       "      <th>P(rel=3)@10 -</th>\n",
       "      <th>P(rel=3)@10 p-value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BM25</td>\n",
       "      <td>0.778395</td>\n",
       "      <td>0.658551</td>\n",
       "      <td>0.698940</td>\n",
       "      <td>0.527218</td>\n",
       "      <td>0.6250</td>\n",
       "      <td>23.860467</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BM25 &gt;&gt; VBERT</td>\n",
       "      <td>0.744747</td>\n",
       "      <td>0.576042</td>\n",
       "      <td>0.611716</td>\n",
       "      <td>0.464210</td>\n",
       "      <td>0.5375</td>\n",
       "      <td>6222.305281</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.157504</td>\n",
       "      <td>...</td>\n",
       "      <td>0.372333</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.275448</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.214372</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.421693</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            name      ndcg  ndcg_cut_5  ndcg_cut_10  AP(rel=3)  P(rel=3)@10  \\\n",
       "0           BM25  0.778395    0.658551     0.698940   0.527218       0.6250   \n",
       "1  BM25 >> VBERT  0.744747    0.576042     0.611716   0.464210       0.5375   \n",
       "\n",
       "           mrt  ndcg +  ndcg -  ndcg p-value  ...  ndcg_cut_5 p-value  \\\n",
       "0    23.860467     NaN     NaN           NaN  ...                 NaN   \n",
       "1  6222.305281     3.0     5.0      0.157504  ...            0.372333   \n",
       "\n",
       "   ndcg_cut_10 +  ndcg_cut_10 -  ndcg_cut_10 p-value  AP(rel=3) +  \\\n",
       "0            NaN            NaN                  NaN          NaN   \n",
       "1            3.0            5.0             0.275448          2.0   \n",
       "\n",
       "   AP(rel=3) -  AP(rel=3) p-value  P(rel=3)@10 +  P(rel=3)@10 -  \\\n",
       "0          NaN                NaN            NaN            NaN   \n",
       "1          6.0           0.214372            2.0            4.0   \n",
       "\n",
       "   P(rel=3)@10 p-value  \n",
       "0                  NaN  \n",
       "1             0.421693  \n",
       "\n",
       "[2 rows x 22 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pyterrier.measures import *\n",
    "pt.Experiment(\n",
    "    [(bm25%50), ((bm25%50) >> pt.apply.generic(lambda df: df.merge(dataset,on='docno')) >> vbert)%50],\n",
    "    test_data,\n",
    "    qrels,\n",
    "    names=['BM25', 'BM25 >> VBERT'],\n",
    "    baseline=0,\n",
    "    eval_metrics=[AP(rel=3), P(rel=3)@10, \"ndcg\", \"ndcg_cut_5\", \"ndcg_cut_10\", 'mrt']\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 2: Sentiment analysis on negative review (Store into cache)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "SENTIMENT_CACHE_NAME = 'dataset_sentiment.json'\n",
    "\n",
    "if not os.path.exists(SENTIMENT_CACHE_NAME):\n",
    "    import nltk\n",
    "    from nltk.sentiment import SentimentIntensityAnalyzer\n",
    "    nltk.download([\n",
    "            \"names\",\n",
    "            \"stopwords\",\n",
    "            \"averaged_perceptron_tagger\",\n",
    "            \"vader_lexicon\",\n",
    "            \"punkt\",\n",
    "    ])\n",
    "    sia = SentimentIntensityAnalyzer()\n",
    "    senti = dataset['neg_review_content'].apply(lambda x: sia.polarity_scores(x)['neg']).tolist()\n",
    "    fw = open(SENTIMENT_CACHE_NAME,\"w\")\n",
    "    fw.write(json.dumps(senti))\n",
    "    fw.close() \n",
    "else:\n",
    "    cache_file = open(SENTIMENT_CACHE_NAME, 'r')\n",
    "    cache_content = cache_file.read()\n",
    "    senti = json.loads(cache_content)\n",
    "    cache_file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 3: Sentence Transformer model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer, util\n",
    "model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 4: Feature analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ltr_feats1: Only 5 basic feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------\n",
      "Training starts...\n",
      "---------------------------\n",
      "[+] Random restart #1/5...\n",
      "[+] Random restart #3/5...\n",
      "[+] Random restart #2/5...\n",
      "[+] Random restart #4/5...\n",
      "[+] Random restart #5/5...\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   0|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   2|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "Shuffle features and optimize!\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   3|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   4|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "----------------------------------------\n",
      "   1|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   0|4               |    0.000|    0.745\n",
      "   2|4               |    0.000|    0.792\n",
      "   4|4               |    0.000|    0.758\n",
      "   1|4               |    0.000|    0.775\n",
      "   0|1               |    0.000|    0.758\n",
      "   2|2               |    0.000|    0.794\n",
      "   4|3               |    0.000|    0.771\n",
      "   1|3               |    0.000|    0.777\n",
      "   2|1               |    0.000|    0.797\n",
      "   1|0               |    0.000|    0.777\n",
      "   2|0               |    0.778|    0.797\n",
      "   2|0               |    0.678|    0.798\n",
      "   2|0               |    0.478|    0.800\n",
      "   1|0               |    0.011|    0.778\n",
      "   2|0               |    0.078|    0.816\n",
      "   3|0               | -409.660|    0.737\n",
      "   3|0               | -819.260|    0.738\n",
      "   3|0               |-1638.460|    0.742\n",
      "   3|0               |-3276.860|    0.746\n",
      "   3|0               |-6553.660|    0.749\n",
      "   3|0               |-13107.260|    0.750\n",
      "   3|0               |-26214.460|    0.750\n",
      "   4|0               |    0.373|    0.775\n",
      "   4|0               |    0.473|    0.776\n",
      "   0|3               |    0.307|    0.759\n",
      "   4|0               |    0.673|    0.781\n",
      "   0|3               |    0.407|    0.761\n",
      "   4|0               |    1.073|    0.786\n",
      "   0|3               |    0.607|    0.768\n",
      "   4|0               |    1.873|    0.789\n",
      "   0|3               |    1.007|    0.775\n",
      "   4|0               |    3.473|    0.793\n",
      "   0|3               |    1.807|    0.780\n",
      "   4|0               |    6.673|    0.795\n",
      "   0|3               |    3.407|    0.782\n",
      "   3|3               |   -0.001|    0.750\n",
      "   0|3               |    6.607|    0.793\n",
      "   3|3               |   -0.002|    0.750\n",
      "   0|3               |   13.007|    0.802\n",
      "   0|3               |  409.807|    0.802\n",
      "   2|3               |    0.737|    0.816\n",
      "   4|1               |   -0.048|    0.795\n",
      "   2|3               |    0.837|    0.820\n",
      "   2|3               |    1.037|    0.823\n",
      "   1|1               |    0.336|    0.781\n",
      "   2|3               |    1.437|    0.823\n",
      "   1|1               |    0.536|    0.795\n",
      "   2|3               |    2.237|    0.835\n",
      "   1|1               |    0.936|    0.806\n",
      "   1|1               |    1.736|    0.813\n",
      "   3|3               |    0.002|    0.750\n",
      "   1|1               |    3.336|    0.818\n",
      "   3|3               |    0.004|    0.750\n",
      "   3|3               |    0.008|    0.750\n",
      "   3|3               |    0.017|    0.751\n",
      "   3|3               |    0.033|    0.752\n",
      "   3|3               |    0.067|    0.753\n",
      "   3|3               |    0.133|    0.755\n",
      "   3|3               |    0.266|    0.767\n",
      "   3|3               |    0.533|    0.770\n",
      "   3|3               |    1.066|    0.774\n",
      "   3|3               |    2.132|    0.787\n",
      "   3|3               |    4.263|    0.789\n",
      "   3|3               |    8.526|    0.792\n",
      "   3|3               |   17.053|    0.793\n",
      "   3|4               |    0.000|    0.794\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   2|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   4|1               |    0.145|    0.796\n",
      "   0|2               |    0.001|    0.802\n",
      "   4|1               |    0.319|    0.797\n",
      "   0|2               |    0.001|    0.802\n",
      "   4|1               |    0.668|    0.800\n",
      "   0|2               |    0.001|    0.803\n",
      "   4|1               |    1.364|    0.805\n",
      "   0|2               |    0.002|    0.803\n",
      "   4|1               |    2.756|    0.808\n",
      "   0|2               |    0.003|    0.805\n",
      "   0|2               |    0.005|    0.806\n",
      "   4|1               |   11.109|    0.811\n",
      "   0|2               |    0.008|    0.809\n",
      "   4|1               |   22.246|    0.812\n",
      "   0|2               |    0.015|    0.810\n",
      "   4|1               |   44.521|    0.812\n",
      "   4|1               |   89.070|    0.813\n",
      "   4|1               |  356.368|    0.813\n",
      "   4|1               |  712.764|    0.813\n",
      "   4|2               |    0.000|    0.813\n",
      "   3|4               |    0.000|    0.794\n",
      "   3|4               |    0.000|    0.794\n",
      "   4|2               |   -0.000|    0.813\n",
      "   3|4               |    0.000|    0.797\n",
      "   4|2               |   -0.001|    0.813\n",
      "   3|4               |    0.000|    0.798\n",
      "   4|2               |   -0.001|    0.813\n",
      "   0|0               |   -0.001|    0.810\n",
      "   3|4               |    0.000|    0.803\n",
      "   4|2               |   -0.002|    0.814\n",
      "   3|4               |    0.000|    0.804\n",
      "   3|4               |    0.000|    0.809\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   1|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   4|2               |    0.009|    0.814\n",
      "   0|0               |    0.000|    0.811\n",
      "   4|2               |    0.019|    0.815\n",
      "   0|0               |    0.001|    0.812\n",
      "   0|0               |    0.004|    0.814\n",
      "   0|0               |    0.008|    0.814\n",
      "   4|2               |    0.149|    0.815\n",
      "   0|0               |    0.017|    0.819\n",
      "   4|2               |    0.298|    0.817\n",
      "   0|0               |    0.035|    0.825\n",
      "   0|0               |    0.070|    0.827\n",
      "   0|0               |    0.140|    0.829\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   4|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   0|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   3|2               |    0.000|    0.810\n",
      "   3|2               |    0.001|    0.810\n",
      "   3|2               |    0.004|    0.810\n",
      "   1|0               |    0.001|    0.818\n",
      "   1|0               |   -0.006|    0.819\n",
      "   1|0               |   -0.031|    0.819\n",
      "   1|0               |   -0.269|    0.824\n",
      "   4|3               |    0.000|    0.817\n",
      "   0|2               |    0.000|    0.835\n",
      "   3|1               |    0.000|    0.810\n",
      "   3|1               |    0.000|    0.810\n",
      "   3|1               |    0.001|    0.810\n",
      "   3|1               |    0.001|    0.811\n",
      "   3|1               |    0.011|    0.811\n",
      "   3|1               |    0.181|    0.831\n",
      "   3|1               |    0.363|    0.846\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   3|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   4|3               |    0.050|    0.818\n",
      "   3|0               |   -0.043|    0.846\n",
      "   4|3               |    0.150|    0.821\n",
      "   4|3               |    0.350|    0.827\n",
      "   4|3               |    0.750|    0.831\n",
      "   4|3               |    1.550|    0.836\n",
      "   4|2               |    0.076|    0.836\n",
      "   4|2               |    0.058|    0.836\n",
      "   1|1               |    0.585|    0.825\n",
      "   3|2               |   -0.002|    0.846\n",
      "   3|2               |   -0.033|    0.846\n",
      "   1|3               |    0.050|    0.827\n",
      "   1|3               |    0.150|    0.835\n",
      "   1|3               |    0.350|    0.841\n",
      "   1|3               |    0.750|    0.843\n",
      "   4|1               |    0.362|    0.836\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   1|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   0|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   4|0               |    0.000|    0.837\n",
      "   4|0               |   -0.019|    0.837\n",
      "   4|0               |   -0.077|    0.843\n",
      "   3|4               |    0.000|    0.847\n",
      "   3|4               |    0.000|    0.849\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   4|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   4|0               |   -0.075|    0.843\n",
      "   4|0               |   -0.097|    0.845\n",
      "   4|2               |    0.000|    0.845\n",
      "   4|2               |    0.049|    0.845\n",
      "   4|2               |    0.044|    0.846\n",
      "   4|2               |    0.013|    0.846\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   3|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   3|4               |    0.000|    0.849\n",
      "   1|2               |    0.087|    0.844\n",
      "   1|2               |    0.078|    0.844\n",
      "   1|2               |    0.060|    0.845\n",
      "   4|3               |    0.514|    0.848\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   1|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   4|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   4|4               |    0.000|    0.848\n",
      "   1|3               |    0.493|    0.845\n",
      "   1|2               |    0.056|    0.845\n",
      "   1|2               |    0.050|    0.845\n",
      "   1|2               |    0.079|    0.846\n",
      "---------------------------\n",
      "Finished successfully.\n",
      "CPU times: user 5.75 s, sys: 406 ms, total: 6.15 s\n",
      "Wall time: 2.24 s\n"
     ]
    }
   ],
   "source": [
    "ltr_feats1 = ((bm25>>qe>>bm25)%50) >>   pt.apply.generic(lambda df: df.merge(dataset,on='docno'))>> (\n",
    "        pt.transformer.IdentityTransformer()\n",
    "        ** # new feature: TF-IDF indicator\n",
    "        pt.BatchRetrieve(index, wmodel=\"TF_IDF\")\n",
    "        ** #new feature: review number\n",
    "        (pt.apply.doc_score(lambda row: math.log(row['review_num'])))\n",
    "        ** #new feature: overall rate\n",
    "        (pt.apply.doc_score(lambda row: row['overall_rate']))\n",
    "        **# new feature: positive review title length\n",
    "        (pt.apply.doc_score(lambda row: len(row['pos_review_title']) ))\n",
    "    )\n",
    "\n",
    "import fastrank\n",
    "\n",
    "qrels['label'] = qrels['label'].apply(lambda x: float(x))\n",
    "train_request = fastrank.TrainRequest.coordinate_ascent()\n",
    "\n",
    "params = train_request.params\n",
    "params.init_random = True\n",
    "params.normalize = True\n",
    "params.seed = 1234567\n",
    "\n",
    "ca_pipe1 = ltr_feats1 >> pt.ltr.apply_learned_model(train_request, form='fastrank')\n",
    "\n",
    "%time ca_pipe1.fit(train_data, qrels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ltr_feats2: ltr_feats1 + VBERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-12-17 21:25:27,718][onir_pt][DEBUG] using GPU (deterministic)\n",
      "[2022-12-17 21:25:27,722][onir_pt][DEBUG] [starting] batches\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "batches:   0%|          | 0/300 [14ms<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-12-17 21:25:35,709][onir_pt][DEBUG] [finished] batches: [7.99s] [300it] [37.57it/s]\n",
      "---------------------------\n",
      "Training starts...\n",
      "---------------------------\n",
      "[+] Random restart #1/5...\n",
      "[+] Random restart #3/5...\n",
      "[+] Random restart #2/5...\n",
      "[+] Random restart #4/5...\n",
      "[+] Random restart #5/5...\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   2|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   3|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   1|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   0|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   4|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   2|5               |    0.000|    0.733\n",
      "   1|5               |    0.000|    0.728\n",
      "   0|0               |    0.000|    0.732\n",
      "   3|5               |    0.166|    0.733\n",
      "   0|2               |    0.000|    0.733\n",
      "   0|2               |   -0.116|    0.733\n",
      "   0|2               |   -0.516|    0.733\n",
      "   0|2               |   -1.316|    0.735\n",
      "   0|2               |   -2.916|    0.736\n",
      "   0|2               |   -6.116|    0.742\n",
      "   0|2               |  -12.516|    0.744\n",
      "   4|5               |   -3.206|    0.733\n",
      "   0|2               |  -25.316|    0.750\n",
      "   0|2               |  -50.916|    0.754\n",
      "   0|2               | -102.116|    0.758\n",
      "   0|2               | -204.516|    0.760\n",
      "   2|5               |-1638.548|    0.733\n",
      "   0|2               | -409.316|    0.761\n",
      "   0|2               | -818.916|    0.762\n",
      "   0|2               |-1638.116|    0.763\n",
      "   4|5               | -418.949|    0.733\n",
      "   0|2               |-3276.516|    0.764\n",
      "   0|2               |-6553.316|    0.764\n",
      "   1|5               |-52428.962|    0.728\n",
      "   4|5               |-3352.061|    0.733\n",
      "   0|2               |-13106.916|    0.764\n",
      "   2|5               |-104857.748|    0.733\n",
      "   4|5               |-6704.188|    0.733\n",
      "   0|2               |-26214.116|    0.764\n",
      "   2|5               |-209715.348|    0.733\n",
      "   1|5               |-209715.362|    0.728\n",
      "   4|5               |-13408.443|    0.733\n",
      "   2|5               |-419430.548|    0.733\n",
      "   1|5               |-419430.562|    0.729\n",
      "   1|5               |-838860.962|    0.729\n",
      "   1|5               |-1677721.762|    0.729\n",
      "   0|3               |    0.000|    0.764\n",
      "   0|3               |   -0.000|    0.764\n",
      "   0|3               |   -0.000|    0.764\n",
      "   3|5               |  409.766|    0.733\n",
      "   4|5               |  838.093|    0.733\n",
      "   1|0               |   -0.000|    0.729\n",
      "   3|5               |52428.966|    0.733\n",
      "   4|5               | 3352.188|    0.733\n",
      "   1|0               |   -0.000|    0.736\n",
      "   3|5               |104857.766|    0.733\n",
      "   1|0               |   -0.001|    0.738\n",
      "   4|5               |13408.571|    0.733\n",
      "   1|0               |   -0.001|    0.742\n",
      "   4|5               |26817.082|    0.733\n",
      "   1|0               |   -0.002|    0.746\n",
      "   3|5               |838860.966|    0.735\n",
      "   1|0               |   -0.004|    0.749\n",
      "   3|5               |1677721.766|    0.736\n",
      "   2|5               |1677721.352|    0.734\n",
      "   1|0               |   -0.009|    0.750\n",
      "   2|4               |    0.000|    0.788\n",
      "   4|3               |    0.000|    0.734\n",
      "   3|0               |   -0.000|    0.736\n",
      "   1|2               |   -0.000|    0.750\n",
      "   1|2               |   -0.000|    0.750\n",
      "   1|2               |   -0.000|    0.750\n",
      "   1|2               |   -0.000|    0.751\n",
      "   1|2               |   -0.000|    0.752\n",
      "   3|0               |   -0.000|    0.738\n",
      "   1|2               |   -0.003|    0.753\n",
      "   3|0               |   -0.001|    0.742\n",
      "   1|2               |   -0.005|    0.761\n",
      "   3|0               |   -0.001|    0.746\n",
      "   1|2               |   -0.010|    0.762\n",
      "   3|0               |   -0.003|    0.749\n",
      "   3|0               |   -0.005|    0.750\n",
      "   3|0               |   -0.011|    0.750\n",
      "   0|3               |    0.132|    0.768\n",
      "   0|3               |    0.265|    0.770\n",
      "   0|3               |    0.529|    0.777\n",
      "   0|3               |    1.058|    0.778\n",
      "   2|0               |    0.000|    0.788\n",
      "   0|3               |    2.116|    0.781\n",
      "   2|0               |    0.000|    0.789\n",
      "   3|3               |    0.000|    0.750\n",
      "   0|3               |    4.232|    0.784\n",
      "   2|0               |    0.000|    0.791\n",
      "   3|3               |    0.000|    0.750\n",
      "   0|3               |    8.465|    0.789\n",
      "   2|0               |    0.000|    0.793\n",
      "   2|0               |    0.000|    0.796\n",
      "   4|3               |    0.000|    0.734\n",
      "   3|3               |   -0.000|    0.750\n",
      "   4|3               |    0.000|    0.735\n",
      "   3|3               |   -0.000|    0.750\n",
      "   4|3               |    0.000|    0.743\n",
      "   4|3               |    0.001|    0.756\n",
      "   1|1               |   -0.000|    0.762\n",
      "   4|3               |    0.002|    0.764\n",
      "   1|1               |   -0.000|    0.762\n",
      "   4|3               |    0.003|    0.767\n",
      "   4|3               |    0.007|    0.772\n",
      "   4|3               |    0.014|    0.780\n",
      "   4|3               |    0.028|    0.788\n",
      "   4|3               |    0.055|    0.789\n",
      "   4|3               |    0.110|    0.790\n",
      "   2|3               |    0.000|    0.796\n",
      "   3|3               |    0.000|    0.751\n",
      "   3|3               |    0.000|    0.751\n",
      "   0|1               |    0.000|    0.792\n",
      "   3|3               |    0.000|    0.751\n",
      "   0|1               |    0.000|    0.793\n",
      "   1|1               |    0.000|    0.762\n",
      "   3|3               |    0.000|    0.752\n",
      "   3|3               |    0.000|    0.752\n",
      "   0|1               |    0.000|    0.793\n",
      "   3|3               |    0.001|    0.753\n",
      "   0|1               |    0.000|    0.793\n",
      "   1|1               |    0.000|    0.762\n",
      "   3|3               |    0.002|    0.757\n",
      "   0|1               |    0.000|    0.795\n",
      "   1|1               |    0.000|    0.763\n",
      "   3|3               |    0.003|    0.773\n",
      "   3|3               |    0.006|    0.774\n",
      "   3|3               |    0.013|    0.778\n",
      "   3|3               |    0.026|    0.787\n",
      "   3|3               |    0.052|    0.790\n",
      "   3|3               |    0.103|    0.792\n",
      "   3|3               |    0.207|    0.793\n",
      "   3|4               |    0.000|    0.794\n",
      "   0|1               |    0.012|    0.795\n",
      "   1|1               |    0.009|    0.766\n",
      "   0|1               |    0.024|    0.798\n",
      "   1|1               |    0.019|    0.777\n",
      "   0|1               |    0.049|    0.802\n",
      "   2|3               |    0.000|    0.796\n",
      "   1|1               |    0.037|    0.803\n",
      "   3|4               |   -0.000|    0.794\n",
      "   0|1               |    0.097|    0.807\n",
      "   2|3               |    0.000|    0.796\n",
      "   1|1               |    0.074|    0.806\n",
      "   0|1               |    0.195|    0.820\n",
      "   2|3               |    0.000|    0.796\n",
      "   2|3               |    0.000|    0.797\n",
      "   0|1               |    0.389|    0.828\n",
      "   1|3               |   -0.000|    0.806\n",
      "   2|3               |    0.000|    0.797\n",
      "   4|0               |    0.000|    0.790\n",
      "   2|3               |    0.000|    0.799\n",
      "   3|4               |   -0.000|    0.794\n",
      "   4|0               |    0.000|    0.790\n",
      "   2|3               |    0.000|    0.799\n",
      "   0|5               |    0.000|    0.828\n",
      "   4|0               |    0.000|    0.790\n",
      "   2|3               |    0.000|    0.802\n",
      "   4|0               |    0.000|    0.791\n",
      "   2|3               |    0.000|    0.807\n",
      "   4|0               |    0.000|    0.794\n",
      "   2|3               |    0.000|    0.811\n",
      "   4|0               |    0.000|    0.796\n",
      "   2|3               |    0.000|    0.827\n",
      "   4|0               |    0.000|    0.800\n",
      "   4|0               |    0.000|    0.809\n",
      "   4|0               |    0.001|    0.814\n",
      "   4|0               |    0.001|    0.818\n",
      "   4|0               |    0.003|    0.823\n",
      "   4|0               |    0.005|    0.824\n",
      "   1|3               |   -0.000|    0.806\n",
      "   4|0               |    0.010|    0.828\n",
      "   0|5               |   -0.000|    0.828\n",
      "   2|2               |    0.000|    0.827\n",
      "   4|4               |    0.000|    0.832\n",
      "   3|4               |    0.000|    0.794\n",
      "   4|2               |    0.000|    0.832\n",
      "   3|4               |    0.000|    0.794\n",
      "   3|4               |    0.000|    0.794\n",
      "   3|4               |    0.000|    0.797\n",
      "   3|4               |    0.000|    0.799\n",
      "   3|4               |    0.000|    0.804\n",
      "   3|4               |    0.000|    0.804\n",
      "   3|4               |    0.000|    0.809\n",
      "   4|2               |   -0.000|    0.832\n",
      "   4|2               |   -0.000|    0.832\n",
      "   1|3               |    0.000|    0.806\n",
      "   1|3               |    0.000|    0.806\n",
      "   1|3               |    0.000|    0.806\n",
      "   1|3               |    0.000|    0.806\n",
      "   1|3               |    0.001|    0.806\n",
      "   1|3               |    0.002|    0.810\n",
      "   1|3               |    0.004|    0.811\n",
      "   1|3               |    0.007|    0.813\n",
      "   1|3               |    0.015|    0.814\n",
      "   3|2               |    0.000|    0.809\n",
      "   1|3               |    0.030|    0.822\n",
      "   3|2               |   -0.000|    0.809\n",
      "   1|3               |    0.060|    0.830\n",
      "   1|4               |    0.000|    0.830\n",
      "   2|2               |    0.000|    0.827\n",
      "   0|4               |    0.000|    0.828\n",
      "   3|2               |   -0.000|    0.810\n",
      "   2|2               |    0.000|    0.827\n",
      "   3|2               |   -0.000|    0.810\n",
      "   4|2               |    0.000|    0.832\n",
      "   4|2               |    0.000|    0.832\n",
      "   4|2               |    0.000|    0.832\n",
      "   4|2               |    0.001|    0.835\n",
      "   4|2               |    0.002|    0.836\n",
      "   0|4               |   -0.000|    0.828\n",
      "   0|4               |   -0.000|    0.829\n",
      "   2|1               |   -0.000|    0.827\n",
      "   1|4               |   -0.000|    0.830\n",
      "   1|4               |   -0.000|    0.830\n",
      "   4|1               |   -0.000|    0.836\n",
      "   1|4               |    0.000|    0.830\n",
      "   1|4               |    0.000|    0.831\n",
      "   1|4               |    0.000|    0.831\n",
      "   1|4               |    0.000|    0.832\n",
      "   0|4               |    0.000|    0.829\n",
      "   1|4               |    0.000|    0.835\n",
      "   0|4               |    0.000|    0.829\n",
      "   1|4               |    0.000|    0.837\n",
      "   3|2               |    0.000|    0.810\n",
      "   0|4               |    0.000|    0.830\n",
      "   3|2               |    0.000|    0.810\n",
      "   0|4               |    0.000|    0.832\n",
      "   0|4               |    0.000|    0.835\n",
      "   3|2               |    0.001|    0.810\n",
      "   3|2               |    0.001|    0.811\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   1|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   1|5               |    0.000|    0.837\n",
      "   3|1               |   -0.000|    0.811\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   0|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   1|5               |   -1.012|    0.837\n",
      "   2|1               |    0.000|    0.827\n",
      "   3|1               |   -0.000|    0.811\n",
      "   2|1               |    0.000|    0.829\n",
      "   2|1               |    0.000|    0.831\n",
      "   4|1               |    0.000|    0.836\n",
      "   0|5               |   -0.000|    0.835\n",
      "   4|1               |    0.000|    0.837\n",
      "   1|5               |  -26.412|    0.837\n",
      "   4|1               |    0.000|    0.837\n",
      "   1|5               | -103.212|    0.837\n",
      "   4|1               |    0.000|    0.837\n",
      "   1|5               | -205.612|    0.837\n",
      "   4|1               |    0.000|    0.837\n",
      "   1|5               | -410.412|    0.837\n",
      "   4|1               |    0.001|    0.838\n",
      "   4|1               |    0.002|    0.838\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   2|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   2|5               |    0.000|    0.833\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   4|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   3|1               |    0.000|    0.811\n",
      "   3|1               |    0.000|    0.811\n",
      "   3|1               |    0.016|    0.816\n",
      "   1|5               |13106.288|    0.840\n",
      "   3|1               |    0.033|    0.828\n",
      "   3|1               |    0.066|    0.843\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   3|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   0|5               |   15.870|    0.835\n",
      "   0|5               |  126.963|    0.835\n",
      "   3|5               |    0.418|    0.843\n",
      "   3|5               |   -0.782|    0.843\n",
      "   3|5               |   -2.382|    0.843\n",
      "   3|5               |   -5.582|    0.843\n",
      "   3|5               | -101.582|    0.843\n",
      "   3|5               | -203.982|    0.843\n",
      "   3|5               | -408.782|    0.843\n",
      "   3|5               |-3275.982|    0.844\n",
      "   2|0               |    0.000|    0.834\n",
      "   2|0               |    0.065|    0.834\n",
      "   2|0               |    0.019|    0.837\n",
      "   1|3               |    0.000|    0.840\n",
      "   0|3               |    0.005|    0.836\n",
      "   1|3               |    0.000|    0.840\n",
      "   1|3               |    0.000|    0.841\n",
      "   1|3               |    0.000|    0.841\n",
      "   1|3               |    0.000|    0.845\n",
      "   2|3               |    0.632|    0.837\n",
      "   0|1               |    0.002|    0.836\n",
      "   0|4               |    0.000|    0.836\n",
      "   3|2               |    0.000|    0.846\n",
      "   0|4               |    0.000|    0.836\n",
      "   4|5               |    0.000|    0.838\n",
      "   2|2               |    0.013|    0.837\n",
      "   2|2               |    0.017|    0.837\n",
      "   4|5               |   -0.662|    0.838\n",
      "   4|5               |   -2.262|    0.838\n",
      "   4|5               |   -5.462|    0.838\n",
      "   4|5               |  -24.662|    0.838\n",
      "   0|2               |    0.000|    0.838\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   0|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   0|5               |    0.000|    0.838\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   2|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   0|5               |    0.642|    0.838\n",
      "   1|2               |   -0.000|    0.845\n",
      "   0|5               |   -5.358|    0.838\n",
      "   0|5               |  -11.758|    0.838\n",
      "   0|5               | -203.758|    0.839\n",
      "   2|5               | -102.350|    0.837\n",
      "   2|5               | -204.750|    0.837\n",
      "   2|5               |-1638.350|    0.837\n",
      "   4|5               | 1639.238|    0.838\n",
      "   4|5               | 3277.638|    0.838\n",
      "   2|5               |-6553.550|    0.838\n",
      "   0|5               |  103.342|    0.839\n",
      "   0|5               |  205.742|    0.841\n",
      "   0|5               |  820.142|    0.843\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   3|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   0|0               |    0.000|    0.843\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   1|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   1|5               |    0.850|    0.846\n",
      "   2|0               |    0.000|    0.838\n",
      "   1|4               |    0.000|    0.846\n",
      "   2|0               |   -0.000|    0.844\n",
      "   2|2               |    0.000|    0.844\n",
      "   0|1               |    0.000|    0.843\n",
      "   2|1               |    0.000|    0.844\n",
      "   2|1               |    0.000|    0.845\n",
      "   2|1               |    0.000|    0.847\n",
      "   2|3               |    0.000|    0.847\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   2|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   2|5               |   -1.350|    0.848\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   0|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   1|0               |   -0.000|    0.846\n",
      "   2|2               |    0.000|    0.848\n",
      "   2|2               |    0.000|    0.848\n",
      "   2|4               |    0.000|    0.848\n",
      "   0|5               |    0.950|    0.843\n",
      "---------------------------\n",
      "Finished successfully.\n",
      "CPU times: user 10.6 s, sys: 92.1 ms, total: 10.7 s\n",
      "Wall time: 9.89 s\n"
     ]
    }
   ],
   "source": [
    "ltr_feats2 = ((bm25>>qe>>bm25)%50) >>   pt.apply.generic(lambda df: df.merge(dataset,on='docno'))>> (\n",
    "        pt.transformer.IdentityTransformer()\n",
    "        ** # new feature: TF-IDF indicator\n",
    "        pt.BatchRetrieve(index, wmodel=\"TF_IDF\")\n",
    "        ** #new feature: review number\n",
    "        (pt.apply.doc_score(lambda row: math.log(row['review_num'])))\n",
    "        ** #new feature: overall rate\n",
    "        (pt.apply.doc_score(lambda row: row['overall_rate']))\n",
    "        **# new feature: positive review title length\n",
    "        (pt.apply.doc_score(lambda row: len(row['pos_review_title']) ))\n",
    "        **\n",
    "        (vbert)\n",
    "    )\n",
    "\n",
    "import fastrank\n",
    "\n",
    "qrels['label'] = qrels['label'].apply(lambda x: float(x))\n",
    "train_request = fastrank.TrainRequest.coordinate_ascent()\n",
    "\n",
    "params = train_request.params\n",
    "params.init_random = True\n",
    "params.normalize = True\n",
    "params.seed = 1234567\n",
    "\n",
    "ca_pipe2 = ltr_feats2 >> pt.ltr.apply_learned_model(train_request, form='fastrank')\n",
    "\n",
    "%time ca_pipe2.fit(train_data, qrels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ltr_feats3: ltr_feats1 + Sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------\n",
      "Training starts...\n",
      "---------------------------\n",
      "[+] Random restart #1/5...\n",
      "[+] Random restart #3/5...\n",
      "[+] Random restart #2/5...\n",
      "[+] Random restart #4/5...\n",
      "[+] Random restart #5/5...\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   0|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   1|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   3|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   4|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   2|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   0|0               |    0.000|    0.732\n",
      "   2|5               |    0.000|    0.733\n",
      "   4|5               |    0.054|    0.733\n",
      "   4|5               |    0.042|    0.733\n",
      "   1|5               |   -0.962|    0.732\n",
      "   2|5               |   -0.948|    0.733\n",
      "   1|5               |   -1.762|    0.732\n",
      "   2|5               |   -1.748|    0.733\n",
      "   4|5               |   -0.137|    0.733\n",
      "   3|5               | -204.534|    0.737\n",
      "   1|5               | -204.962|    0.733\n",
      "   0|0               | -204.878|    0.736\n",
      "   2|5               | -204.948|    0.734\n",
      "   3|5               | -409.334|    0.741\n",
      "   0|0               | -409.678|    0.738\n",
      "   1|5               | -409.762|    0.739\n",
      "   2|5               | -409.748|    0.740\n",
      "   0|0               | -819.278|    0.742\n",
      "   3|5               | -818.934|    0.744\n",
      "   1|5               | -819.362|    0.742\n",
      "   2|5               | -819.348|    0.743\n",
      "   0|0               |-1638.478|    0.746\n",
      "   3|5               |-1638.134|    0.746\n",
      "   4|5               | -104.687|    0.736\n",
      "   1|5               |-1638.562|    0.744\n",
      "   2|5               |-1638.548|    0.745\n",
      "   0|0               |-3276.878|    0.749\n",
      "   3|5               |-3276.534|    0.750\n",
      "   4|5               | -209.441|    0.741\n",
      "   1|5               |-3276.962|    0.747\n",
      "   2|5               |-3276.948|    0.749\n",
      "   0|0               |-6553.678|    0.750\n",
      "   3|5               |-6553.334|    0.752\n",
      "   4|5               | -418.949|    0.743\n",
      "   1|5               |-6553.762|    0.750\n",
      "   2|5               |-6553.748|    0.751\n",
      "   3|5               |-13106.934|    0.753\n",
      "   4|5               | -837.965|    0.745\n",
      "   1|5               |-13107.362|    0.752\n",
      "   2|5               |-13107.348|    0.753\n",
      "   3|5               |-26214.134|    0.754\n",
      "   4|5               |-1675.997|    0.749\n",
      "   1|5               |-26214.562|    0.752\n",
      "   2|5               |-26214.548|    0.753\n",
      "   3|5               |-52428.534|    0.754\n",
      "   4|5               |-3352.061|    0.751\n",
      "   1|5               |-52428.962|    0.753\n",
      "   2|5               |-52428.948|    0.754\n",
      "   3|5               |-104857.334|    0.754\n",
      "   4|5               |-6704.188|    0.753\n",
      "   1|5               |-104857.762|    0.753\n",
      "   2|5               |-104857.748|    0.754\n",
      "   3|5               |-209714.934|    0.755\n",
      "   4|5               |-13408.443|    0.753\n",
      "   1|5               |-209715.362|    0.753\n",
      "   2|5               |-209715.348|    0.754\n",
      "   3|5               |-419430.134|    0.755\n",
      "   4|5               |-26816.954|    0.754\n",
      "   1|5               |-419430.562|    0.753\n",
      "   2|5               |-419430.548|    0.754\n",
      "   3|5               |-838860.534|    0.755\n",
      "   0|2               |    0.000|    0.750\n",
      "   4|5               |-53633.975|    0.754\n",
      "   1|5               |-838860.962|    0.753\n",
      "   2|5               |-838860.948|    0.754\n",
      "   4|5               |-107268.017|    0.754\n",
      "   2|5               |-1677721.748|    0.754\n",
      "   1|5               |-1677721.762|    0.753\n",
      "   4|3               |    0.000|    0.754\n",
      "   2|4               |    0.000|    0.784\n",
      "   1|0               |   -0.000|    0.753\n",
      "   1|0               |   -0.000|    0.754\n",
      "   4|3               |   -0.000|    0.755\n",
      "   0|2               |   -0.013|    0.751\n",
      "   0|2               |   -0.025|    0.751\n",
      "   0|2               |   -0.051|    0.751\n",
      "   0|2               |   -0.102|    0.751\n",
      "   0|2               |   -0.204|    0.754\n",
      "   0|2               |   -0.408|    0.757\n",
      "   0|2               |   -0.817|    0.761\n",
      "   0|2               |   -1.633|    0.761\n",
      "   3|0               |   -0.001|    0.755\n",
      "   3|0               |   -0.001|    0.757\n",
      "   3|0               |   -0.003|    0.758\n",
      "   1|0               |   -0.001|    0.755\n",
      "   1|0               |   -0.001|    0.757\n",
      "   0|3               |    0.000|    0.761\n",
      "   2|0               |    0.000|    0.785\n",
      "   2|0               |    0.000|    0.785\n",
      "   0|3               |   -0.000|    0.761\n",
      "   2|0               |    0.000|    0.786\n",
      "   2|0               |    0.000|    0.787\n",
      "   2|0               |    0.000|    0.787\n",
      "   1|2               |   -0.000|    0.757\n",
      "   2|0               |    0.000|    0.788\n",
      "   1|2               |   -0.000|    0.757\n",
      "   0|3               |   -0.001|    0.761\n",
      "   4|3               |    0.000|    0.755\n",
      "   4|3               |    0.000|    0.757\n",
      "   4|3               |    0.000|    0.761\n",
      "   4|3               |    0.001|    0.762\n",
      "   3|3               |   -0.000|    0.760\n",
      "   4|3               |    0.002|    0.768\n",
      "   4|3               |    0.003|    0.775\n",
      "   4|3               |    0.007|    0.782\n",
      "   1|2               |   -0.001|    0.770\n",
      "   4|3               |    0.014|    0.784\n",
      "   4|3               |    0.028|    0.784\n",
      "   4|3               |    0.055|    0.785\n",
      "   1|2               |   -0.005|    0.770\n",
      "   4|3               |    0.110|    0.786\n",
      "   2|0               |    0.010|    0.797\n",
      "   4|3               |    0.883|    0.787\n",
      "   4|3               |    1.766|    0.788\n",
      "   4|3               |    3.531|    0.789\n",
      "   2|3               |    0.000|    0.797\n",
      "   4|0               |    0.000|    0.789\n",
      "   1|1               |    0.000|    0.770\n",
      "   4|0               |    0.000|    0.789\n",
      "   2|3               |    0.000|    0.797\n",
      "   0|3               |    0.005|    0.762\n",
      "   4|0               |   -0.000|    0.792\n",
      "   2|3               |   -0.000|    0.797\n",
      "   0|3               |    0.175|    0.768\n",
      "   4|0               |   -0.001|    0.795\n",
      "   0|3               |    0.350|    0.771\n",
      "   4|0               |   -0.001|    0.795\n",
      "   0|3               |    0.700|    0.775\n",
      "   0|3               |    1.401|    0.777\n",
      "   2|3               |   -0.001|    0.798\n",
      "   0|3               |    2.802|    0.780\n",
      "   0|3               |    5.603|    0.787\n",
      "   0|3               |   11.207|    0.790\n",
      "   3|4               |   -0.000|    0.760\n",
      "   4|4               |    0.000|    0.796\n",
      "   3|4               |    0.000|    0.760\n",
      "   3|4               |    0.000|    0.778\n",
      "   3|4               |    0.000|    0.790\n",
      "   0|1               |   -0.000|    0.790\n",
      "   3|4               |    0.000|    0.794\n",
      "   3|4               |    0.000|    0.795\n",
      "   3|4               |    0.000|    0.797\n",
      "   2|3               |    0.000|    0.798\n",
      "   2|3               |    0.000|    0.799\n",
      "   3|4               |    0.000|    0.799\n",
      "   2|3               |    0.000|    0.799\n",
      "   3|4               |    0.000|    0.801\n",
      "   1|1               |    0.000|    0.770\n",
      "   3|4               |    0.001|    0.802\n",
      "   1|1               |    0.000|    0.772\n",
      "   2|3               |    0.001|    0.799\n",
      "   3|4               |    0.002|    0.803\n",
      "   1|1               |    0.001|    0.773\n",
      "   2|3               |    0.001|    0.800\n",
      "   3|4               |    0.003|    0.805\n",
      "   2|3               |    0.002|    0.801\n",
      "   3|4               |    0.007|    0.806\n",
      "   2|3               |    0.004|    0.802\n",
      "   1|1               |    0.005|    0.774\n",
      "   2|3               |    0.008|    0.803\n",
      "   3|4               |    0.027|    0.807\n",
      "   1|1               |    0.009|    0.799\n",
      "   1|1               |    0.019|    0.807\n",
      "   2|3               |    0.033|    0.804\n",
      "   1|1               |    0.038|    0.813\n",
      "   3|4               |    0.217|    0.807\n",
      "   1|3               |    0.000|    0.813\n",
      "   1|3               |   -0.000|    0.813\n",
      "   4|4               |   -0.000|    0.797\n",
      "   0|1               |    0.000|    0.790\n",
      "   4|4               |    0.000|    0.799\n",
      "   2|2               |   -0.000|    0.804\n",
      "   0|1               |    0.000|    0.790\n",
      "   4|4               |    0.000|    0.805\n",
      "   4|4               |    0.000|    0.806\n",
      "   0|1               |    0.001|    0.790\n",
      "   4|4               |    0.000|    0.807\n",
      "   0|1               |    0.002|    0.790\n",
      "   0|1               |    0.003|    0.790\n",
      "   4|4               |    0.000|    0.809\n",
      "   0|1               |    0.012|    0.792\n",
      "   0|1               |    0.025|    0.793\n",
      "   1|3               |   -0.000|    0.813\n",
      "   0|1               |    0.050|    0.795\n",
      "   1|3               |   -0.000|    0.813\n",
      "   3|2               |   -0.000|    0.807\n",
      "   0|1               |    0.100|    0.811\n",
      "   4|4               |    0.000|    0.812\n",
      "   0|1               |    0.200|    0.830\n",
      "   3|2               |   -0.003|    0.807\n",
      "   0|1               |    0.400|    0.832\n",
      "   4|2               |    0.000|    0.812\n",
      "   2|2               |    0.000|    0.804\n",
      "   2|2               |    0.000|    0.804\n",
      "   0|5               |   -0.001|    0.832\n",
      "   2|2               |    0.000|    0.804\n",
      "   2|2               |    0.000|    0.804\n",
      "   4|2               |   -0.000|    0.812\n",
      "   2|2               |    0.000|    0.805\n",
      "   0|5               |   -0.079|    0.832\n",
      "   2|2               |    0.001|    0.808\n",
      "   1|3               |    0.001|    0.814\n",
      "   1|3               |    0.002|    0.814\n",
      "   2|2               |    0.002|    0.810\n",
      "   1|3               |    0.004|    0.814\n",
      "   1|3               |    0.008|    0.815\n",
      "   1|3               |    0.016|    0.817\n",
      "   1|4               |   -0.000|    0.817\n",
      "   1|4               |   -0.000|    0.817\n",
      "   2|1               |   -0.000|    0.810\n",
      "   1|4               |   -0.000|    0.817\n",
      "   2|1               |   -0.000|    0.810\n",
      "   2|1               |   -0.000|    0.810\n",
      "   3|1               |   -0.000|    0.807\n",
      "   2|1               |   -0.000|    0.811\n",
      "   0|5               |    0.010|    0.833\n",
      "   0|5               |    0.020|    0.833\n",
      "   4|2               |    0.000|    0.812\n",
      "   0|5               |    0.039|    0.834\n",
      "   4|2               |    0.000|    0.812\n",
      "   4|2               |    0.000|    0.812\n",
      "   0|5               |    0.157|    0.834\n",
      "   4|2               |    0.001|    0.813\n",
      "   0|5               |    0.314|    0.835\n",
      "   4|2               |    0.002|    0.813\n",
      "   0|4               |    0.000|    0.835\n",
      "   4|2               |    0.015|    0.813\n",
      "   1|4               |    0.000|    0.818\n",
      "   1|4               |    0.000|    0.819\n",
      "   3|1               |    0.001|    0.807\n",
      "   3|1               |    0.002|    0.807\n",
      "   3|1               |    0.004|    0.807\n",
      "   3|1               |    0.033|    0.807\n",
      "   3|1               |    0.066|    0.807\n",
      "   2|1               |    0.007|    0.811\n",
      "   3|1               |    0.132|    0.807\n",
      "   2|1               |    0.014|    0.816\n",
      "   3|1               |    0.263|    0.808\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   3|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   3|5               |   -0.697|    0.808\n",
      "   2|1               |    0.112|    0.818\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   2|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   2|5               |    0.000|    0.818\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   1|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   0|4               |    0.000|    0.835\n",
      "   1|5               |    0.000|    0.829\n",
      "   0|4               |    0.000|    0.836\n",
      "   0|4               |    0.000|    0.836\n",
      "   0|4               |    0.000|    0.836\n",
      "   0|4               |    0.000|    0.837\n",
      "   0|4               |    0.000|    0.843\n",
      "   0|4               |    0.000|    0.843\n",
      "   4|1               |    0.000|    0.814\n",
      "   4|1               |    0.001|    0.814\n",
      "   4|1               |    0.002|    0.814\n",
      "   4|1               |    0.003|    0.816\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   0|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   0|5               |    0.189|    0.845\n",
      "   4|1               |    0.025|    0.819\n",
      "   4|1               |    0.050|    0.823\n",
      "   4|1               |    0.099|    0.827\n",
      "   3|5               |    0.903|    0.808\n",
      "   4|1               |    0.199|    0.830\n",
      "   4|1               |    0.397|    0.834\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   4|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   4|0               |    0.000|    0.834\n",
      "   3|5               |   24.903|    0.811\n",
      "   1|4               |    0.000|    0.829\n",
      "   3|5               |   50.503|    0.811\n",
      "   3|5               |  101.703|    0.811\n",
      "   4|0               |   -0.002|    0.834\n",
      "   4|0               |   -0.002|    0.834\n",
      "   4|0               |   -0.004|    0.835\n",
      "   0|0               |   -0.020|    0.845\n",
      "   0|0               |   -0.024|    0.845\n",
      "   3|3               |   -0.000|    0.811\n",
      "   3|3               |   -0.000|    0.811\n",
      "   4|0               |    0.022|    0.835\n",
      "   4|2               |    0.000|    0.835\n",
      "   1|3               |    0.320|    0.832\n",
      "   1|3               |    0.420|    0.835\n",
      "   1|3               |    0.620|    0.836\n",
      "   1|3               |    1.020|    0.837\n",
      "   1|3               |    1.820|    0.842\n",
      "   3|3               |    0.000|    0.811\n",
      "   4|2               |   -0.006|    0.836\n",
      "   3|3               |    0.000|    0.811\n",
      "   4|2               |   -0.022|    0.836\n",
      "   3|3               |    0.068|    0.812\n",
      "   3|3               |    0.135|    0.813\n",
      "   3|3               |    0.271|    0.815\n",
      "   3|3               |    0.542|    0.819\n",
      "   3|1               |    0.000|    0.819\n",
      "   2|0               |    0.040|    0.819\n",
      "   2|0               |   -0.132|    0.823\n",
      "   2|0               |   -0.327|    0.834\n",
      "   0|3               |    0.572|    0.845\n",
      "   0|3               |    0.672|    0.847\n",
      "   1|0               |   -0.007|    0.842\n",
      "   3|1               |    0.003|    0.820\n",
      "   1|0               |   -0.006|    0.842\n",
      "   3|1               |    0.005|    0.820\n",
      "   3|1               |    0.010|    0.820\n",
      "   3|1               |    0.018|    0.820\n",
      "   3|1               |    0.035|    0.821\n",
      "   3|1               |    0.069|    0.822\n",
      "   3|1               |    0.137|    0.825\n",
      "   3|1               |    1.086|    0.827\n",
      "   3|1               |    2.170|    0.827\n",
      "   4|3               |    0.581|    0.837\n",
      "   4|3               |    0.681|    0.837\n",
      "   4|3               |    0.881|    0.837\n",
      "   3|2               |   -0.000|    0.827\n",
      "   3|2               |   -0.004|    0.827\n",
      "   3|2               |    0.000|    0.827\n",
      "   3|2               |    0.001|    0.827\n",
      "   3|2               |    0.004|    0.827\n",
      "   3|2               |    0.029|    0.828\n",
      "   3|2               |    0.117|    0.830\n",
      "   0|4               |    0.000|    0.849\n",
      "   2|3               |    0.210|    0.834\n",
      "   2|3               |    0.310|    0.839\n",
      "   1|2               |    0.000|    0.843\n",
      "   4|1               |    0.354|    0.837\n",
      "   3|0               |   -0.000|    0.830\n",
      "   3|0               |   -0.000|    0.830\n",
      "   3|0               |   -0.001|    0.830\n",
      "   3|0               |   -0.001|    0.831\n",
      "   3|0               |   -0.021|    0.831\n",
      "   3|0               |   -0.041|    0.833\n",
      "   2|2               |    0.000|    0.840\n",
      "   2|2               |    0.006|    0.841\n",
      "   1|2               |   -0.022|    0.843\n",
      "   4|5               |    0.000|    0.838\n",
      "   1|2               |   -0.008|    0.843\n",
      "   4|5               |   -0.111|    0.838\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   2|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   0|2               |   -0.024|    0.849\n",
      "   0|2               |   -0.021|    0.850\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   1|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   4|5               |   -0.082|    0.838\n",
      "   4|5               |   -0.063|    0.838\n",
      "   4|5               |    0.053|    0.838\n",
      "   4|5               |    0.208|    0.840\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   0|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   3|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   1|5               |    0.050|    0.844\n",
      "   1|5               |    0.150|    0.846\n",
      "   4|4               |    0.000|    0.841\n",
      "   4|4               |    0.000|    0.844\n",
      "   1|4               |    0.000|    0.846\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   4|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   0|0               |   -0.028|    0.851\n",
      "   1|3               |    0.587|    0.847\n",
      "   1|3               |    0.687|    0.848\n",
      "   3|3               |    0.110|    0.834\n",
      "   3|3               |    0.167|    0.834\n",
      "   3|3               |    0.243|    0.836\n",
      "   3|3               |    0.396|    0.838\n",
      "   3|3               |    0.701|    0.841\n",
      "   3|3               |    1.311|    0.842\n",
      "   2|2               |    0.007|    0.841\n",
      "   2|2               |    0.007|    0.841\n",
      "   3|4               |    0.000|    0.843\n",
      "   3|4               |    0.000|    0.845\n",
      "   1|2               |   -0.007|    0.848\n",
      "   1|2               |   -0.006|    0.848\n",
      "   4|3               |    0.561|    0.844\n",
      "   4|3               |    0.661|    0.845\n",
      "   4|3               |    0.861|    0.846\n",
      "   4|0               |    0.000|    0.848\n",
      "   4|2               |   -0.010|    0.848\n",
      "   4|2               |   -0.009|    0.848\n",
      "   4|2               |   -0.008|    0.848\n",
      "   3|0               |   -0.019|    0.845\n",
      "   3|0               |   -0.024|    0.846\n",
      "   3|0               |   -0.045|    0.849\n",
      "   1|0               |   -0.006|    0.848\n",
      "   1|0               |   -0.007|    0.848\n",
      "   1|0               |   -0.009|    0.848\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   3|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   3|5               |    0.073|    0.849\n",
      "   3|5               |    0.066|    0.849\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   0|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   1|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   4|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   3|2               |    0.000|    0.851\n",
      "   1|4               |    0.000|    0.848\n",
      "   1|0               |   -0.023|    0.849\n",
      "   1|2               |   -0.007|    0.849\n",
      "   1|2               |   -0.008|    0.850\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   3|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   3|5               |    0.073|    0.851\n",
      "   3|5               |    0.094|    0.851\n",
      "   3|5               |    0.121|    0.851\n",
      "   0|5               |    0.222|    0.851\n",
      "   3|3               |    0.630|    0.851\n",
      "   3|4               |    0.000|    0.851\n",
      "---------------------------\n",
      "Finished successfully.\n",
      "CPU times: user 4.53 s, sys: 195 ms, total: 4.73 s\n",
      "Wall time: 2.95 s\n"
     ]
    }
   ],
   "source": [
    "ltr_feats3 = ((bm25>>qe>>bm25)%50) >>   pt.apply.generic(lambda df: df.merge(dataset,on='docno'))>> (\n",
    "        pt.transformer.IdentityTransformer()\n",
    "        ** # new feature: TF-IDF indicator\n",
    "        pt.BatchRetrieve(index, wmodel=\"TF_IDF\")\n",
    "        ** #new feature: review number\n",
    "        (pt.apply.doc_score(lambda row: math.log(row['review_num'])))\n",
    "        ** #new feature: overall rate\n",
    "        (pt.apply.doc_score(lambda row: row['overall_rate']))\n",
    "        **# new feature: positive review title length\n",
    "        (pt.apply.doc_score(lambda row: len(row['pos_review_title']) ))\n",
    "        ** # negative relevance\n",
    "        (pt.text.scorer(body_attr=\"neg_review_content\", wmodel='BM25') >> (pt.apply.doc_score(lambda row: row['score']* senti[int(row['docid'])])))\n",
    "    )\n",
    "\n",
    "import fastrank\n",
    "\n",
    "qrels['label'] = qrels['label'].apply(lambda x: float(x))\n",
    "train_request = fastrank.TrainRequest.coordinate_ascent()\n",
    "\n",
    "params = train_request.params\n",
    "params.init_random = True\n",
    "params.normalize = True\n",
    "params.seed = 1234567\n",
    "\n",
    "ca_pipe3 = ltr_feats3 >> pt.ltr.apply_learned_model(train_request, form='fastrank')\n",
    "\n",
    "%time ca_pipe3.fit(train_data, qrels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ltr_feats4: ltr_feats1 + Sentence transformer&Cosine similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------\n",
      "Training starts...\n",
      "---------------------------\n",
      "[+] Random restart #1/5...\n",
      "[+] Random restart #3/5...\n",
      "[+] Random restart #2/5...\n",
      "[+] Random restart #4/5...\n",
      "[+] Random restart #5/5...\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   0|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   2|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   1|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   3|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   4|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   0|0               |    0.000|    0.732\n",
      "   2|5               |    0.000|    0.733\n",
      "   1|5               |    0.000|    0.728\n",
      "   0|2               |    0.000|    0.733\n",
      "   3|5               |    0.166|    0.733\n",
      "   4|5               |    0.054|    0.733\n",
      "   2|5               |   -0.948|    0.733\n",
      "   0|2               |   -0.116|    0.733\n",
      "   0|2               |   -0.516|    0.733\n",
      "   0|2               |   -1.316|    0.735\n",
      "   0|2               |   -2.916|    0.736\n",
      "   0|2               |   -6.116|    0.742\n",
      "   0|2               |  -12.516|    0.744\n",
      "   0|2               |  -25.316|    0.750\n",
      "   0|2               |  -50.916|    0.754\n",
      "   0|2               | -102.116|    0.758\n",
      "   0|2               | -204.516|    0.760\n",
      "   0|2               | -409.316|    0.761\n",
      "   0|2               | -818.916|    0.762\n",
      "   0|2               |-1638.116|    0.763\n",
      "   0|2               |-3276.516|    0.764\n",
      "   0|2               |-6553.316|    0.764\n",
      "   0|2               |-13106.916|    0.764\n",
      "   0|2               |-26214.116|    0.764\n",
      "   3|5               |    0.266|    0.735\n",
      "   4|5               |    0.086|    0.733\n",
      "   1|5               |    0.538|    0.728\n",
      "   3|5               |    0.966|    0.735\n",
      "   1|5               |    1.338|    0.730\n",
      "   2|5               |    1.352|    0.735\n",
      "   4|5               |    0.163|    0.733\n",
      "   3|5               |    1.766|    0.736\n",
      "   1|5               |    2.938|    0.734\n",
      "   2|5               |    2.952|    0.735\n",
      "   4|5               |    0.265|    0.733\n",
      "   1|5               |    6.138|    0.735\n",
      "   2|5               |    6.152|    0.736\n",
      "   4|5               |    0.470|    0.733\n",
      "   3|5               |    6.566|    0.736\n",
      "   1|5               |   12.538|    0.736\n",
      "   0|3               |   -0.000|    0.764\n",
      "   4|5               |    0.879|    0.735\n",
      "   2|5               |   12.552|    0.737\n",
      "   0|3               |   -0.000|    0.764\n",
      "   1|5               |   25.338|    0.738\n",
      "   2|5               |   25.352|    0.739\n",
      "   4|5               |    1.698|    0.735\n",
      "   3|5               |   25.766|    0.740\n",
      "   0|3               |   -0.000|    0.764\n",
      "   1|5               |   50.938|    0.742\n",
      "   4|5               |    3.334|    0.736\n",
      "   2|5               |   50.952|    0.743\n",
      "   3|5               |   51.366|    0.745\n",
      "   4|5               |    6.608|    0.736\n",
      "   1|5               |  102.138|    0.746\n",
      "   2|5               |  102.152|    0.747\n",
      "   3|5               |  102.566|    0.751\n",
      "   4|5               |   13.155|    0.737\n",
      "   1|5               |  204.538|    0.759\n",
      "   3|5               |  204.966|    0.765\n",
      "   2|5               |  204.552|    0.761\n",
      "   4|5               |   26.249|    0.743\n",
      "   1|5               |  409.338|    0.774\n",
      "   3|5               |  409.766|    0.779\n",
      "   2|5               |  409.352|    0.776\n",
      "   4|5               |   52.438|    0.750\n",
      "   1|5               |  818.938|    0.782\n",
      "   3|5               |  819.366|    0.789\n",
      "   2|5               |  818.952|    0.784\n",
      "   4|5               |  104.815|    0.761\n",
      "   1|5               | 1638.138|    0.800\n",
      "   3|5               | 1638.566|    0.803\n",
      "   2|5               | 1638.152|    0.802\n",
      "   4|5               |  209.569|    0.776\n",
      "   1|5               | 3276.538|    0.814\n",
      "   3|5               | 3276.966|    0.816\n",
      "   2|5               | 3276.552|    0.815\n",
      "   4|5               |  419.077|    0.787\n",
      "   1|5               | 6553.338|    0.821\n",
      "   3|5               | 6553.766|    0.826\n",
      "   2|5               | 6553.352|    0.823\n",
      "   4|5               |  838.093|    0.802\n",
      "   1|5               |13106.938|    0.833\n",
      "   3|5               |13107.366|    0.841\n",
      "   2|5               |13106.952|    0.840\n",
      "   4|5               | 1676.125|    0.815\n",
      "   3|5               |26214.566|    0.841\n",
      "   1|5               |26214.138|    0.841\n",
      "   2|5               |26214.152|    0.841\n",
      "   4|5               | 3352.188|    0.823\n",
      "   3|5               |52428.966|    0.844\n",
      "   1|5               |52428.538|    0.843\n",
      "   2|5               |52428.552|    0.844\n",
      "   4|5               | 6704.316|    0.841\n",
      "   3|5               |104857.766|    0.844\n",
      "   2|5               |104857.352|    0.844\n",
      "   1|5               |104857.338|    0.843\n",
      "   4|5               |13408.571|    0.842\n",
      "   3|5               |209715.366|    0.844\n",
      "   2|5               |209714.952|    0.844\n",
      "   1|5               |209714.938|    0.844\n",
      "   4|5               |26817.082|    0.844\n",
      "   1|5               |419430.138|    0.844\n",
      "   4|5               |107268.145|    0.844\n",
      "   3|0               |    0.000|    0.844\n",
      "   1|0               |    0.000|    0.844\n",
      "   2|4               |   -0.000|    0.844\n",
      "   1|0               |   -0.000|    0.844\n",
      "   3|0               |   -0.000|    0.844\n",
      "   0|3               |    0.132|    0.768\n",
      "   0|3               |    0.265|    0.770\n",
      "   0|3               |    0.529|    0.777\n",
      "   0|3               |    1.058|    0.778\n",
      "   0|3               |    2.116|    0.781\n",
      "   0|3               |    4.232|    0.784\n",
      "   0|3               |    8.465|    0.789\n",
      "   0|1               |    0.000|    0.792\n",
      "   2|4               |   -0.000|    0.844\n",
      "   2|4               |   -0.000|    0.844\n",
      "   4|3               |    0.000|    0.844\n",
      "   4|3               |    0.000|    0.844\n",
      "   3|0               |    0.000|    0.844\n",
      "   3|0               |    0.000|    0.844\n",
      "   1|0               |    0.000|    0.844\n",
      "   4|3               |    0.000|    0.844\n",
      "   3|0               |    0.000|    0.844\n",
      "   4|3               |    0.000|    0.844\n",
      "   0|5               |   -0.000|    0.792\n",
      "   1|0               |    0.000|    0.844\n",
      "   3|0               |    0.000|    0.844\n",
      "   4|3               |    0.001|    0.844\n",
      "   1|0               |    0.000|    0.844\n",
      "   3|0               |    0.001|    0.845\n",
      "   4|3               |    0.003|    0.845\n",
      "   0|5               |   -0.000|    0.792\n",
      "   1|0               |    0.000|    0.844\n",
      "   4|3               |    0.007|    0.846\n",
      "   0|5               |   -0.001|    0.792\n",
      "   1|0               |    0.000|    0.844\n",
      "   4|3               |    0.014|    0.848\n",
      "   1|0               |    0.001|    0.845\n",
      "   4|3               |    0.028|    0.853\n",
      "   3|0               |    0.011|    0.845\n",
      "   4|3               |    0.055|    0.856\n",
      "   4|3               |    0.110|    0.861\n",
      "   1|0               |    0.009|    0.846\n",
      "   1|2               |    0.000|    0.846\n",
      "   4|0               |    0.000|    0.861\n",
      "   1|2               |    0.000|    0.846\n",
      "   2|0               |    0.000|    0.844\n",
      "   4|0               |   -0.000|    0.861\n",
      "   1|2               |   -0.000|    0.846\n",
      "   3|3               |   -0.000|    0.845\n",
      "   1|2               |   -0.000|    0.846\n",
      "   1|2               |   -0.000|    0.846\n",
      "   1|2               |   -0.000|    0.846\n",
      "   0|5               |    0.214|    0.793\n",
      "   0|5               |    0.429|    0.795\n",
      "   0|5               |    0.857|    0.798\n",
      "   0|5               |    1.715|    0.809\n",
      "   2|0               |    0.000|    0.844\n",
      "   3|3               |    0.000|    0.845\n",
      "   2|0               |    0.000|    0.844\n",
      "   4|0               |    0.000|    0.861\n",
      "   0|4               |   -0.000|    0.809\n",
      "   4|0               |    0.000|    0.861\n",
      "   2|0               |    0.000|    0.844\n",
      "   3|3               |    0.000|    0.845\n",
      "   4|0               |    0.000|    0.861\n",
      "   3|3               |    0.000|    0.845\n",
      "   4|0               |    0.000|    0.861\n",
      "   3|3               |    0.000|    0.845\n",
      "   4|0               |    0.000|    0.861\n",
      "   2|0               |    0.000|    0.844\n",
      "   3|3               |    0.000|    0.845\n",
      "   4|0               |    0.001|    0.862\n",
      "   2|0               |    0.000|    0.844\n",
      "   3|3               |    0.001|    0.845\n",
      "   2|0               |    0.001|    0.845\n",
      "   4|0               |    0.003|    0.863\n",
      "   3|3               |    0.003|    0.847\n",
      "   4|0               |    0.005|    0.866\n",
      "   3|3               |    0.006|    0.849\n",
      "   4|0               |    0.010|    0.869\n",
      "   3|3               |    0.013|    0.852\n",
      "   3|3               |    0.026|    0.859\n",
      "   3|3               |    0.052|    0.866\n",
      "   3|3               |    0.103|    0.868\n",
      "   1|1               |    0.000|    0.846\n",
      "   3|4               |   -0.000|    0.868\n",
      "   2|3               |    0.000|    0.845\n",
      "   0|4               |   -0.000|    0.809\n",
      "   2|3               |    0.000|    0.845\n",
      "   0|4               |    0.000|    0.809\n",
      "   0|4               |    0.000|    0.810\n",
      "   2|3               |   -0.000|    0.845\n",
      "   2|3               |   -0.000|    0.845\n",
      "   0|4               |    0.000|    0.810\n",
      "   0|4               |    0.000|    0.811\n",
      "   2|3               |   -0.000|    0.845\n",
      "   0|4               |    0.000|    0.816\n",
      "   1|1               |   -0.001|    0.847\n",
      "   2|3               |   -0.000|    0.845\n",
      "   0|4               |    0.000|    0.821\n",
      "   4|4               |   -0.000|    0.869\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   0|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   4|4               |    0.000|    0.869\n",
      "   3|4               |    0.000|    0.869\n",
      "   3|4               |    0.000|    0.869\n",
      "   2|3               |    0.001|    0.845\n",
      "   2|3               |    0.001|    0.845\n",
      "   2|3               |    0.002|    0.845\n",
      "   2|3               |    0.004|    0.846\n",
      "   2|3               |    0.008|    0.847\n",
      "   2|3               |    0.017|    0.851\n",
      "   2|3               |    0.033|    0.857\n",
      "   2|3               |    0.066|    0.861\n",
      "   4|2               |    0.000|    0.869\n",
      "   3|2               |    0.000|    0.869\n",
      "   1|3               |   -0.000|    0.847\n",
      "   0|5               |    0.682|    0.822\n",
      "   0|5               |    0.782|    0.823\n",
      "   0|5               |    0.982|    0.824\n",
      "   0|5               |    1.382|    0.832\n",
      "   0|5               |    2.182|    0.840\n",
      "   3|2               |   -0.000|    0.869\n",
      "   2|2               |   -0.000|    0.861\n",
      "   0|5               |    3.782|    0.849\n",
      "   3|2               |   -0.000|    0.869\n",
      "   4|2               |   -0.000|    0.870\n",
      "   0|5               |   13.382|    0.850\n",
      "   3|2               |   -0.000|    0.869\n",
      "   1|3               |   -0.000|    0.847\n",
      "   4|2               |   -0.000|    0.870\n",
      "   1|3               |   -0.000|    0.847\n",
      "   3|2               |   -0.000|    0.869\n",
      "   2|2               |   -0.001|    0.861\n",
      "   2|2               |   -0.001|    0.861\n",
      "   1|3               |    0.000|    0.847\n",
      "   1|3               |    0.000|    0.847\n",
      "   1|3               |    0.000|    0.847\n",
      "   1|3               |    0.001|    0.847\n",
      "   1|3               |    0.001|    0.847\n",
      "   1|3               |    0.008|    0.848\n",
      "   1|3               |    0.016|    0.852\n",
      "   1|3               |    0.032|    0.860\n",
      "   1|3               |    0.065|    0.866\n",
      "   1|4               |   -0.000|    0.866\n",
      "   1|4               |   -0.000|    0.866\n",
      "   2|1               |   -0.000|    0.861\n",
      "   3|1               |    0.000|    0.869\n",
      "   4|1               |   -0.000|    0.870\n",
      "   3|1               |   -0.000|    0.869\n",
      "   2|1               |   -0.000|    0.861\n",
      "   3|1               |   -0.000|    0.869\n",
      "   1|4               |    0.000|    0.866\n",
      "   1|4               |    0.000|    0.867\n",
      "   4|1               |    0.000|    0.870\n",
      "   2|1               |    0.000|    0.861\n",
      "   4|1               |    0.001|    0.870\n",
      "   3|1               |    0.001|    0.869\n",
      "   4|1               |    0.002|    0.871\n",
      "   2|1               |    0.000|    0.861\n",
      "   4|1               |    0.003|    0.871\n",
      "   3|1               |    0.002|    0.870\n",
      "   3|1               |    0.005|    0.871\n",
      "   2|1               |    0.014|    0.864\n",
      "   0|3               |    0.025|    0.850\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   1|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   4|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   0|3               |    0.042|    0.851\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   3|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   1|5               |    0.880|    0.867\n",
      "   0|3               |    0.061|    0.854\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   2|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   1|5               |    0.780|    0.869\n",
      "   0|3               |    0.099|    0.858\n",
      "   4|0               |    0.009|    0.872\n",
      "   4|0               |    0.007|    0.874\n",
      "   2|5               |    0.874|    0.864\n",
      "   2|5               |    0.774|    0.872\n",
      "   2|5               |    0.574|    0.872\n",
      "   0|1               |    0.000|    0.858\n",
      "   1|4               |    0.000|    0.869\n",
      "   4|2               |   -0.001|    0.874\n",
      "   1|4               |    0.000|    0.869\n",
      "   4|2               |   -0.001|    0.874\n",
      "   1|4               |    0.000|    0.870\n",
      "   2|4               |    0.000|    0.873\n",
      "   4|2               |   -0.001|    0.874\n",
      "   3|5               |    1.242|    0.871\n",
      "   4|2               |   -0.001|    0.874\n",
      "   0|1               |    0.050|    0.858\n",
      "   3|3               |    0.066|    0.871\n",
      "   2|4               |    0.000|    0.873\n",
      "   2|4               |    0.000|    0.874\n",
      "   0|4               |    0.000|    0.859\n",
      "   1|3               |    0.075|    0.871\n",
      "   3|3               |    0.072|    0.871\n",
      "   1|3               |    0.082|    0.872\n",
      "   3|3               |    0.079|    0.872\n",
      "   3|3               |    0.093|    0.872\n",
      "   2|0               |    0.000|    0.874\n",
      "   2|0               |   -0.000|    0.874\n",
      "   2|0               |   -0.002|    0.875\n",
      "   2|0               |   -0.005|    0.876\n",
      "   0|4               |    0.000|    0.859\n",
      "   0|4               |    0.000|    0.860\n",
      "   1|0               |    0.009|    0.872\n",
      "   3|1               |    0.004|    0.873\n",
      "   2|1               |    0.022|    0.876\n",
      "   2|1               |    0.024|    0.877\n",
      "   1|1               |   -0.001|    0.872\n",
      "   1|1               |   -0.001|    0.872\n",
      "   4|1               |    0.003|    0.874\n",
      "   0|2               |    0.001|    0.862\n",
      "   3|2               |    0.000|    0.873\n",
      "   4|1               |    0.004|    0.875\n",
      "   0|2               |    0.005|    0.862\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   0|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   1|1               |   -0.001|    0.872\n",
      "   1|1               |   -0.000|    0.872\n",
      "   1|1               |    0.001|    0.872\n",
      "   3|2               |    0.000|    0.873\n",
      "   1|2               |   -0.000|    0.872\n",
      "   1|2               |   -0.000|    0.872\n",
      "   3|0               |    0.007|    0.873\n",
      "   2|2               |    0.000|    0.877\n",
      "   1|2               |   -0.001|    0.872\n",
      "   2|2               |   -0.000|    0.877\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   1|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   4|4               |    0.000|    0.875\n",
      "   4|4               |    0.000|    0.875\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   2|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   3|4               |    0.000|    0.874\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   4|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   0|3               |    0.119|    0.864\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   3|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   2|4               |    0.000|    0.877\n",
      "   2|4               |    0.000|    0.877\n",
      "   0|1               |    0.044|    0.865\n",
      "   0|1               |    0.030|    0.869\n",
      "   1|4               |    0.000|    0.873\n",
      "   1|4               |    0.000|    0.873\n",
      "   2|4               |    0.000|    0.877\n",
      "   0|4               |    0.000|    0.876\n",
      "   0|2               |    0.005|    0.876\n",
      "   0|2               |    0.003|    0.876\n",
      "   3|2               |    0.000|    0.874\n",
      "   2|0               |   -0.005|    0.878\n",
      "   0|2               |    0.006|    0.876\n",
      "   1|3               |    0.085|    0.873\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   0|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   1|2               |   -0.001|    0.873\n",
      "   2|2               |   -0.000|    0.878\n",
      "   2|2               |   -0.001|    0.878\n",
      "   3|3               |    0.096|    0.874\n",
      "   3|4               |    0.000|    0.874\n",
      "   3|4               |    0.000|    0.874\n",
      "   4|2               |   -0.001|    0.875\n",
      "   1|1               |    0.001|    0.873\n",
      "   1|1               |    0.002|    0.873\n",
      "   4|2               |   -0.001|    0.875\n",
      "   0|1               |    0.032|    0.876\n",
      "   0|5               |    0.895|    0.876\n",
      "---------------------------\n",
      "Finished successfully.\n",
      "CPU times: user 21 s, sys: 302 ms, total: 21.3 s\n",
      "Wall time: 20 s\n"
     ]
    }
   ],
   "source": [
    "ltr_feats4 = ((bm25>>qe>>bm25)%50) >>   pt.apply.generic(lambda df: df.merge(dataset,on='docno'))>> (\n",
    "        pt.transformer.IdentityTransformer()\n",
    "        ** # new feature: TF-IDF indicator\n",
    "        pt.BatchRetrieve(index, wmodel=\"TF_IDF\")\n",
    "        ** #new feature: review number\n",
    "        (pt.apply.doc_score(lambda row: math.log(row['review_num'])))\n",
    "        ** #new feature: overall rate\n",
    "        (pt.apply.doc_score(lambda row: row['overall_rate']))\n",
    "        **# new feature: positive review title length\n",
    "        (pt.apply.doc_score(lambda row: len(row['pos_review_title']) ))\n",
    "        **\n",
    "        (pt.apply.doc_score(lambda row: float(util.pytorch_cos_sim(model.encode(row[\"query_0\"], convert_to_tensor=True), model.encode(row[\"Description\"], convert_to_tensor=True)))))\n",
    "    )\n",
    "\n",
    "import fastrank\n",
    "\n",
    "qrels['label'] = qrels['label'].apply(lambda x: float(x))\n",
    "train_request = fastrank.TrainRequest.coordinate_ascent()\n",
    "\n",
    "params = train_request.params\n",
    "params.init_random = True\n",
    "params.normalize = True\n",
    "params.seed = 1234567\n",
    "\n",
    "ca_pipe4 = ltr_feats4 >> pt.ltr.apply_learned_model(train_request, form='fastrank')\n",
    "\n",
    "%time ca_pipe4.fit(train_data, qrels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ltr_feats_all: All 8 features pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-12-17 21:26:19,026][onir_pt][DEBUG] using GPU (deterministic)\n",
      "[2022-12-17 21:26:19,032][onir_pt][DEBUG] [starting] batches\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "batches:   0%|          | 0/300 [15ms<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-12-17 21:26:27,040][onir_pt][DEBUG] [finished] batches: [8.01s] [300it] [37.47it/s]\n",
      "---------------------------\n",
      "Training starts...\n",
      "---------------------------\n",
      "[+] Random restart #1/5...\n",
      "[+] Random restart #3/5...\n",
      "[+] Random restart #2/5...\n",
      "[+] Random restart #4/5...\n",
      "[+] Random restart #5/5...\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   0|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   2|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   1|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   3|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   4|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   1|8               |    0.000|    0.728\n",
      "   2|8               |   -0.043|    0.733\n",
      "   0|8               |   -6.214|    0.728\n",
      "   2|8               |   -0.304|    0.733\n",
      "   1|8               |   -0.745|    0.728\n",
      "   0|8               |-3276.614|    0.728\n",
      "   4|8               | -273.162|    0.733\n",
      "   2|8               |-4332.594|    0.733\n",
      "   3|8               |-5850.542|    0.735\n",
      "   3|8               |-11701.031|    0.735\n",
      "   0|8               |-838860.614|    0.729\n",
      "   2|8               |-34660.475|    0.733\n",
      "   1|8               |-53192.507|    0.728\n",
      "   1|8               |-106385.080|    0.728\n",
      "   4|8               |17488.032|    0.735\n",
      "   4|8               |34975.984|    0.735\n",
      "   3|8               |46803.854|    0.735\n",
      "   0|8               |838860.886|    0.732\n",
      "   3|8               |93607.767|    0.735\n",
      "   0|8               |1677721.686|    0.734\n",
      "   3|0               |    0.000|    0.735\n",
      "   0|7               |    0.000|    0.734\n",
      "   2|7               |   -0.000|    0.733\n",
      "   1|7               |   -0.000|    0.729\n",
      "   4|0               |   -0.006|    0.737\n",
      "   4|0               |   -0.012|    0.742\n",
      "   4|0               |   -0.024|    0.746\n",
      "   3|0               |   -0.004|    0.737\n",
      "   4|0               |   -0.048|    0.749\n",
      "   3|0               |   -0.009|    0.742\n",
      "   4|0               |   -0.097|    0.750\n",
      "   3|0               |   -0.017|    0.746\n",
      "   4|0               |   -0.193|    0.750\n",
      "   3|0               |   -0.034|    0.750\n",
      "   3|0               |   -0.068|    0.750\n",
      "   3|0               |   -0.136|    0.750\n",
      "   3|2               |   -0.000|    0.750\n",
      "   4|6               |   -0.000|    0.750\n",
      "   0|7               |    0.000|    0.734\n",
      "   2|7               |    0.000|    0.735\n",
      "   1|7               |    0.000|    0.730\n",
      "   0|7               |    0.000|    0.734\n",
      "   2|7               |    0.000|    0.735\n",
      "   1|7               |    0.000|    0.734\n",
      "   0|7               |    0.000|    0.736\n",
      "   3|2               |   -0.000|    0.750\n",
      "   4|6               |   -0.000|    0.750\n",
      "   2|7               |    0.000|    0.736\n",
      "   1|7               |    0.000|    0.734\n",
      "   0|7               |    0.000|    0.736\n",
      "   3|2               |   -0.000|    0.750\n",
      "   2|7               |    0.000|    0.738\n",
      "   1|7               |    0.000|    0.736\n",
      "   3|2               |   -0.000|    0.751\n",
      "   2|7               |    0.001|    0.741\n",
      "   1|7               |    0.000|    0.739\n",
      "   0|7               |    0.000|    0.736\n",
      "   4|6               |   -0.000|    0.750\n",
      "   2|7               |    0.001|    0.745\n",
      "   1|7               |    0.000|    0.743\n",
      "   0|7               |    0.000|    0.740\n",
      "   4|6               |   -0.001|    0.750\n",
      "   2|7               |    0.003|    0.755\n",
      "   0|7               |    0.000|    0.745\n",
      "   1|7               |    0.001|    0.747\n",
      "   3|2               |   -0.002|    0.751\n",
      "   4|6               |   -0.004|    0.751\n",
      "   2|7               |    0.006|    0.769\n",
      "   0|7               |    0.000|    0.753\n",
      "   1|7               |    0.001|    0.761\n",
      "   4|6               |   -0.007|    0.751\n",
      "   2|7               |    0.011|    0.780\n",
      "   0|7               |    0.000|    0.766\n",
      "   1|7               |    0.003|    0.777\n",
      "   3|2               |   -0.010|    0.752\n",
      "   2|7               |    0.022|    0.796\n",
      "   0|7               |    0.000|    0.779\n",
      "   1|7               |    0.005|    0.784\n",
      "   2|7               |    0.044|    0.808\n",
      "   0|7               |    0.000|    0.786\n",
      "   1|7               |    0.010|    0.802\n",
      "   3|2               |   -0.038|    0.754\n",
      "   2|7               |    0.088|    0.818\n",
      "   1|7               |    0.020|    0.814\n",
      "   0|7               |    0.000|    0.804\n",
      "   3|2               |   -0.076|    0.762\n",
      "   2|7               |    0.176|    0.828\n",
      "   1|7               |    0.041|    0.823\n",
      "   0|7               |    0.001|    0.816\n",
      "   2|7               |    0.353|    0.841\n",
      "   1|7               |    0.082|    0.840\n",
      "   0|7               |    0.002|    0.826\n",
      "   2|7               |    0.706|    0.842\n",
      "   1|7               |    0.163|    0.842\n",
      "   4|6               |   -0.936|    0.753\n",
      "   0|7               |    0.003|    0.841\n",
      "   2|7               |    1.412|    0.843\n",
      "   1|7               |    0.326|    0.844\n",
      "   0|7               |    0.006|    0.841\n",
      "   4|1               |    0.000|    0.753\n",
      "   2|7               |    2.824|    0.844\n",
      "   0|7               |    0.013|    0.844\n",
      "   2|7               |    5.647|    0.844\n",
      "   1|7               |    1.306|    0.844\n",
      "   0|7               |    0.026|    0.844\n",
      "   0|7               |    0.051|    0.844\n",
      "   3|3               |    0.000|    0.762\n",
      "   2|0               |    0.000|    0.844\n",
      "   0|6               |    0.000|    0.844\n",
      "   3|3               |   -0.000|    0.762\n",
      "   4|1               |   -0.000|    0.753\n",
      "   3|3               |   -0.001|    0.762\n",
      "   0|6               |   -0.000|    0.844\n",
      "   3|3               |   -0.004|    0.763\n",
      "   0|6               |   -0.001|    0.845\n",
      "   4|1               |    0.000|    0.753\n",
      "   2|0               |    0.000|    0.844\n",
      "   4|1               |    0.000|    0.753\n",
      "   2|0               |    0.000|    0.844\n",
      "   4|1               |    0.000|    0.754\n",
      "   4|1               |    0.000|    0.754\n",
      "   2|0               |    0.000|    0.844\n",
      "   4|1               |    0.001|    0.754\n",
      "   4|1               |    0.002|    0.758\n",
      "   2|0               |    0.000|    0.844\n",
      "   4|1               |    0.003|    0.759\n",
      "   1|0               |    0.000|    0.844\n",
      "   2|0               |    0.000|    0.844\n",
      "   4|1               |    0.006|    0.762\n",
      "   1|0               |    0.000|    0.844\n",
      "   2|0               |    0.000|    0.844\n",
      "   4|1               |    0.013|    0.767\n",
      "   1|0               |    0.000|    0.844\n",
      "   2|0               |    0.001|    0.845\n",
      "   4|1               |    0.026|    0.774\n",
      "   1|0               |    0.000|    0.844\n",
      "   3|3               |    0.017|    0.764\n",
      "   4|1               |    0.052|    0.776\n",
      "   1|0               |    0.000|    0.844\n",
      "   3|3               |    0.034|    0.775\n",
      "   4|1               |    0.103|    0.794\n",
      "   1|0               |    0.001|    0.845\n",
      "   4|1               |    0.207|    0.825\n",
      "   2|0               |    0.012|    0.847\n",
      "   3|3               |    0.135|    0.777\n",
      "   3|3               |    0.269|    0.782\n",
      "   3|3               |    0.539|    0.785\n",
      "   3|3               |    1.078|    0.791\n",
      "   1|0               |    0.009|    0.846\n",
      "   3|3               |    2.156|    0.793\n",
      "   3|7               |   -0.000|    0.793\n",
      "   1|5               |    0.000|    0.846\n",
      "   4|2               |   -0.000|    0.825\n",
      "   1|5               |   -0.000|    0.846\n",
      "   0|1               |   -0.000|    0.845\n",
      "   0|1               |   -0.000|    0.845\n",
      "   0|1               |   -0.000|    0.845\n",
      "   3|7               |   -0.000|    0.793\n",
      "   3|7               |   -0.003|    0.793\n",
      "   3|7               |   -0.005|    0.793\n",
      "   2|5               |   -0.000|    0.847\n",
      "   4|2               |    0.000|    0.825\n",
      "   4|2               |    0.000|    0.825\n",
      "   4|2               |    0.000|    0.825\n",
      "   4|2               |    0.000|    0.825\n",
      "   4|2               |    0.000|    0.825\n",
      "   4|2               |    0.001|    0.826\n",
      "   3|7               |    0.000|    0.793\n",
      "   0|1               |    0.000|    0.845\n",
      "   3|7               |    0.000|    0.793\n",
      "   0|1               |    0.000|    0.845\n",
      "   0|1               |    0.000|    0.845\n",
      "   1|5               |    0.000|    0.846\n",
      "   3|7               |    0.001|    0.793\n",
      "   0|1               |    0.000|    0.845\n",
      "   1|5               |    0.001|    0.846\n",
      "   4|2               |    0.026|    0.829\n",
      "   1|5               |    0.002|    0.846\n",
      "   1|5               |    0.003|    0.846\n",
      "   2|5               |    0.004|    0.847\n",
      "   3|7               |    0.011|    0.793\n",
      "   1|5               |    0.006|    0.846\n",
      "   2|5               |    0.008|    0.847\n",
      "   3|7               |    0.022|    0.795\n",
      "   1|5               |    0.013|    0.846\n",
      "   2|5               |    0.015|    0.847\n",
      "   3|7               |    0.692|    0.800\n",
      "   4|4               |    0.000|    0.829\n",
      "   3|4               |    0.000|    0.800\n",
      "   2|4               |    0.000|    0.848\n",
      "   0|5               |    0.000|    0.845\n",
      "   4|4               |   -0.000|    0.830\n",
      "   2|4               |   -0.000|    0.848\n",
      "   3|4               |   -0.000|    0.800\n",
      "   0|5               |   -0.000|    0.845\n",
      "   1|1               |   -0.000|    0.846\n",
      "   1|1               |   -0.000|    0.846\n",
      "   0|5               |   -0.000|    0.845\n",
      "   1|1               |   -0.000|    0.847\n",
      "   1|1               |   -0.004|    0.847\n",
      "   4|5               |    0.000|    0.830\n",
      "   2|4               |   -0.000|    0.848\n",
      "   2|4               |    0.000|    0.848\n",
      "   3|4               |    0.000|    0.801\n",
      "   2|4               |    0.000|    0.848\n",
      "   3|4               |    0.000|    0.801\n",
      "   3|4               |    0.000|    0.801\n",
      "   3|4               |    0.000|    0.804\n",
      "   3|4               |    0.000|    0.809\n",
      "   3|4               |    0.000|    0.819\n",
      "   0|5               |    0.000|    0.845\n",
      "   0|5               |    0.000|    0.845\n",
      "   0|5               |    0.000|    0.846\n",
      "   0|5               |    0.001|    0.846\n",
      "   0|5               |    0.001|    0.847\n",
      "   0|5               |    0.002|    0.848\n",
      "   1|3               |    0.000|    0.847\n",
      "   3|5               |    0.000|    0.819\n",
      "   2|1               |   -0.000|    0.848\n",
      "   1|3               |   -0.000|    0.847\n",
      "   0|3               |   -0.000|    0.848\n",
      "   1|3               |   -0.000|    0.847\n",
      "   4|5               |    0.001|    0.830\n",
      "   2|1               |   -0.000|    0.848\n",
      "   4|5               |    0.006|    0.830\n",
      "   2|1               |   -0.001|    0.849\n",
      "   4|5               |    0.011|    0.830\n",
      "   2|1               |   -0.004|    0.850\n",
      "   4|5               |    0.365|    0.832\n",
      "   4|5               |    0.730|    0.832\n",
      "   4|3               |   -0.000|    0.832\n",
      "   3|5               |    0.000|    0.819\n",
      "   2|3               |    0.000|    0.850\n",
      "   4|3               |   -0.000|    0.832\n",
      "   4|3               |   -0.000|    0.832\n",
      "   0|3               |    0.000|    0.848\n",
      "   1|3               |    0.000|    0.847\n",
      "   0|3               |    0.000|    0.848\n",
      "   1|3               |    0.000|    0.847\n",
      "   0|3               |    0.000|    0.848\n",
      "   1|3               |    0.000|    0.847\n",
      "   3|5               |    0.001|    0.819\n",
      "   0|3               |    0.000|    0.848\n",
      "   1|3               |    0.000|    0.847\n",
      "   3|5               |    0.002|    0.820\n",
      "   3|5               |    0.003|    0.820\n",
      "   0|3               |    0.000|    0.849\n",
      "   1|3               |    0.002|    0.847\n",
      "   3|5               |    0.007|    0.820\n",
      "   0|3               |    0.001|    0.850\n",
      "   1|3               |    0.004|    0.852\n",
      "   3|5               |    0.013|    0.820\n",
      "   0|3               |    0.001|    0.855\n",
      "   1|3               |    0.008|    0.855\n",
      "   3|5               |    0.026|    0.820\n",
      "   1|3               |    0.015|    0.859\n",
      "   3|5               |    0.052|    0.821\n",
      "   0|3               |    0.005|    0.857\n",
      "   1|3               |    0.031|    0.861\n",
      "   3|5               |    0.104|    0.823\n",
      "   3|5               |    0.209|    0.827\n",
      "   3|5               |    0.418|    0.830\n",
      "   1|4               |    0.000|    0.862\n",
      "   0|4               |    0.000|    0.858\n",
      "   0|2               |    0.000|    0.858\n",
      "   3|6               |   -0.000|    0.830\n",
      "   4|3               |    0.000|    0.832\n",
      "   0|2               |   -0.000|    0.858\n",
      "   4|3               |    0.000|    0.832\n",
      "   4|3               |    0.000|    0.832\n",
      "   4|3               |    0.001|    0.833\n",
      "   4|3               |    0.001|    0.833\n",
      "   2|3               |    0.000|    0.850\n",
      "   4|3               |    0.003|    0.833\n",
      "   4|3               |    0.006|    0.833\n",
      "   3|6               |   -0.000|    0.830\n",
      "   2|3               |    0.000|    0.850\n",
      "   4|3               |    0.012|    0.834\n",
      "   2|3               |    0.000|    0.850\n",
      "   2|3               |    0.001|    0.850\n",
      "   3|6               |   -0.003|    0.830\n",
      "   2|3               |    0.001|    0.850\n",
      "   4|3               |    0.047|    0.840\n",
      "   2|3               |    0.002|    0.852\n",
      "   4|3               |    0.093|    0.842\n",
      "   2|3               |    0.005|    0.854\n",
      "   3|6               |   -0.022|    0.831\n",
      "   2|3               |    0.010|    0.855\n",
      "   3|6               |   -0.044|    0.831\n",
      "   2|3               |    0.020|    0.859\n",
      "   2|3               |    0.039|    0.863\n",
      "   2|3               |    0.079|    0.867\n",
      "   1|4               |    0.000|    0.862\n",
      "   4|7               |   -0.000|    0.842\n",
      "   0|2               |    0.000|    0.858\n",
      "   0|2               |    0.000|    0.859\n",
      "   0|2               |    0.000|    0.859\n",
      "   2|2               |   -0.001|    0.867\n",
      "   3|6               |    0.044|    0.831\n",
      "   3|6               |    0.088|    0.832\n",
      "   3|6               |    0.176|    0.833\n",
      "   3|1               |    0.000|    0.833\n",
      "   3|1               |    0.000|    0.833\n",
      "   1|2               |    0.000|    0.862\n",
      "   0|0               |   -0.000|    0.859\n",
      "   0|0               |   -0.000|    0.859\n",
      "   1|2               |   -0.000|    0.862\n",
      "   3|1               |   -0.000|    0.833\n",
      "   4|7               |    0.002|    0.842\n",
      "   4|7               |    0.004|    0.843\n",
      "   4|7               |    0.007|    0.843\n",
      "   2|2               |    0.000|    0.867\n",
      "   4|7               |    0.015|    0.843\n",
      "   1|2               |   -0.002|    0.865\n",
      "   4|7               |    0.059|    0.844\n",
      "   4|7               |    0.118|    0.847\n",
      "   4|7               |    0.236|    0.849\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   4|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   4|8               |   -0.000|    0.849\n",
      "   1|6               |    0.000|    0.865\n",
      "   4|8               |   -3.000|    0.849\n",
      "   4|8               |   -6.200|    0.849\n",
      "   4|8               |  -12.600|    0.849\n",
      "   4|8               |  -25.400|    0.849\n",
      "   0|0               |    0.000|    0.860\n",
      "   4|8               |  -51.000|    0.849\n",
      "   0|0               |    0.000|    0.860\n",
      "   0|0               |    0.000|    0.860\n",
      "   3|1               |    0.000|    0.833\n",
      "   0|0               |    0.000|    0.860\n",
      "   3|1               |    0.000|    0.833\n",
      "   0|0               |    0.000|    0.860\n",
      "   1|6               |   -0.000|    0.865\n",
      "   4|8               |-3276.600|    0.849\n",
      "   3|1               |    0.001|    0.834\n",
      "   1|6               |   -0.001|    0.865\n",
      "   2|6               |   -0.000|    0.867\n",
      "   3|1               |    0.001|    0.834\n",
      "   1|6               |   -0.001|    0.865\n",
      "   2|6               |   -0.000|    0.867\n",
      "   3|1               |    0.003|    0.835\n",
      "   1|6               |   -0.003|    0.865\n",
      "   2|6               |   -0.001|    0.867\n",
      "   3|1               |    0.006|    0.838\n",
      "   1|6               |   -0.005|    0.866\n",
      "   0|0               |    0.000|    0.863\n",
      "   3|1               |    0.012|    0.841\n",
      "   0|0               |    0.001|    0.867\n",
      "   3|1               |    0.023|    0.852\n",
      "   3|1               |    0.047|    0.856\n",
      "   3|1               |    0.093|    0.856\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   3|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   1|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   0|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   3|8               |    0.055|    0.856\n",
      "   0|8               |    0.594|    0.867\n",
      "   1|8               |   -1.157|    0.866\n",
      "   3|8               |   -0.449|    0.856\n",
      "   0|8               |    0.194|    0.867\n",
      "   1|8               |   -6.757|    0.866\n",
      "   0|8               |   -5.406|    0.867\n",
      "   1|8               |  -13.157|    0.866\n",
      "   0|8               |  -11.806|    0.867\n",
      "   1|8               |  -25.957|    0.866\n",
      "   4|8               |  409.700|    0.849\n",
      "   3|8               |  -34.319|    0.856\n",
      "   4|8               |  819.300|    0.850\n",
      "   1|8               | -102.757|    0.866\n",
      "   0|8               |  -50.206|    0.868\n",
      "   3|8               |  -68.726|    0.856\n",
      "   0|8               | -101.406|    0.868\n",
      "   4|8               | 3276.900|    0.850\n",
      "   0|8               | -203.806|    0.868\n",
      "   4|8               | 6553.700|    0.852\n",
      "   2|6               |    0.001|    0.867\n",
      "   3|8               |-2201.962|    0.857\n",
      "   3|8               |-17616.313|    0.858\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   2|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   2|8               |   -0.285|    0.867\n",
      "   0|7               |    0.000|    0.869\n",
      "   2|8               |  -12.885|    0.867\n",
      "   2|8               |  -25.685|    0.867\n",
      "   2|8               | -409.685|    0.867\n",
      "   2|8               |-3276.885|    0.868\n",
      "   4|7               |    0.000|    0.853\n",
      "   1|6               |   -0.000|    0.866\n",
      "   4|7               |    0.001|    0.853\n",
      "   2|5               |    0.000|    0.871\n",
      "   2|6               |   -0.000|    0.871\n",
      "   2|6               |   -0.000|    0.871\n",
      "   3|5               |    0.000|    0.858\n",
      "   3|5               |    0.000|    0.859\n",
      "   3|5               |    0.000|    0.859\n",
      "   3|5               |    0.000|    0.859\n",
      "   0|0               |    0.000|    0.869\n",
      "   0|0               |    0.000|    0.871\n",
      "   4|3               |    0.000|    0.854\n",
      "   4|3               |    0.000|    0.858\n",
      "   2|6               |    0.000|    0.871\n",
      "   3|1               |    0.000|    0.859\n",
      "   4|5               |    0.000|    0.861\n",
      "   0|1               |    0.000|    0.871\n",
      "   0|1               |    0.000|    0.871\n",
      "   4|4               |   -0.000|    0.862\n",
      "   2|2               |    0.000|    0.871\n",
      "   0|1               |    0.000|    0.871\n",
      "   2|2               |    0.000|    0.871\n",
      "   3|7               |    0.000|    0.859\n",
      "   0|4               |    0.000|    0.871\n",
      "   4|1               |    0.000|    0.863\n",
      "   1|5               |    0.000|    0.866\n",
      "   4|1               |    0.000|    0.863\n",
      "   1|5               |    0.000|    0.866\n",
      "   4|1               |    0.000|    0.865\n",
      "   4|1               |    0.000|    0.868\n",
      "   3|7               |    0.000|    0.859\n",
      "   3|7               |    0.000|    0.859\n",
      "   3|7               |    0.000|    0.861\n",
      "   3|7               |    0.000|    0.862\n",
      "   3|7               |    0.000|    0.862\n",
      "   3|7               |    0.000|    0.863\n",
      "   4|0               |   -0.000|    0.868\n",
      "   4|0               |   -0.000|    0.869\n",
      "   1|2               |   -0.000|    0.867\n",
      "   3|3               |    0.000|    0.867\n",
      "   3|3               |    0.000|    0.867\n",
      "   2|3               |    0.000|    0.872\n",
      "   1|4               |    0.000|    0.867\n",
      "   3|4               |    0.000|    0.869\n",
      "   3|6               |    0.000|    0.870\n",
      "   0|5               |    0.000|    0.871\n",
      "   4|2               |    0.000|    0.870\n",
      "   0|5               |    0.000|    0.873\n",
      "   4|2               |    0.000|    0.870\n",
      "   4|2               |    0.000|    0.870\n",
      "   0|2               |    0.000|    0.873\n",
      "   4|6               |   -0.000|    0.870\n",
      "   2|1               |    0.000|    0.872\n",
      "   3|2               |    0.000|    0.871\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   3|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   3|0               |   -0.000|    0.872\n",
      "   3|0               |   -0.000|    0.873\n",
      "   4|6               |   -0.000|    0.870\n",
      "   4|6               |   -0.000|    0.872\n",
      "   3|2               |    0.000|    0.873\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   4|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   2|4               |    0.000|    0.872\n",
      "   2|4               |    0.000|    0.872\n",
      "   4|8               |   -0.551|    0.872\n",
      "   4|8               |   -2.151|    0.873\n",
      "   4|8               |   -5.351|    0.876\n",
      "   0|3               |    0.000|    0.873\n",
      "   2|7               |    0.000|    0.873\n",
      "   3|6               |    0.000|    0.873\n",
      "   3|6               |    0.000|    0.874\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   0|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   0|8               |   -1.050|    0.873\n",
      "   0|8               |   -1.150|    0.873\n",
      "   0|8               |   -1.350|    0.873\n",
      "   4|6               |   -0.000|    0.876\n",
      "   4|6               |   -0.000|    0.877\n",
      "   4|6               |    0.000|    0.881\n",
      "   1|1               |   -0.000|    0.867\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   2|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   2|0               |    0.000|    0.873\n",
      "   2|0               |    0.000|    0.875\n",
      "   3|3               |    0.000|    0.874\n",
      "   2|7               |    0.000|    0.875\n",
      "   4|5               |    0.000|    0.881\n",
      "   0|5               |    0.000|    0.873\n",
      "   0|5               |    0.000|    0.873\n",
      "   2|2               |   -0.000|    0.876\n",
      "   3|1               |    0.000|    0.874\n",
      "   0|7               |    0.000|    0.874\n",
      "   0|2               |    0.000|    0.874\n",
      "   2|3               |    0.000|    0.876\n",
      "   0|2               |    0.000|    0.874\n",
      "   0|2               |    0.000|    0.874\n",
      "   3|5               |    0.000|    0.878\n",
      "   3|7               |    0.000|    0.878\n",
      "   2|4               |    0.000|    0.876\n",
      "   2|4               |    0.000|    0.876\n",
      "   2|4               |    0.000|    0.877\n",
      "   0|3               |    0.000|    0.874\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   3|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   3|8               |   -1.350|    0.878\n",
      "   4|4               |    0.000|    0.881\n",
      "   4|4               |    0.000|    0.882\n",
      "   4|7               |    0.000|    0.882\n",
      "   2|6               |    0.000|    0.877\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   4|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   2|6               |    0.000|    0.877\n",
      "   0|1               |    0.000|    0.874\n",
      "   3|1               |    0.000|    0.878\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   2|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   4|6               |    0.000|    0.882\n",
      "   4|6               |    0.000|    0.882\n",
      "   4|6               |    0.000|    0.882\n",
      "   2|6               |    0.000|    0.877\n",
      "   2|6               |    0.000|    0.877\n",
      "   3|7               |    0.000|    0.878\n",
      "   3|7               |    0.000|    0.878\n",
      "   4|4               |    0.000|    0.882\n",
      "   2|0               |    0.000|    0.877\n",
      "   4|5               |    0.000|    0.882\n",
      "   3|6               |    0.000|    0.878\n",
      "   3|6               |    0.000|    0.880\n",
      "---------------------------\n",
      "Shuffle features and optimize!\n",
      "----------------------------------------\n",
      "   3|Feature         |   Weight|     NDCG\n",
      "----------------------------------------\n",
      "   3|0               |   -0.000|    0.880\n",
      "   3|3               |    0.000|    0.881\n",
      "   3|8               |   -1.350|    0.881\n",
      "   3|8               |   -0.950|    0.881\n",
      "   3|5               |    0.000|    0.881\n",
      "   3|6               |    0.000|    0.881\n",
      "   3|6               |    0.000|    0.881\n",
      "---------------------------\n",
      "Finished successfully.\n",
      "CPU times: user 30 s, sys: 221 ms, total: 30.2 s\n",
      "Wall time: 28.6 s\n"
     ]
    }
   ],
   "source": [
    "ltr_feats_all = ((bm25>>qe>>bm25)%50) >>   pt.apply.generic(lambda df: df.merge(dataset,on='docno'))>> (\n",
    "        pt.transformer.IdentityTransformer()\n",
    "        ** # new feature: TF-IDF indicator\n",
    "        pt.BatchRetrieve(index, wmodel=\"TF_IDF\")\n",
    "        ** #new feature: review number\n",
    "        (pt.apply.doc_score(lambda row: math.log(row['review_num'])))\n",
    "        ** #new feature: overall rate\n",
    "        (pt.apply.doc_score(lambda row: row['overall_rate']))\n",
    "        **# new feature: positive review title length\n",
    "        (pt.apply.doc_score(lambda row: len(row['pos_review_title']) ))\n",
    "        ** # new feature: state in query\n",
    "        (pt.apply.doc_score(lambda row: int( row[\"query_0\"] is not None and (row ['State'].lower()in row[\"query_0\"].lower()) )))\n",
    "        ** # negative relevance\n",
    "        (pt.text.scorer(body_attr=\"neg_review_content\", wmodel='BM25') >> (pt.apply.doc_score(lambda row: row['score']* senti[int(row['docid'])])))\n",
    "        **\n",
    "        (pt.apply.doc_score(lambda row: float(util.pytorch_cos_sim(model.encode(row[\"query_0\"], convert_to_tensor=True), model.encode(row[\"Description\"], convert_to_tensor=True)))))\n",
    "        **\n",
    "        (vbert)\n",
    "    )\n",
    "\n",
    "myfnames=[\"BM25\", \"TF-IDF\",'revienum','overall_rate','pos_reviewLen', 'State', 'Sentiment of neg_review', 'Sentense transformer', 'VBERT']\n",
    "\n",
    "import fastrank\n",
    "\n",
    "qrels['label'] = qrels['label'].apply(lambda x: float(x))\n",
    "train_request = fastrank.TrainRequest.coordinate_ascent()\n",
    "\n",
    "params = train_request.params\n",
    "params.init_random = True\n",
    "params.normalize = True\n",
    "params.seed = 1234567\n",
    "\n",
    "ca_pipe_all = ltr_feats_all >> pt.ltr.apply_learned_model(train_request, form='fastrank')\n",
    "\n",
    "%time ca_pipe_all.fit(train_data, qrels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-12-17 21:26:29,141][onir_pt][DEBUG] using GPU (deterministic)\n",
      "[2022-12-17 21:26:29,145][onir_pt][DEBUG] [starting] batches\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "batches:   0%|          | 0/100 [14ms<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-12-17 21:26:31,590][onir_pt][DEBUG] [finished] batches: [2.44s] [100it] [40.92it/s]\n",
      "[2022-12-17 21:26:46,246][onir_pt][DEBUG] using GPU (deterministic)\n",
      "[2022-12-17 21:26:46,250][onir_pt][DEBUG] [starting] batches\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "batches:   0%|          | 0/100 [14ms<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-12-17 21:26:48,692][onir_pt][DEBUG] [finished] batches: [2.44s] [100it] [40.95it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>AP(rel=3)</th>\n",
       "      <th>P(rel=3)@10</th>\n",
       "      <th>ndcg</th>\n",
       "      <th>ndcg_cut_5</th>\n",
       "      <th>ndcg_cut_10</th>\n",
       "      <th>mrt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BM25(Baseline)</td>\n",
       "      <td>0.527218</td>\n",
       "      <td>0.6250</td>\n",
       "      <td>0.778395</td>\n",
       "      <td>0.658551</td>\n",
       "      <td>0.698940</td>\n",
       "      <td>21.557884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pipe_basic(5 feature)</td>\n",
       "      <td>0.567570</td>\n",
       "      <td>0.6500</td>\n",
       "      <td>0.796696</td>\n",
       "      <td>0.704020</td>\n",
       "      <td>0.720504</td>\n",
       "      <td>80.271579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>pipe_basic+BERT</td>\n",
       "      <td>0.587849</td>\n",
       "      <td>0.7000</td>\n",
       "      <td>0.807200</td>\n",
       "      <td>0.775564</td>\n",
       "      <td>0.748184</td>\n",
       "      <td>385.148373</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>pipe_basic+Sentiment</td>\n",
       "      <td>0.586948</td>\n",
       "      <td>0.6750</td>\n",
       "      <td>0.807075</td>\n",
       "      <td>0.749246</td>\n",
       "      <td>0.742542</td>\n",
       "      <td>132.022660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pipe_basic+SentenceTransform</td>\n",
       "      <td>0.566731</td>\n",
       "      <td>0.6875</td>\n",
       "      <td>0.806556</td>\n",
       "      <td>0.768570</td>\n",
       "      <td>0.757627</td>\n",
       "      <td>829.645116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>pipe_baic+ALL features</td>\n",
       "      <td>0.577326</td>\n",
       "      <td>0.7000</td>\n",
       "      <td>0.810882</td>\n",
       "      <td>0.765056</td>\n",
       "      <td>0.763536</td>\n",
       "      <td>1182.609474</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           name  AP(rel=3)  P(rel=3)@10      ndcg  ndcg_cut_5  \\\n",
       "0                BM25(Baseline)   0.527218       0.6250  0.778395    0.658551   \n",
       "1         pipe_basic(5 feature)   0.567570       0.6500  0.796696    0.704020   \n",
       "2               pipe_basic+BERT   0.587849       0.7000  0.807200    0.775564   \n",
       "3          pipe_basic+Sentiment   0.586948       0.6750  0.807075    0.749246   \n",
       "4  pipe_basic+SentenceTransform   0.566731       0.6875  0.806556    0.768570   \n",
       "5        pipe_baic+ALL features   0.577326       0.7000  0.810882    0.765056   \n",
       "\n",
       "   ndcg_cut_10          mrt  \n",
       "0     0.698940    21.557884  \n",
       "1     0.720504    80.271579  \n",
       "2     0.748184   385.148373  \n",
       "3     0.742542   132.022660  \n",
       "4     0.757627   829.645116  \n",
       "5     0.763536  1182.609474  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qrels['label'] = qrels['label'].apply(lambda x: int(x))\n",
    "from pyterrier.measures import *\n",
    "pt.Experiment(\n",
    "    [pt.BatchRetrieve(index,wmodel='BM25')%50, ca_pipe1, ca_pipe2,\n",
    "     ca_pipe3, ca_pipe4, ca_pipe_all],\n",
    "    test_data,\n",
    "    qrels,\n",
    "    names=['BM25(Baseline)','pipe_basic(5 feature)', 'pipe_basic+BERT', 'pipe_basic+Sentiment', 'pipe_basic+SentenceTransform', 'pipe_baic+ALL features'],\n",
    "    eval_metrics=[AP(rel=3), P(rel=3)@10, \"ndcg\", \"ndcg_cut_5\", \"ndcg_cut_10\", 'mrt']\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What features matters in ltr_feats_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-12-17 21:26:55,722][onir_pt][DEBUG] using GPU (deterministic)\n",
      "[2022-12-17 21:26:55,729][onir_pt][DEBUG] [starting] batches\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "batches:   0%|          | 0/100 [16ms<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-12-17 21:26:58,155][onir_pt][DEBUG] [finished] batches: [2.43s] [100it] [41.23it/s]\n",
      "[2022-12-17 21:27:05,174][onir_pt][DEBUG] using GPU (deterministic)\n",
      "[2022-12-17 21:27:05,179][onir_pt][DEBUG] [starting] batches\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "batches:   0%|          | 0/100 [15ms<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-12-17 21:27:07,607][onir_pt][DEBUG] [finished] batches: [2.43s] [100it] [41.21it/s]\n",
      "[2022-12-17 21:27:14,570][onir_pt][DEBUG] using GPU (deterministic)\n",
      "[2022-12-17 21:27:14,575][onir_pt][DEBUG] [starting] batches\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "batches:   0%|          | 0/100 [14ms<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-12-17 21:27:16,999][onir_pt][DEBUG] [finished] batches: [2.42s] [100it] [41.27it/s]\n",
      "[2022-12-17 21:27:23,967][onir_pt][DEBUG] using GPU (deterministic)\n",
      "[2022-12-17 21:27:23,971][onir_pt][DEBUG] [starting] batches\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "batches:   0%|          | 0/100 [14ms<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-12-17 21:27:26,391][onir_pt][DEBUG] [finished] batches: [2.42s] [100it] [41.34it/s]\n",
      "[2022-12-17 21:27:33,352][onir_pt][DEBUG] using GPU (deterministic)\n",
      "[2022-12-17 21:27:33,356][onir_pt][DEBUG] [starting] batches\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "batches:   0%|          | 0/100 [14ms<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-12-17 21:27:35,775][onir_pt][DEBUG] [finished] batches: [2.42s] [100it] [41.35it/s]\n",
      "[2022-12-17 21:27:42,701][onir_pt][DEBUG] using GPU (deterministic)\n",
      "[2022-12-17 21:27:42,706][onir_pt][DEBUG] [starting] batches\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "batches:   0%|          | 0/100 [14ms<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-12-17 21:27:45,129][onir_pt][DEBUG] [finished] batches: [2.42s] [100it] [41.28it/s]\n",
      "[2022-12-17 21:27:53,319][onir_pt][DEBUG] using GPU (deterministic)\n",
      "[2022-12-17 21:27:53,325][onir_pt][DEBUG] [starting] batches\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "batches:   0%|          | 0/100 [15ms<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-12-17 21:27:56,146][onir_pt][DEBUG] [finished] batches: [2.82s] [100it] [35.47it/s]\n",
      "[2022-12-17 21:28:04,527][onir_pt][DEBUG] using GPU (deterministic)\n",
      "[2022-12-17 21:28:04,532][onir_pt][DEBUG] [starting] batches\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "batches:   0%|          | 0/100 [15ms<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-12-17 21:28:07,358][onir_pt][DEBUG] [finished] batches: [2.82s] [100it] [35.41it/s]\n",
      "[2022-12-17 21:28:15,568][onir_pt][DEBUG] using GPU (deterministic)\n",
      "[2022-12-17 21:28:15,572][onir_pt][DEBUG] [starting] batches\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "batches:   0%|          | 0/100 [15ms<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-12-17 21:28:18,012][onir_pt][DEBUG] [finished] batches: [2.44s] [100it] [41.00it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>AP(rel=3)</th>\n",
       "      <th>P(rel=3)@10</th>\n",
       "      <th>ndcg</th>\n",
       "      <th>ndcg_cut_5</th>\n",
       "      <th>ndcg_cut_10</th>\n",
       "      <th>mrt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BM25</td>\n",
       "      <td>0.510320</td>\n",
       "      <td>0.6250</td>\n",
       "      <td>0.780160</td>\n",
       "      <td>0.654993</td>\n",
       "      <td>0.684473</td>\n",
       "      <td>1179.999167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TF-IDF</td>\n",
       "      <td>0.573852</td>\n",
       "      <td>0.6500</td>\n",
       "      <td>0.799244</td>\n",
       "      <td>0.708145</td>\n",
       "      <td>0.718224</td>\n",
       "      <td>1180.269699</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>revienum</td>\n",
       "      <td>0.480125</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.761884</td>\n",
       "      <td>0.583944</td>\n",
       "      <td>0.605530</td>\n",
       "      <td>1173.429900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>overall_rate</td>\n",
       "      <td>0.525961</td>\n",
       "      <td>0.6250</td>\n",
       "      <td>0.773886</td>\n",
       "      <td>0.606003</td>\n",
       "      <td>0.659752</td>\n",
       "      <td>1173.073315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pos_reviewLen</td>\n",
       "      <td>0.547511</td>\n",
       "      <td>0.6750</td>\n",
       "      <td>0.789753</td>\n",
       "      <td>0.718946</td>\n",
       "      <td>0.710416</td>\n",
       "      <td>1172.590542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>State</td>\n",
       "      <td>0.508380</td>\n",
       "      <td>0.4875</td>\n",
       "      <td>0.768920</td>\n",
       "      <td>0.600673</td>\n",
       "      <td>0.600827</td>\n",
       "      <td>1168.038791</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Sentiment of neg_review</td>\n",
       "      <td>0.499994</td>\n",
       "      <td>0.6000</td>\n",
       "      <td>0.763875</td>\n",
       "      <td>0.622076</td>\n",
       "      <td>0.635395</td>\n",
       "      <td>1380.136042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Sentense transformer</td>\n",
       "      <td>0.534188</td>\n",
       "      <td>0.6750</td>\n",
       "      <td>0.786907</td>\n",
       "      <td>0.698437</td>\n",
       "      <td>0.707197</td>\n",
       "      <td>1401.549737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>VBERT</td>\n",
       "      <td>0.484065</td>\n",
       "      <td>0.5500</td>\n",
       "      <td>0.761977</td>\n",
       "      <td>0.596652</td>\n",
       "      <td>0.624728</td>\n",
       "      <td>1326.604957</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      name  AP(rel=3)  P(rel=3)@10      ndcg  ndcg_cut_5  \\\n",
       "0                     BM25   0.510320       0.6250  0.780160    0.654993   \n",
       "1                   TF-IDF   0.573852       0.6500  0.799244    0.708145   \n",
       "2                 revienum   0.480125       0.5000  0.761884    0.583944   \n",
       "3             overall_rate   0.525961       0.6250  0.773886    0.606003   \n",
       "4            pos_reviewLen   0.547511       0.6750  0.789753    0.718946   \n",
       "5                    State   0.508380       0.4875  0.768920    0.600673   \n",
       "6  Sentiment of neg_review   0.499994       0.6000  0.763875    0.622076   \n",
       "7     Sentense transformer   0.534188       0.6750  0.786907    0.698437   \n",
       "8                    VBERT   0.484065       0.5500  0.761977    0.596652   \n",
       "\n",
       "   ndcg_cut_10          mrt  \n",
       "0     0.684473  1179.999167  \n",
       "1     0.718224  1180.269699  \n",
       "2     0.605530  1173.429900  \n",
       "3     0.659752  1173.073315  \n",
       "4     0.710416  1172.590542  \n",
       "5     0.600827  1168.038791  \n",
       "6     0.635395  1380.136042  \n",
       "7     0.707197  1401.549737  \n",
       "8     0.624728  1326.604957  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pt.Experiment(\n",
    "    [ltr_feats_all >> pt.ltr.feature_to_score(i) for i in range(len(myfnames))],\n",
    "    test_data,\n",
    "    qrels, \n",
    "    names=myfnames,\n",
    "    eval_metrics=[AP(rel=3), P(rel=3)@10, \"ndcg\", \"ndcg_cut_5\", \"ndcg_cut_10\", 'mrt'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Weight comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-12-17 21:28:38,888][onir_pt][DEBUG] using GPU (deterministic)\n",
      "[2022-12-17 21:28:38,893][onir_pt][DEBUG] [starting] batches\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "batches:   0%|          | 0/300 [14ms<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-12-17 21:28:46,933][onir_pt][DEBUG] [finished] batches: [8.04s] [300it] [37.32it/s]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=2)]: Using backend ThreadingBackend with 2 concurrent workers.\n",
      "[Parallel(n_jobs=2)]: Done  46 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=2)]: Done 196 tasks      | elapsed:    0.5s\n",
      "[Parallel(n_jobs=2)]: Done 400 out of 400 | elapsed:    1.0s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 30.6 s, sys: 275 ms, total: 30.9 s\n",
      "Wall time: 29.9 s\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "rf = RandomForestRegressor(n_estimators=400, verbose=1, random_state=SEED, n_jobs=2)\n",
    "\n",
    "rf_pipe = ltr_feats_all >> pt.ltr.apply_learned_model(rf)\n",
    "\n",
    "%time rf_pipe.fit(train_data, qrels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_79976/1284213065.py:18: UserWarning: Matplotlib is currently using module://matplotlib_inline.backend_inline, which is a non-GUI backend, so cannot show the figure.\n",
      "  fig.show()\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlwAAAHaCAYAAADVHZfoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABUsElEQVR4nO3deZwcVbn/8c+XhLBDWMIeCMiiEZElsqgoKshO0CsCgsiiCIKCghpXUFHQn6IiSEThIooiImoUFPHKIldA4sIO18hiwhpEdgUCz++P5zQpmplkprtrZrrn+369+pXuquo6p3q6nzx16tQ5igjMzMzMrD6LDHcFzMzMzHqdEy4zMzOzmjnhMjMzM6uZEy4zMzOzmjnhMjMzM6uZEy4zMzOzmjnhspZJCknrlefTJX1quOtkZjZYko6T9P3hrof1NidcXU7SOyTNlPS4pHsl/UrSa4e6HhFxaER8rt39SJpUErmxbe5nqfKZXNRundpVTUzNbGAk3Snp3+V3fJ+ksyQtPdz1aoekbSU9V46p8fjFEJbfkfhqrXHC1cUkfQj4GvAFYBVgLeCbwNQOl9ONP863AU8Bb5a02nBXxsxasltELA1sAmwKfGx4q9MR90TE0pXHboPdgaQxdVTM6uWEq0tJWg74LHB4RFwQEU9ExDMR8YuI+HDZZjFJX5N0T3l8TdJilX28R9IsSQ9JmiFp9cq6kHS4pL8BfyvLPlxa0e6RdFBTfc6SdHx5vq2kOZKOlvRAec+BlW13kfQXSY9Kmi3puMqurij/PlzO/rYu7zlI0i2S/iXpYklrL+QjehcwHbge2Leprh+VdLekxyTdJulNZfkYSR+X9Pey7k+SJpZ1L5V0SfmsbpP09qZjP1XSheV910h6SVnXOJ7ryvHstZB6m1mTiLgPuJhMvACQNK3yW71Z0lsq6w6QdKWkL5eYcYeknSrr15F0eXnvJcBK1fIk7S7pJkkPS7pM0ssq6+4ssfB6SU9IOkPSKuXqwmOSfitp+cEeo6SXlbIeLmXvXll3lqTTJF0k6QngDZJWl/QTSXPL8X2gsv0Wyisfj0q6X9JJZdWL4quk9cpn8YikByX9aLB1twGKCD+68AHsCMwDxi5gm88CVwMrAxOAPwCfK+veCDwIbAYsBnwDuKLy3gAuAVYAlijl3Q9sBCwF/KBss17Z/izg+PJ821K3zwKLAjsDTwLLV9a/gkz4Ny773aOsm1T2O7ZSlz2AWcDLgLHAJ4E/LOC41wKeAyYDRwPXV9ZtCMwGVq+U95Ly/MPADWUbAa8EVizHOxs4sJS/WfnsXl459oeALcr6c4Bzmz7L9Yb7O+OHH930AO4EtivP1yy/za9X1u8JrF7iyF7AE8BqZd0BwDPAe4AxwGHAPYDK+quAk0rsex3wGPD9sm6Dsq/tS/z6SIk/4yr1upq8qrAG8ADwZ7IFbjHgd8Cx/RzTtsCcPpYvWsr4ODCOjM+PARuW9WcBjwCvKce7JPAn4NNl+3WB24EdKsf3zvJ8aWCr8nwSL46vPwQ+Ufa7OPDa4f7b9+pj2CvgR4t/uGy1uW8h2/wd2LnyegfgzvL8DOBLlXVLlwA1qbwO4I2V9WcCJ1Zeb8CCE65/N/2oH2j86Puo59eAr5bnfQWEXwEHV14vQiZwa/ezv08Cfy3PVweeBTYtr9crddkOWLTpfbcBU/vY317A75uWfasRVMuxf6eybmfg1sprJ1x++DHIB5nYPF4SjwD+Bxi/gO3/2vj9kgnXrMq6Jcs+ViVPyOYBS1XW/4D5CdengPMq6xYB7ga2rdRr38r6nwCnVV6/H/hZP3XcljwZfLjyeDuwDXAfsEhl2x8Cx5XnZwFnV9ZtCfyjad8fA/67PL8C+AywUtM2fcXXs4HTgTWH+2/e6w9fUuxe/wRW0oL7V60O3FV5fVdZ9qJ1EfF42ecale1nN+2r+rq63z7rFxHzKq+fJJM6JG0p6dLSFP4IcChNTfpN1ga+XpraHyZbk9RU16r9yVYmIuIe4HLyEiMRMQs4CjgOeEDSuZVLqRPJJLWv8rdslF/qsC8ZvBvu6+tYzawte0TEMmSi8lIqcULS/pL+WvlNbsQL48jzv8mIeLI8XZqMZf+KiCcq21bjWXNsfI6MfdV4c3/l+b/7eL2g3/89ETG+8jivlDm7lFWtU3/xeG1g9aaY9HGy1Q3gYPKk+FZJ10radQH1+QgZT/9YLmUetIBtrQ1OuLrXVcB/yMtt/bmH/GE2rFWWvWidpKXIy2d3V7aPyvN7yYSkuq9W/QCYAUyMiOXIvlbqo8yG2cB7m4LUEhHxh+YNJb0aWB/4mPLOpvvIs8F9GslpRPwgIl5LHn8AX6yU85J+yr+8qfylI+KwVj8AMxu4iLicbOX5MkDpw/lt4AhgxYgYD9zI/DiyIPcCy5eY11CNZ82xUWTsq8bGTrsHmCip+n/yWvQfj2cDdzTFpGUiYmeAiPhbROxDdif5InB+Od4XxdeIuC8i3hMRqwPvBb4p31VdCydcXSoiHiGv358qaQ9JS0paVNJOkr5UNvsh8ElJEyStVLZvjDXzA+BASZsoO9J/AbgmIu7sp8jzgAMkTZa0JHBsG9VfBngoIv4jaQvgHZV1c8km93Ury6aTCdTLIW8YkLRnP/t+F9n3bDLZwXYT8sx3SWAnSRtKemM55v+QZ6PPlvd+B/icpPWVNpa0IvBLYANJ7yyf8aKSXlXtSLsQ9zcdj5kN3teA7SVtQvarDDJeoLwpZ6OB7CQi7gJmAp+RNE45jE71TsHzgF0kvUnSomQ/0KfIPrB1uYbsN/aREl+2LXU6t5/t/wg8qrwBaAnlDT8bSXoVgKT9JE0oLWYPl/c8Sx/xVdKektYsL/9Ffq6NmGgd5ISri0XEScCHyD5Lc8mzniOAn5VNjicDy/Vkh9M/l2VExP+QfRV+Qp7xvQTYewFl/YoMeL8jO3f+ro2qvw/4rKTHyCTwvEo5TwKfB/63NJVvFRE/Jc/SzpX0KHkmu1PzTiUtTvaH+EY5a2s87gC+RyZjiwEnkp3e7yPPAD9ednFSqctvgEfJfm5LRMRjwJvJz+ee8r4vln0NxHHAd8vxvH1hG5vZi0XEXLK/0aci4mbgK2RL//3kTTj/O4jdvYNs+X6IPHk8u1LObcB+5I1ED5KJz24R8XQHDqNPZd+7k3HtQXJ4n/0j4tZ+tn+21GsT4I7ynu8Ay5VNdgRukvQ48HVg74j4T1/xFXgVcE3ZdgZwZImZ1mGNOzbMzMzMrCZu4TIzMzOrmRMuMzMzs5o54TIzMzOrmRMuMzMzs5o54bJRQ9JaZf6whU78KmmScj7Jbpy428zMRhgnXD2uJA1PlETj8TIicSf2OSQD45XJVR+tJkmSvt3PsukL2ldE/KMMWNr2GDOSjpP0/YVvaWbNuj0ulfLOknT8UJW3II5H3cEJ1+jwypJoLF1GZB5WA2lhqphJTj67WWXZNuR4WNVlryPnDzOz7tDNcWnEcCt893DCNUpJWl3ST5TzGd4h6QOVdVtIuqoMjHevpFMkjSvrGknNdeXMdC9JB0i6smn/z59tljPB0yRdJOkJ4A0LKr8qIp4BriYTKiStDIwDftS0bAPgCkmLSJom6e+S/inpPEkrlO1ecJlQ0jqSrpD0mKTfSjq1j7PEfSX9Q9KDkj5R3rcjOVjqXuUzuK4sP0DS7WV/d0jat5W/jdlo1S1xqY96N2LLgZJmS/qXpEOVM1JcX+p8SmX7AyT9r6RvSHpE0q2S3tT0OcyQ9JCkWZLeU1l3nKTzJX1fORD0ofQdjw6UdEuJR7dLem9lH9tKmiPpaEkPlM/zwMr6JSR9RdJdpX5XSlqirNtK0h/KMV2nHBXfBmK4Z8/2o94HOU3Dek3LFgH+RI7yPo6c5uF2YIeyfnNgK2AsObv8LcBR/e0TOAC4sr9yyTnQHgFeU8peckHl93EMxwI/L8/fRo4KvX3TstvL86PIBG1NciT4bwE/LOsmlXqNLa+vIudmGwe8lhxd/vtN234bWAJ4JTm9x8vK+uMa25bXS5X3b1herwa8fLj//n74MRIfPRKXzgKOL88b8WI6sDg5M8V/yFk/ViYnoX4AeH2lbvOADwKLAnuVuqxQ1l9Ojja/ODma/FzgTWXdccAz5Dy6i5T49IJ4VLbbhZxBRMDrgSeBzcq6bUv5ny3l71zWL1/WnwpcVuo9Bng1GU/XAP5Ztl+EjMP/BCYM93eqGx5u4Rod/qz5s8qfTE7lMCEiPhsRT0fE7WRisTdARPwpIq6OiHmRcyt+i/zBtuPnEfG/kXN7vWJB5ffhcuC1kkReTvw9mSxtVVl2edn2vcAnImJORDxFBqK3qanZXdJa5XP4dKnDleS0Fs0+ExH/jojrgOvIxKs/zwEbSVoiIu6NiJsW8pmYjWbdHpf68rnIKXR+Q86N+MOIeCAi7ibj1qaVbR8AvhYRz0TEj4DbyDkcJ5IngB8t+/orOW3POyvvvSoifhYRz0XEv/uqSERcGBF/j3Q5OWXZNpVNngE+W8q/CHgc2FA5gfZB5BQ/d0fEsxHxhxJP9wMuioiLStmXkN0+dh7EZzRq+drv6LBZRMxqvFDO57e6XthRdQwZEJC0ATmv4BTyrG8seebXjtmV52svqPw+XA0sTU5O+zrgtIh4XNLsyrKTK/v+qaTnKu9/FlilaZ+rkxNoP9lUx4lN291Xef5kqceLRMQTkvYCjgHOkPS/wNHRz1xoZtb1cakv91ee/7uP19X4cXdEVOfWu4uMS43Y9FjTuin91LtPknYirw5swPwWvBsqm/wzIuZVXjfi20pky9rf+9jt2sCekqqTfS8KXLqw+pj7cI1Ws4E7ImJ85bFMRDTOUk4DbgXWj4hlyf4BWsD+niB/zABIWrWPbaqBZWHlv/CNEf8BrgV2BVarJDG/L8s2Zn6H+dnATk37XrycYVbdC6wgacnKsuZka0FeNAlpRFwcEduTlxNvJc+OzWxguioudcAapYW+YS3yZqB7yNi0TNO6agxrjj8veC1pMeAnZJeJVSJvSriIBX9eDQ+Sl0Nf0se62cD3mj6jpSLixAHsd9RzwjU6/RF4VNJHS+fIMZI2kvSqsn4Zsj/S45JeChzW9P77yf4NDdcBL5e0iaTFyct47ZTflyvI/ll/qCy7siy7LyIaZ2PTgc9LWhtA0gRJU5t3FhF3kU3hx0kaJ2lrYLfm7RbgfmBSaX5H0iqSdpe0FNnX63GyZc3MBqYb41I7VgY+IGlRSXsCLyMv180m49wJkhaXtDFwMHDOAvb1gnhE9kFbjOz7Na+0dr15IJUql1fPBE4qnffHKIfnWQz4PrCbpB3K8sVLB/w1B3/4o48TrlEochyq3cjOmHeQZzTfAZYrmxwDvAN4jGyl+VHTLo4Dvlv6Xrw9Iv6P7Hz5W+BvZCLUTvl9uZwMUNV9X1mWVYeD+DrZF+s3kh4jL0du2c8+9wW2Jjt9Hl+O86kF1b3ix+Xff0r6M/lbOpo8O32I7FvyvgHuy2zU69K41I5rgPVLOZ8H3hYR/yzr9iE74t8D/BQ4tvSX6s8L4lG5HPkB4DzgX+Tn1lcf1f4cQ15+vJaMZ18EFinJ4FSydXEu2eL1YZxLDIheeAnZbPSS9CPg1og4drjrYma9S9IBwLsj4rXDXRcbOs5KbdRSjpHzEuXYXTuSZ24/G+ZqmZlZD/JdijaarQpcAKwIzAEOi4i/DG+VzMysF/mSopmZmVnNfEnRzMzMrGZOuMzMzMxqNqL7cK200koxadKk4a6GmQ2hP/3pTw9GxIThrke7HL/MRp8Fxa8RnXBNmjSJmTNnDnc1zGwISbpruOvQCY5fZqPPguKXLymamZmZ1cwJl5mZmVnNnHCZmZmZ1cwJl5mZmVnNnHCZmZmZ1cwJl5mZmVnNnHCZmZmZ1cwJl5mZmVnNnHCZmZmZ1cwJl5mZmVnNnHCZmZmZ1cwJl5mZmVnNnHCZmZmZ1WzscFfAzEaOSdMu7Pg+7zxxl47v08ys2zjhMjMzGwCfkFg7fEnRzMzMrGZOuMzMzMxq5oTLzMzMrGZOuMzMzMxq5oTLzMzMrGZOuMzMzMxq5oTLzMzMrGZOuMzMzMxq5oTLzMzMrGZOuMzMzMxq5oTLzMzMrGZOuMzMzMxq5oTLzMzMrGZOuMzMzMxq5oTLzEYNSTtKuk3SLEnT+lj/UklXSXpK0jFN68ZLOl/SrZJukbT10NXczLrd2OGugJnZUJA0BjgV2B6YA1wraUZE3FzZ7CHgA8Aefezi68CvI+JtksYBS9ZcZTPrIW7hMrPRYgtgVkTcHhFPA+cCU6sbRMQDEXEt8Ex1uaRlgdcBZ5Ttno6Ih4ek1mbWE5xwmdlosQYwu/J6Tlk2EOsCc4H/lvQXSd+RtFSnK2hmvcsJl5mNFupjWQzwvWOBzYDTImJT4Amgrz5gh0iaKWnm3LlzW6+pmfUc9+Eys9FiDjCx8npN4J5BvHdORFxTXp9PHwlXRJwOnA4wZcqUgSZzQ2rStAs7vs87T9yl4/s06zVu4TKz0eJaYH1J65RO73sDMwbyxoi4D5gtacOy6E3AzQt4i5nZC7iFy8xGhYiYJ+kI4GJgDHBmRNwk6dCyfrqkVYGZwLLAc5KOAiZHxKPA+4FzSrJ2O3BgJ+vnliez3uaEy8xGjYi4CLioadn0yvP7yEuNfb33r8CUOutnZr3LlxTNzMzMajZkCZekdSWdIen8oSrTzMzMbCQYUMIl6UxJD0i6sWn5AqfJqCqDDR7cTmXNzMzMutFA+3CdBZwCnN1Y0N80GWRn1BOa3n9QRDzQdm3NzMzMutCAEq6IuELSpKbFz0+TASDpXGBqRJwA7NrRWpqZmZl1sXb6cA1qmgxJK0qaDmwq6WML2M4jNZuZmVlPaWdYiEFNkxER/wQOXdhOu2GkZjMzM7PBaKeFq51pMszMzMxGjXZauJ6fJgO4m5wm4x0dqZWZmZn1BM+ikAY6LMQPgauADSXNkXRwRMwDGtNk3AKcFxE31VdVMzMzs+400LsU9+ln+YumyTAzMzOzF/LUPmZmZmY1c8JlZmZmVjMnXGZmZmY1c8JlZmZmVjMnXGZmZmY1c8JlZmZmVjMnXGZmZmY1c8JlZmZmVjMnXGZmZmY1c8JlZmZmVjMnXGZmZmY1c8JlZmZmVjMnXGZmZmY1c8JlZmZmVjMnXGZmZmY1c8JlZmZmVjMnXGY2akjaUdJtkmZJmtbH+pdKukrSU5KO6WP9GEl/kfTLoamxmfUKJ1xmNipIGgOcCuwETAb2kTS5abOHgA8AX+5nN0cCt9RWSTPrWU64zGy02AKYFRG3R8TTwLnA1OoGEfFARFwLPNP8ZklrArsA3xmKyppZb3HCZWajxRrA7MrrOWXZQH0N+AjwXAfrZGajhBMuMxst1MeyGNAbpV2BByLiTwvZ7hBJMyXNnDt3bit1NLMe5YTLzEaLOcDEyus1gXsG+N7XALtLupO8FPlGSd9v3igiTo+IKRExZcKECe3W18x6iBMuMxstrgXWl7SOpHHA3sCMgbwxIj4WEWtGxKTyvt9FxH71VdXMes3Y4a6AmdlQiIh5ko4ALgbGAGdGxE2SDi3rp0taFZgJLAs8J+koYHJEPDpc9Taz3uCEy8xGjYi4CLioadn0yvP7yEuNC9rHZcBlNVTPzHqYLymamZmZ1cwJl5mZmVnNnHCZmZmZ1cwJl5mZmVnNnHCZmZmZ1cwJl5mZmVnNnHCZmZmZ1cwJl5mZmVnNnHCZmZmZ1cwJl5mZmVnNnHCZmZmZ1cwJl5mZmVnNnHCZmZmZ1cwJl5mZmVnNnHCZmZmZ1cwJl5mZmVnNnHCZmZmZ1WzscFfAzMx6z6RpF3Z8n3eeuEvH92k2VNzCZWZmZlYzJ1xmZmZmNXPCZWZmZlYzJ1xmZmZmNXPCZWZmZlYzJ1xmZmZmNXPCZWZmZlYzJ1xmZmZmNRuyhEvSyyRNl3S+pMOGqlwzMzOz4TaghEvSmZIekHRj0/IdJd0maZakaQvaR0TcEhGHAm8HprReZTMzM7PuMtAWrrOAHasLJI0BTgV2AiYD+0iaLOkVkn7Z9Fi5vGd34Ergfzp2BGZmA7Swk0RJL5V0laSnJB1TWT5R0qWSbpF0k6Qjh7bmZtbtBjSXYkRcIWlS0+ItgFkRcTuApHOBqRFxArBrP/uZAcyQdCHwg5ZrbWY2SJWTxO2BOcC1kmZExM2VzR4CPgDs0fT2ecDREfFnScsAf5J0SdN7zcz61U4frjWA2ZXXc8qyPknaVtLJkr4FXLSA7Q6RNFPSzLlz57ZRPTOzF3j+JDEingbOBaZWN4iIByLiWuCZpuX3RsSfy/PHgFtYQLwzM2s2oBaufqiPZdHfxhFxGXDZwnYaEacDpwNMmTKl3/2ZmQ1SXyeJWw52J6W1f1Pgmj7WHQIcArDWWmu1VEkz603ttHDNASZWXq8J3NNedczMajOok8Q+dyAtDfwEOCoiHn3RziJOj4gpETFlwoQJLVbTzHpROwnXtcD6ktaRNA7YG5jRmWqZmXVcWyeJkhYlk61zIuKCDtfNzHrcQIeF+CFwFbChpDmSDo6IecARwMVkf4bzIuKm+qpqZtaWlk8SJQk4A7glIk6qsY5m1qMGepfiPv0sv4gFdIA3MxspImKepMZJ4hjgzIi4SdKhZf10SasCM4FlgeckHUUOe7Mx8E7gBkl/Lbv8eImBZmYL1U6neTOzrtLXSWJETK88v4+81NjsSvruA2ZmNiCeS9HMzMysZk64zMzMzGrmhMvMzMysZk64zMzMzGrmhMvMzMysZk64zMzMzGrmhMvMzMysZk64zMzMzGrmhMvMzMysZk64zMzMzGrmhMvMzMysZk64zMzMzGrmhMvMzMysZk64zMzMzGrmhMvMzMysZk64zMzMzGrmhMvMzMysZk64zMzMzGrmhMvMzMysZmOHuwJmtnCTpl3Y8X3eeeIuHd+nmZn1zS1cZmZmZjVzwmVmZmZWMydcZmZmZjVzwmVmZmZWMydcZmZmZjVzwmVmZmZWMydcZmZmZjVzwmVmo4akHSXdJmmWpGl9rH+ppKskPSXpmMG818xsQZxwmdmoIGkMcCqwEzAZ2EfS5KbNHgI+AHy5hfeamfXLCZeZjRZbALMi4vaIeBo4F5ha3SAiHoiIa4FnBvteM7MFccJlZqPFGsDsyus5ZVnd7zUzc8JlZqOG+lgWnXyvpEMkzZQ0c+7cuYOqnJn1NidcZjZazAEmVl6vCdzTyfdGxOkRMSUipkyYMKHlippZ73HCZWajxbXA+pLWkTQO2BuYMQTvNTNj7HBXwMxsKETEPElHABcDY4AzI+ImSYeW9dMlrQrMBJYFnpN0FDA5Ih7t673DciBm1pWccJnZqBERFwEXNS2bXnl+H3m5cEDvNTMbKF9SNDMzM6uZEy4zMzOzmjnhMjMzM6uZEy4zMzOzmjnhMjMzM6uZEy4zMzOzmjnhMjMzM6uZEy4zMzOzmjnhMjMzM6uZEy4zMzOzmjnhMjMzM6uZEy4zMzOzmnnyamPStAs7vs87T9yl4/s0MzPrVkPWwiVpW0m/lzRd0rZDVa6ZmZnZcBtQwiXpTEkPSLqxafmOkm6TNEvStIXsJoDHgcWBOa1V18zMzKz7DPSS4lnAKcDZjQWSxgCnAtuTCdS1kmYAY4ATmt5/EPD7iLhc0irAScC+7VXdzMzMrDsMKOGKiCskTWpavAUwKyJuB5B0LjA1Ik4Adl3A7v4FLNZCXc3MzMy6Ujud5tcAZldezwG27G9jSW8FdgDGk61l/W13CHAIwFprrdVG9czMzMxGhnYSLvWxLPrbOCIuAC5Y2E4j4nTgdIApU6b0uz8zMzOzbtHOXYpzgImV12sC97RXHTMzM7Pe007CdS2wvqR1JI0D9gZmdKZaZmZmZr1joMNC/BC4CthQ0hxJB0fEPOAI4GLgFuC8iLipvqqamZmZdaeB3qW4Tz/LLwIu6miNzMzMzHqM51I0MzMzq5kTLjMzM7OaOeEyMzMzq1k743CNWpOmXdjxfd554i4d36eZmZmNDG7hMjMzM6uZEy4zGzUk7SjpNkmzJE3rY70knVzWXy9ps8q6D0q6SdKNkn4oafGhrb2ZdTMnXGY2KkgaA5wK7ARMBvaRNLlps52A9cvjEOC08t41gA8AUyJiI2AMOdizmdmAOOEys9FiC2BWRNweEU8D5wJTm7aZCpwd6WpgvKTVyrqxwBKSxgJL4qnMzGwQnHCZ2WixBjC78npOWbbQbSLibuDLwD+Ae4FHIuI3zQVIOkTSTEkz586d29HKm1l3c8JlZqOF+lgWA9lG0vJk69c6wOrAUpL2e9GGEadHxJSImDJhwoS2K2xmvcMJl5mNFnOAiZXXa/Liy4L9bbMdcEdEzI2IZ4ALgFfXWFcz6zFOuMxstLgWWF/SOpLGkZ3eZzRtMwPYv9ytuBV56fBe8lLiVpKWlCTgTcAtQ1l5M+tuHvjUzEaFiJgn6QjgYvIuwzMj4iZJh5b104GLgJ2BWcCTwIFl3TWSzgf+DMwD/gKcPvRHYWbdygmXmY0aEXERmVRVl02vPA/g8H7eeyxwbK0VNLOe5UuKZmZmZjVzwmVmZmZWMydcZmZmZjVzwmVmZmZWMydcZmZmZjXrmbsUJ027sOP7vPPEXTq+T+st/t6ZmdlAuIXLzMzMrGZOuMzMzMxq5oTLzMzMrGZOuMzMzMxq1jOd5s3MzGzgfNPP0HILl5mZmVnNnHCZmZmZ1cwJl5mZmVnN3IdrBPP1dTMzs97gFi4zMzOzmjnhMjMzM6uZEy4zMzOzmjnhMjMzM6uZEy4zMzOzmjnhMjMzM6uZEy4zMzOzmjnhMjMzM6uZBz41M7Ou5QGirVu4hcvMzMysZk64zMzMzGrmhMvMzMysZk64zGzUkLSjpNskzZI0rY/1knRyWX+9pM0q68ZLOl/SrZJukbT10NbezLqZO82b2aggaQxwKrA9MAe4VtKMiLi5stlOwPrlsSVwWvkX4OvAryPibZLGAUsOWeXNbKFG+g0UbuEys9FiC2BWRNweEU8D5wJTm7aZCpwd6WpgvKTVJC0LvA44AyAino6Ih4ew7mbW5ZxwmdlosQYwu/J6Tlk2kG3WBeYC/y3pL5K+I2mpOitrZr3FCZeZjRbqY1kMcJuxwGbAaRGxKfAE0FcfsEMkzZQ0c+7cue3W18x6iBMuMxst5gATK6/XBO4Z4DZzgDkRcU1Zfj6ZgL1ARJweEVMiYsqECRM6VnEz635OuMxstLgWWF/SOqXT+97AjKZtZgD7l7sVtwIeiYh7I+I+YLakDct2bwJuxsxsgHyXopmNChExT9IRwMXAGODMiLhJ0qFl/XTgImBnYBbwJHBgZRfvB84pydrtTevMzBbICZeZjRoRcRGZVFWXTa88D+Dwft77V2BKnfUzs941ZAmXpG2AfUuZkyPi1UNVtpmZmdlwGlAfLklnSnpA0o1Nyxc4anNVRPw+Ig4Ffgl8t/Uqm5mZmXWXgbZwnQWcApzdWNDfqM1k34gTmt5/UEQ8UJ6/A3h3G3U2MzMz6yoDSrgi4gpJk5oWPz9qM4Ckc4GpEXECsGtf+5G0FnnXz6OtV9nMzMysu7QzLMRARm1udjDw3wvawAMHmpmZWa9pJ+EayKjNL1wZcWxE/GEh23jgQDMzM+sp7SRcAxm12czMzGzUayfhGsiozWZmZmaj3kCHhfghcBWwoaQ5kg6OiHlAY9TmW4DzIuKm+qpqZmZm1p0GepfiPv0sf9GozWZmZmb2Qp682szMzKxmTrjMzMzMauaEy8zMzKxmTrjMzMzMauaEy8zMzKxmTrjMzMzMauaEy8zMzKxmTrjMzMzMauaEy8zMzKxmTrjMzMzMauaEy8zMzKxmTrjMzMzMauaEy8zMzKxmTrjMzMzMauaEy8zMzKxmTrjMzMzMauaEy8zMzKxmTrjMbNSQtKOk2yTNkjStj/WSdHJZf72kzZrWj5H0F0m/HLpam1kvcMJlZqOCpDHAqcBOwGRgH0mTmzbbCVi/PA4BTmtafyRwS81VNbMe5ITLzEaLLYBZEXF7RDwNnAtMbdpmKnB2pKuB8ZJWA5C0JrAL8J2hrLSZ9QYnXGY2WqwBzK68nlOWDXSbrwEfAZ6rqX5m1sOccJnZaKE+lsVAtpG0K/BARPxpgQVIh0iaKWnm3LlzW62nmfUgJ1xmNlrMASZWXq8J3DPAbV4D7C7pTvJS5Bslfb+5gIg4PSKmRMSUCRMmdLLuZtblnHCZ2WhxLbC+pHUkjQP2BmY0bTMD2L/crbgV8EhE3BsRH4uINSNiUnnf7yJivyGtvZl1tbHDXQEzs6EQEfMkHQFcDIwBzoyImyQdWtZPBy4CdgZmAU8CBw5Xfc2stzjhMrNRIyIuIpOq6rLplecBHL6QfVwGXFZD9cysh/mSopmZmVnNnHCZmZmZ1cwJl5mZmVnNnHCZmZmZ1cwJl5mZmVnNfJeimZnZCDJp2oUd3+edJ+7S8X3a4LiFy8zMzKxmTrjMzMzMauaEy8zMzKxmTrjMzMzMauaEy8zMzKxmTrjMzMzMauaEy8zMzKxmTrjMzMzMauaEy8zMzKxmTrjMzMzMauaEy8zMzKxmTrjMzMzMauaEy8zMzKxmTrjMzMzMauaEy8zMzKxmTrjMzMzMauaEy8zMzKxmTrjMzMzMauaEy8zMzKxmQ5ZwSZos6TxJp0l621CVa2ZmZjbcBpRwSTpT0gOSbmxavqOk2yTNkjRtIbvZCfhGRBwG7N9ifc3MzMy6ztgBbncWcApwdmOBpDHAqcD2wBzgWkkzgDHACU3vPwj4HnCspN2BFdurtpmZmVn3GFALV0RcATzUtHgLYFZE3B4RTwPnAlMj4oaI2LXp8UB5HA5MAx7s6FGYmQ3AwlrllU4u66+XtFlZPlHSpZJukXSTpCOHvvZm1s3a6cO1BjC78npOWdYnSZMknU62kv2/BWx3iKSZkmbOnTu3jeqZmc1XaZXfCZgM7CNpctNmOwHrl8chwGll+Tzg6Ih4GbAVcHgf7zUz69dALyn2RX0si/42jog7yQC2QBFxOnA6wJQpU/rdn5nZID3fKg8g6VxgKnBzZZupwNkREcDVksZLWi0i7gXuBYiIxyTdQp5g3oyZ2QC008I1B5hYeb0mcE971TEzq81AWuUXuo2kScCmwDWdr6KZ9ap2Eq5rgfUlrSNpHLA3MKMz1TIz67iBtMovcBtJSwM/AY6KiEdfVIC7RJhZPwY6LMQPgauADSXNkXRwRMwDjgAuBm4BzouIm+qrqplZWwbSKt/vNpIWJZOtcyLigr4KiIjTI2JKREyZMGFCxypuZt1vQH24ImKffpZfBFzU0RqZmdXj+VZ54G6yVf4dTdvMAI4o/bu2BB6JiHslCTgDuCUiThrKSptZb2in07yZWdeIiHmSGq3yY4AzI+ImSYeW9dPJE8idgVnAk8CB5e2vAd4J3CDpr2XZx8tJp5nZQjnhMrNRo69W+ZJoNZ4HcHgf77uSvvt3mZkNiCevNjMzM6uZEy4zMzOzmjnhMjMzM6uZEy4zMzOzmjnhMjMzM6uZEy4zMzOzmjnhMjMzM6uZEy4zMzOzmjnhMjMzM6uZEy4zMzOzmjnhMjMzM6uZEy4zMzOzmjnhMjMzM6uZEy4zMzOzmjnhMjMzM6uZEy4zMzOzmo0d7grY6DFp2oUd3+edJ+7S8X2amZl1mlu4zMzMzGrmhMvMzMysZk64zMzMzGrmhMvMzMysZk64zMzMzGrmhMvMzMysZk64zMzMzGrmhMvMzMysZk64zMzMzGrmhMvMzMysZk64zGzUkLSjpNskzZI0rY/1knRyWX+9pM0G+l4zswVxwmVmo4KkMcCpwE7AZGAfSZObNtsJWL88DgFOG8R7zcz65YTLzEaLLYBZEXF7RDwNnAtMbdpmKnB2pKuB8ZJWG+B7zcz65YTLzEaLNYDZlddzyrKBbDOQ95qZ9UsRMdx16JekucBdNex6JeDBGvbb6+UMZVkuZ2SXU2dZa0fEhE7vVNKewA4R8e7y+p3AFhHx/so2FwInRMSV5fX/AB8B1l3Ye8vyQ8hLkQAbArd1+jjwd8nl9G45Q1nWkMevsTUU1jF1BF0ASTMjYkod++7lcoayLJczsssZ6rI6ZA4wsfJ6TeCeAW4zbgDvJSJOB07vRGX74++Sy+nVcoayrOGIX76kaGajxbXA+pLWkTQO2BuY0bTNDGD/crfiVsAjEXHvAN9rZtavEd3CZWbWKRExT9IRwMXAGODMiLhJ0qFl/XTgImBnYBbwJHDggt47DIdhZl1qtCZctTb593A5Q1mWyxnZ5Qx1WR0REReRSVV12fTK8wAOH+h7h4m/Sy6nV8sZyrKGPH6N6E7zZmZmZr3AfbjMzMzMauaEq0Wl8+yY4a6HmVkrHMPMhpYTrhZI2pnsy7H6cNfFRg9JGqJy/B9xj3MMs6E2VPGrlDUiY1jPJVySXidp8xr3vzPwUeDIiJgtafEayxqyL6i1r+6/V+nQjaTDJR1VRxnl+30hHkV9WNQdv0oZjmHWpzr/XkMRv8r+R2wM66mES9KbgVOo4e7LMi7PqsAvgZ9ExG8krQucK2ntOr6olS/ojpI2k1TLQLDDqa4feH/7rbG8yeR3odY7f8t3/A3A92vY9w7A14D9I+IfknoqPox0dcavsn/HsA7rlfhV9l17DKszfpX9j+gY1jPDQkjaBfg6sHtE3FwGJ6RMNNvuvlUCx30lM/+opLuAI4BfRkQd0w81yj4Q+BhwB3CNpF9FxFUdLkMREZI2AZ4CiIhbOllGH2VtCMwt5T1RRxnl+V7kicW/gV9ExLN1lEOO2/QQsBPwi06V0VTeOsC7gFUj4sGybJGIeK4D+34z8B1yKppFASLiuaZjtJrUGb/K/ns2hjl+daYsao5hdcavsq8RH8NGVPbXpnWBdUuwWhQ4A/ixpGmSXtbmvhdvZP0RcTJwLPBT4KaI+Ho5c+zIZ1k9g5E0HtgA2Iqcn+05YDflCNgdUwLI7uS4JHsDX5X0hk6WAS8IVtsDvwa+B3ygA3+fF6gEqw8B7wXGkwH/bZ0uR9I2kk4CngVuAF7Rqf33cTb7D+C7wMOSji51eK7d756kN5ItKx8Evg0cLGm3sv+o86zanldn/IIejmGOX+2VVVcMG6r4VcrqihjWMwlXRHwD+LSkx4A/AX8ATgLWA/Zsdb+lifJM4AJJV0raJiK+A+wLvFPS5uUH0vYftOnM5mjyjPe/gFXKGej5wDPAPpK2aLe8RpmSVgOOBnYgJ/NcGrheHe50WL74W5GBYzfgC8CSwH6dCFqNH5WkRSStCEyJiDcCK5Bno+dLWrLdcppsTv5H8u7yer9yRt+2ynfhUEmfAT5OTjHzbWBdSUeW7Vo+Qyyfx73AgRFxPnAN+fvZXdKujXqMlIDVq+qKX9DbMczxqyNqiWFDEb/K/rsnhkVE1z6AbYD9gQMqyz4CHF95vRFwKbBcC/vfAbgJmEp+KT9HntkcVNa/F3gA2KrDx7UD8CtgE/Is6m+N+gOvJL+4E9rY/6KV54sDiwHfIpt7rwTWL+veAKzeoWMSOQHwBcDtleWvBj4LfBl4eRv7H195vl7592fl8/s5sFhZ9g5gow4czwbApmRw/x3wFWA74K/AncBrOvS5HQxcAaxD/kf13vI3260c2/va2PeOwFXAycCRje8FeefaoWRg3KWT320/XvD51xq/yvt7LoY5frUfv8q+ao9hdcavsv+uimHDXoE2PuhdgevIs4zzgW/3s91e5Ye/1CD3vy1wK7BtH1+g3wJblNfHkH0TFqeM3N/CsajyfPPyA/tyZdl04JbGjxIY18bnNgY4ANgaeD1wXgkm5wD3VX7s25JnI+u1+XdqzGbQCBhrlqD4nco22wAnABu0WgZ5dnZs+ZFdXpYfXf6Gm5bXBwI3Amu0eUxjyL4vlwNvBTYDfkTeFfNW8gyrraBYjmlR4BvAhuX4fl392wPbk/0hWtn/LsD/kv8Rvxc4DXh1Zf2a5FnvucAO7RyLH31+/rXGr/Lenothjl/tx6/K51hbDKs7fpX3d10MG/YKtPhBrwdcDWxZXm9AXhtesWm79wB/AV7RQhnvL2WsUb6c1YDyJeC3ldfjO3Rck8izqEb/ih0q675PNpOq1aBY2dcmwCPAXcwPuq8GvkmewR1Ufti7d+i4GndfHVnKXoO8bfebnfwMgdvJSwprltcbkre/3wacSvZPaOkslPmB96XAquUY1i2B6ZPlc9ujbNPqmfsifSz7NHmm+3Pmn719Btizjc9pebIvzXHl9WLk5av9m451jfJdWK0T3wM/nv/8a49f5f09GcMcv1ref60xbKjiV9lHV8awYa9ACx/0OGA5YL/KsqXJZsVtKsvGlz/2oLJ0YHIlEH4CmEFm/2p8ocim6u9V3tNWAlT2MQmYSZ5djC0/gK8Ab65s0/LZQGUfjWP4LnA3sFN5vTiwUvmBHw68qRPHRp6F3lwC4g3kLbsim3wvBc5otZzKj6pxTB8i/4P5WtN2mwKvAtZq81imlu/ZKWRz+JrAauXzuo7sFDqur8AzyHK2J8/gVwDeVPa7VVn3NvI/4Q1b3HcjmL+VDO6Nv/+55KWn84GzyVaKJTvx3fbjBZ9/rfGrvLdnY5jjV9t/o9pjWJ3xq+yja2PYsFdgkB/0zuWHNqmybGz59yxg4/L8VeXfQX3Q5Yf0hbKvKWXZJ8jsfPPKD+N95F1EY1v9Yza/rwSMPcim/t3JjP1jZN+EN7ZyPH2VB6xS+cy2JC8l7FdevxxYqUN/K5E3ZUwj+wVsTgbjiWX9UuTZx6va/fxKMFq/8vpyyiUass/DGzpwPGsDvyf/c/xYeb5SZf14yn9ybR7LgeSlkVPJvg+rAO8k/9P8WTm2lpr6ySb431UC1h7Ao+Qt4D8vf/8dybu9zqWpxcWPtr9Dtcavxnt6MYY5fnXkmGqJYUMVv8r+uzqGDXsFBvlhTyfHPDm1EkwaP8Qzyc6Ye5PN1i01IZLZ/WfI68GN5upGwFqT7OTaVtNuU3m7A8uW54uVL9Rl5Ue+JNm/YuUOlTW1BMMfAfuWZa8rQeuz5J0e27Sx/76alN9Wfth/ovQ9IO+6OqJDx3RM+RH/krwjamL5HC8HfkKOLTO5hf1OBL5Yeb0h2TFzP7LfwEvK8q1psUNz9ftbnq9Wvl9rldcfJZv7VyeD/xq0frlyB/Lsecfyesny705kZ9a3NG2/TCf+Pn684DOtPX6VffVkDHP8GvS+a49hQxW/yv66PoYNewUG+YFvXn5sR5NNolMq604rP/Jr2viCNs7+xgHHk30CqgHrerLZtaX9l/00NyP/jDwTaASspchm+L9RmsU79Nm9nuxEujIZ3G8CjirrXgkcBby+jf1PIDvjrtR0nK8izz6OKJ/rZmSw37mdz688/y/gN+X518m+DieVH/5YMhiv2cYxbUpeJmn8rX5E9rNo3AW1Pdk8P7EDx/K+8j34azmuxln8h4G/017fjS2A/1DuOgJeQv7n2zhb3wP4J5V+FYygZvheedQdv8p+ejKGOX61fFy1xbChil9lPz0Rw4a9AgP4oNemnO2VH/JPyCbqw0owaZwpfo48GxjUtWGamqCBMeXfceRZ07crZRxMpem3hWOpfkFXrzw/Hfgf5t82/V7gRCqXHtr8DJcB9inBY1cyqB9UAtgnaeOso1LG7uQlisOAFZrW7U+eWf0veYfP1ObPY6DHUXm+Qgkk65bP61fkGdw15J1LG7dxLGMrz88DLibvuJlavnOnlc/zJmC3Dnx2uzG/z8EZ5W+/RWX9EcA6bex/C/IOofeQnWUvBT5S/RuQZ/KzycsNIy5Qdeuj7vhV3tvTMczxq6XjGbIYVnf8KvvoiRg27BVYyIe8OXknwlVk57ulyGbS48k7Rz5BniluRN55MahAUn68lwPLV5Y1zgRWLP9+muxcuHmbx1INVEcAfyQDb6PfxreAS8gznetosdWkuTyyc+xPgCXK5/ddytltOa4f04HOmGV/U8vf43BefMfVumSz8mrV+g1i38sBHyD7CBxSAtRYst/IdOB1ZbtG/5W2rt2X79RnyvOfl89qabKPwBfIPhBvbvFYqt+F9clBDY8vr1chO+Z+icotzi0ew5KV55uTgfcR4NCm70jjOzjooQf8WODnX2v8KmX0ZAxz/Gq/71FdMWyo4lfZX0/FsGGvwEI+7KXKl+R+8gzqo+VH/TWyGXm1ErxOopzVDWLfO5CBsHGHw5jKuq3J24ob19M/QQfuECz73qMEkI3LcZwMvLasews5DkvLzf1NZW0N/DcvvPvpnLJsa+A3VM5E2ixr0xJU3kT2UTmC0m+jLPtX+Xu10hF4F7IFYFOy2fhuKn1CyEEU/8r8jqCT2jiOxg94E8odSOX1xeRZXLt3IDY3w7+bDMT3US6JACuSc4J9Dli8xXJ2KnX+BqWZnfmXRz5S2e4gshNqRzob+/GCv0Ft8avsv6djmONXy8dSWwwbqvhV9tNzMWzYK9DPB70SpVmXPLM5uXwR1wK+CMyhDKpHNsMO6mygBKqnmH/r8EvIJtZlyAz9Qlq8Rr+QcjcsP6xjy+tlgE+V49uhnR9CP+UdSZ5hv6GybH3yP4E/0H5TcvXH93XykoJKgPkG2QF4D+D/aHHcFeYPELkneUZ4HDmA4pGVbcaVH93ZtN9XYIny7/JkR9ZPVtZdBvy0Q3+bPcj/fBsdTA8i+9dsW16vQE6H0sq+dyAvTbyrBMTqAJRTgB+S/Yj2LN+Djoxc7cfzn3Gt8avyN+7pGOb41fJx1R7D6oxfle93z8WwYa9AHx/0zmRT9XnA58uyZclB884ur9ekxX4I5Q/5R/Juh9+R19EvpXTALNusUXne9lAMldeN23FvrHwxlyBHKT6RGppDyT4c19M0CjLzz97aHafmVczvIHk886cMeVsJjI8B/9UoazDlMf9a/aualk8hO34eUSlr7Q4cy3pkp9LGLewvKQGrOnrxlHbKKPsYT5kvr2n5u8g+CO3cabVd+cwbx7Bz+b5NAz5Qlm1WPte5dKg11Y/nP/9a41d5/6iJYY5fgz6e2mNYnfGr7KdnY9iwV6Dpg96R0imR+XNwNbL1ccAPyLE1Wh3LZQfyzpnG9d5zyTOoxh9x0XaPoVJW9ezpLWSWvhl59vnu8iN4fVm/BG02hy7oMyEvZVwFvKyDx7dIqffd5Nn7+8i+D0dV/ma7M/8OqVaa4pcnLxu8opR1HNlf5Tzyjpj7yEs1/6D1gUCrf6dVyQ6yfyID/WHkYIRT2/2sKs8bHZo3IgdU/FzTtvsC67ZYzlrkoH+XkHdyTSp/m6+W39Y9wKfLtpvRoZsy/Hj+8681fpX99GQMc/xqayDQWmPYUMWv8v6ejmHDXoHKB71CCRxvKa+3IMdV+SbwrbJsHDmA2vebv2gD2P+byb4U51NptiU7YV5VeT22zeNQ07/vI5s8Dyez8TeT17jfU75Ir22zvDWBpRf2eZAdZ/9Ch0beZX7n0X3IpuX9yYEBbwam9fe5DPazJJuNLyYvw5xFBvrXkGfU7yWb7Ce1+bfajuzk+W5yzJiJ5CjGvyRvRb69fPfaPZveD/h/5KWSdckpXS6lTE/Rgb/JRuX38XJyMt3ngPdX1r+K/E+/o5eu/ag/fpXtey6GOX61Hr+a/ka1x7C641cpo6dj2LBXoOnD3qX8qF5JZrifKV+ca4BzyzZLMcgZ4MlOj/9HZt8fIpu+q/0CziNvMe7ED7k6YvBEspPnkuSEq//D/DuIlieve7d7N+IpJVAsU14vKGi1fOZR2ccipe7Xkh1LX0aOwzOZbCr/fflxT+zEj4K8hLE18HbKBLJl+Vm0OR9X2c9OZJ+UPcnbjk9tOtZDaZr8dxD7fv4zYP7Es3uRTf5fJMfZ2ZC8NftFQb7FMj9JGVeHvEX73Mq6DwAXVT9HPzr3qCt+lff1ZAxz/OrI966WGDYc8auU1bMxbNgr0MeHvSOZ1U6rLFu6/NBbulWWzIpfXZ5vSN49cUL1S0jepnt5m3VfimwGHV9Zdhx5Rnpx5ct7OO3fibIy8wcaPIMcgbnx+kVBixfewdTOvF+N5vaXlEB1BDnOz8VkX5WJtNE/ZYB12ZNsMn9Jm/tZnDwDXZe8/fxa5k8ZsUrTtoNtjdgV+DPzz95PoXRiJs9APw6cUF6/tNXvA9mysnTl9dLkGfs25H+S55CXthqjS3dF59JufdQRv8o+eiqGOX61H7/KvmqJYUMVv8r7R00MG/YK9PMH2J4cdXd8eX0g2aTd1lD9lWCxPnl9+wTK+Cdl+Rpt7Lvxg16UbDL+ann9abKD64bl9V7kmUI7A1mqBImzmT/Q4HdL0HrRmSLzr7uPp4WRnyvHthPZ3HsC2RS/ZPkRfIH8T+a9NX8vViP7WNzU7o+uBKd3lGO5uvyQG1N37FiOr6VLM+RZ5f8jz9R2IJveP06e1TZGsl6X7MfR8pQn5e95Kdn0PrXy3TgBOKe8Xpwcg+cxuqhzaTc/6opfje9W+bdrY5jjV/vxq+yvlhg2VPGr8jcdNTFs2CuwgD/ETuVH3ZgyoKNZbQlYx5K3MzemC+hEc/y6ZDP1TOCDJXidSV53voA8a+jEj20ZMvP/ZiVondUctCrBajmymfZ1LZa3LXm5ZEtynJrfVj+zUo/X1/ydWIK8bLNem/tZjwz2G5Bna5cyvyPma8lLN21NSULeWVO9PLF+CSpHk/1fdqVyVt9GOS8h+5/cQ/7HsT15+/lvgX3KNkvTwmUsP9r6u9Qav0oZXRvDHL/a3letMWyo4lcpa9TEsGGvwEL+ELsCT9OhSVb72P9LyVuc254aouxvMjl+yyLl+TXMv3toA7IfRstnoJVyGkFiGfIupdMqQesM8mx62cr2y5Uv74A7t5J3u2xVKettwKuBN5Jnu2uX5Rv2VbeR+CgBfE3y1uWzyuulyakpLiYvyfwF2LWdv0t5viU5/s5/k03ji5Cdjb9B3ql0ObBJB4+tcRb6s3Ic06mMXePHsHzfao1fpYyui2GOX+19dnXFsOGMX5XvV0/HsGGvwAD+CEvWvP+Wb6Nu/nGSZwLXAnuX15PJs4QTO1TXRvBYh/mTjzaucU+vBK3vMf925qXLD3BQdxKRHWQ3LO9flOx3cHcJVsuXbbYDPs8InJV9Icd2KDmhanVsmrHkmX1jMtR2put5c/lsViOn8TgF2KWyfi0qfWQ6eFyN1oDjy39aD3bb36bXHnXHr1JGV8Qwx6+O/s07GsNGQvwq++7pGDbsFeiFBzloXeNHvCM59cC65fUryGx9wmD/E++nrKnk9fqfk82vq5HXuL9LnvGMb9p+JWDTFssaX/a7YwlaJ5MjWI8rP8ibqz/Ekf5oCipHkP1sXtXhMo4h+1JsUF4vR95Z83VK8/gQHd8qtDHSsx+j6zFUMczxq+2/U60xbDjjVx/H13MxbBFs0CSp8vzlZKfSMyRtQvZ7uIEMFETEDWRnwLlRvkVtlLs1OQjgzmT2/07yFvEVyTOexcnm5sb2i0TEgxHxlxbK2gp4grzd+L/IDppfJfsGXFzq8eGIuLD6eYxkERGNukbEKWQQ+ZmkLTqxf0mvJse+eS3wt7LfzSPiZPLs+hWSlulEWX1pOr77I+L+usqy7jYcMczxq311xrDhjl/Q+zFMbeYAo44kNYKOpNXIgQAnkT/og8gxSt5FjiK8e0Q806FyVyXPQoMMUJ8BPkJ2mr2PvKPk9naTukp53wMejoj3S/oI2VR9XkT8TtKi5KSkj1U/j5GoBO3nmpZV/4ZHAn+NiMtb2PcLjl3ShmRQv4Xs87AB2fx+HDk+zmIR8VCrx2LWCcMRwxy/WldXDHP8Gnpu4Rqkypf8Q+RYIdPJsXC+TTa9/h14hOw/sHw7ZTUy/XK29kvy7OzPZMfPz0fEZeTZ2tJk8tzO2WfzWd4XgOckLRcRXwLuBA6WtAPwLPA4zP88RiJJGwA7S1qsurycRS1Snn+93WRL0mRJq0fEbeR3YnVyctU9yKk7louIJxysbCQYqhjm+NW+umKY49cwqeM6ZS8+eOG15e0pAwySt3yf3rR+Gcrgcx0odwuy30F1VOkPk3eQ7E12aN26Q2Vt2yiHbN4/Fzi2sv7jwCuG+28xkL8TeWfNJWQ/jR0oAx42bduYtHYxylQfLXwXjiQ79d5Kjk9UnXfsXeXv9NLh/lz88GM4YpjjV+t/p7pimOPX8D3GYgvVdDZwMLApcKak9wL/Jmd9D0nrAHdFxGPkIG2dsBTZwXMmOdYK5FnIGHLQuxMj4qoOlbUmcIykGeSZ6BHAjyS9MiKui4gvdKic2pS/w+uAr5H9Q/Ynp0MZK+mSiHgaQNKYiJgnaTzwE3IOsgGXUfaxI3m2vjk5BMDBwOKSfk3eln4AsG9E3NqZozNrzTDGMMevQao7hjl+DR8nXANQ+YLuSnYovIQc0PCZiHhDWfchsgn+CHLU4pY0AqOkicBTEXFp+WGcKumOiDgrIh4FTpS0VEQ80Wo/hEpZjQ6XV5Xj2w6YRt4Bszw5N9x1rR7TUKl8Dq8HLom8ZHGZpI+SfUWelXQpMC8ini2B6jxyxvs7BlnWOmSn3xUi+1fMkPQcOaHvouRdWFPL38psWA1VDHP8as9QxTDHr+HhPlwDJGkNcjTiZyPiB+SP+feS9pT0LvLL+41os4NpCSB7kKMIf1XS54G/kUHw8HJG2tj2icZ72ihrZ3KwwU3J+b02jYifkVN7/JEcB+WfLR/Q8PgzsHLp/0BEfJG8Y2kqOUDks5KWJ88KP1eC2gL10UdkDnkb+78kHVM6tv6S/LttR/5n42BlI8ZQxDDHr47paAxz/BoZfJfiIEh6K9kcvh/5Y96TvLPnceDrEXFjB8qYDHyHHHzuk+RM87tFxKOStifvItkBuKfVQFUpazngp+SdSZuRfRx2Bf7ZCLqSFouIp1o9C61b5Sz3dcDLgVnAv8hjuokMwo+Qc3MtAvxfRBwj6VPAZRHx+4GWUZ6/izz7fyYizi2dcN9C/qfy1Yh4TtLSEfF454/WrD11xzDHr8GrO4Y5fo0gMQI6knXTg5xC4QYyiDSWjevg/icDnyM7MP6BMqM8Ze4yYMUOlfMqYJNS1sG8cLC7t9I05cVIfpAdgG8l+4Q8TE7aui05ue+vyP4jk8m7bj5f3rNIC+UcBVwJ7E7ecfWxSvnnMH8KlBE9PYgfo/tRZwxz/Gr5eGqPYY5fw/8Y9gp044OcmHY28LYO7nNDstPj2uSM9jdWgtUuZIfTtmZmr5S1GfA7cgLUHwHzKHOXkXcVXUuX3M1Ddsr9Djka9qvIvhqrlfVLkCMlr1T+k7luMMdVDTzkOD7nl31+grzzaiZwXFn/RmDV4f5M/PBjII9OxzDHr5aPpbYY5vg18h7uNN+CiPiVpIPI8Wo6ZUtg+4j4jqRLyLOZt0iaRc4r9bGIeKDdQiS9grz75NcRMUvS/uSUHd+TdA15ieG4yNGlR7TISPGEpBvJW81fSnb0vFfSO4B7IzvtLkae9e430ONqaobfk7yT6zCyf8POEfEaSW8Hvi/p6eiSO6DMoJYY5vjVgrpimOPXyORO8y2KiEsi4vZ291N+SETE2cBKko4GTiPvIlqDvFvlwxHxiz46PrZiFXIE4Y0lrRMRT0XEG8kz0DuBIyPi5x0qq+Ma9ZK0VukEDPAA2Wl2WkTcKemVZP8RACLiKeDgwQThSrB6E3AgcEVEzCXv7P1V2Wwx8s6hH7V3VGZDrxMxzPFr8IYihjl+jUzuND+MJG1Ezit2Z0ScV25vbpyhPVG2WTQ6ND1QpdxtgPeSAx7+OiL+0cn9103SLsCpZKffhyPiEEnHk5czliMD/WciYkYL+x4TEc+W528ADgFmRcSnyrLdyb4Q/0eeLW4XEXe2fVBmXcbxq3V1xTDHr5HNlxSHWOWOlLXIO3luBT5TznQeIDuCvpac8oJOBqtG2RHxe0mLk4PpLS7pJxFxd6fKqVO5TXoP8i6rm4CzJZ0VEQeUz3Bd8i6lmwd7Z5JyTJuXAH8q/3mMIW/FXl/SRhFxY0TMkPQIORL3SQ5WNpo4frWvrhjm+DXyuYVrGCjHqTkWeAi4mrxNd1lyEtfjyQH8dgCeHEzCsIDyqtfzq893IsfemTbSzxJLM/yK5IB8/yKb1++XNA74MTAmInZts4yNyc7EmwOTI2Ij5QCOnybHrbmgW/qGmNXF8as1dccwx6+Rz324hpikFYFDyc6PbyLHXHmaHE/mq+TM7MdFThbaUrCq9BF4uaRVqvspZ6cqz39F9nn4R/V9I4WklSStClnviHiQnJR2GWBbSYtHTnPx9txcU9opLyKuB1YmL5OcU5bNBr5ETnWxn3KcIbNRyfFrcIYyhjl+jXxOuIbePPLHNqG8/j55vX5qRPwnIj4XEb9tJ3iUoLQ72Xl1UmN5JVCF0piImCtpbGN5q2V2mqQNyVGUlyqvG3W/kAzshwK7SVoiskPprhExs43yGp/318i7hZaRdJik5SPib+SozIuTl03MRivHrwEayhjm+NUdnHANsYh4hPwRvq5cV3+GbGIeI2mxalBptQxJLyeb9g+KiGvKWdZaJVA1/uaLxPy5uL4kaYW2Dqzz9iHHCXpK0pbwgoD1MzJgfRiYOti+Wn1pvD8iZkfEacAtlFu0JR0FbAV8opyhmo1Kjl+DMmQxzPGrOzjhGh4/IicI/aqkz5Lzm/068hbndgJV4yxnFfJMZmVJnybPQm+QtEnk1A1jSrBajgyWP42Ih9o6os47hRzL53rgP+VzUSVgzSCnurizlc+svzPw0p+CiPgecE2pw7uB34WnuzADx6+Bqi2GOX51J3eaHyaSliHnGVsP+GtE/KGNfTXuHFopIh4sP8YTgDcBXwd+SU4eewfwg7LteLLfxaci4so2D6fjJE0C/ge4BzgzIv67LF+E0h2ijX1XO97uR14mWSoizuhj/aLAkuXM3sxw/BqIumKY41f3csLVIyTtCHyIvFPoTvKW34fLui2B75JN9H8oZ0HfAs6KiMuHp8YvVgm84yLiaeWt3xuSHXF/HxEnle0WiYjnOlDeEeScb5+lTIIbEed1sgwzW7heiF8wtDHM8av7OOHqAaXPw8/JEYWXAaaQTclHk3NwnQccHRG/LNuPAVaJiHuGp8b9Uw4I+B4ggF9FxOnK0ZIPBWZGxBc7UIaA5cmBBw8ADifnEpsKjC0dWM1sCPRS/IL6Y5jjV/fywKddqqmT5WLAJZEDAi5C9hk4luw0eSnwliiD6FE6m5LN3COKcrC+zwPHkJO6frOcKZ5Sguz7JE2KFgbra/q8xgGPkbezfxlYHXh76RfyHkl/iYhrOnFMZvZivRi/oL4Y5vjVG5xwdanSbP0aclTiscCekmZEjk0zR9I8YO3SrHxz4z3As8NW6YVbHrg6In4LIGlX4LeSriMD78xWOsc29WnYF1g0Is6SdC8ZGCdExJPKyWLfC+zeoeMxsz70aPyCGmKY41fvcMLVZSp9BLYix6m5nuz3MIecYmMiGaBeDZw9fDVtyTxg1dIh9pGIuE3S2eRZ7TPkyNaDVglW7yeb4N9Rlk+TFMAVkq4kL2W8M3KwQDPrsB6PX1BDDHP86h3uw9WFSrP1CcDHI8epWZccqfg1wBLAXcAvIsd6GXEW1KFT0neBJclgvBg5kN/+7TaRS1qbHPzv7cDjwK7ApsBJwERy3rG5EXFXO+WY2YJ1e/yCoY9hjl+9wS1c3Wk5YFvytulrgH+Qk6CuCHy0EQiarvuPCOWuncnAnyVtRCb9N6iMrRMR75L0BbID6EuBo1oJVM3HHhF3SboR+C3QGM35WbK/xftKvxAzq1/Xxi8Ymhjm+NWbnHB1oYi4RNJbga9IuiMifqicAX5bYCVJc6MY3pr2aVXgNaV5fFtge4DS4bMRsD4OOdZPRDw22AKa+jxsR/aruICcxPXt5J1D/5C0Fxn0R+LnZNaTujx+Qc0xzPGrdznh6lIR8XNJzwHnSNoDeBL4TESM6LmyIuJOSc+S/RC+DdwOzweZ5rO0lkZGrgSrQ8gBE/8N7EGOV3N66UNyGHAwcGB/lwbMrB7dGr+g/hjm+NW7PLVPF4uIXwD7AesDN0TEL1UMc9VepKlO5wD7A48CH9L8edKW0fy50tqdj21b4C3AphGxJdlZ9UPARqVj7kQyWN3Qahlm1rpuil8wtDHM8as3udN8D5D0ZuBM4AMRccFw16c/knYGXgf8nRw5enNgX+A2MqC8muxIO+hpKJqa4ccDR5JziB0cEb+RNJbsYLo82WH3b+WuITMbRt0Sv6C+GOb4NTq4hasHRMRvyFGa/zrMVXmRxlmhpMlkoHiSvLvmu+Qt4d8n+0RMA37bgWRrWbIZ/ytkc/8ekl4TEfPIM8T7gYccrMxGhpEcv6D+GOb4NXq4hctqJ+m1wFHADyPiJ5JWJ4PH6sBhEfGIpAkRMbedO5MkHUOORbMacCJ5e/muwNrAjyPisvaPxsxGm6GIYY5fvc8tXDYUHge2ALYrr+8lp6R4CDhbOaP9g9BWn4e9gTdHxN7Ac8BeEXEzcH7Z9+6Slhip/UPMbESrNYY5fo0ObuGyjmuc4UnamJzvaw45xs5vgJMj4tSy3arA+Ii4tYUyFq02qyuntfgnsDF5q/TUiHhK0jLk3GNExD/bPDQzGwXqjmGOX6OTEy6rhaTdgE8BF5NnhoeRAx6eA5wREV9pY9/LAq8FrgBeT/apWI/sB3IveXY4T9JHgJcB7+7jdm0zs37VFcMcv0Yvj8NlHVf6N3wE2AF4F7AU8ERE3F7O5C6QdH60Pg3FksCaZCBcPiImS7oW2BP4GzBFOQL0vsA+DlZmNhg1xzDHr1HKfbisYyr9C8YCNwA7k0HkgIi4X9J2EfFXYONWAlVj/xFxH9nMvwlwjaSVI+Jx5gfGd5CjP+9T+kGYmS1UnTHM8ct8SdHaVunvsHxE/KssO4cMGluUkZnfQN55846I+HurZTQt2xjYiTxbnB4RN0laLyJmSVosIp5q++DMrOfVHcMcvwyccFmHSNqJbIK/BfgmsDI5UvLKwIVl3ccjYkYL+66OU/NBcrDBf5H9K5YGDgWWBR4GJpGTubY0LZCZjU51xTDHL2vwJUVrWWVAwNWAo8mZ6wEOAhYlB++7ARgPfDAiZrRyW3MlWG0L/BfwPeBZ4NfAE8Ap5MjP2wBfcrAys4EYihjm+GUNbuGytkjamjw7e2NEfFTSOPLMbTngJ8DvowOTq5Y7ht4DXBARZ5VlXyan0tgjIh6QtHhE/Kfdssxs9BiKGOb4ZeAWLmuBpDHl31cDPwLeBhwpab+IeBo4DvgPeZfN+BbLaD6LnAUsAbxaOdcYEXEMcB3w41In93kws4WqO4Y5fllf3MJlAyZpBeDhiHiudPg8DPh5RPxa0i7AF8gm8XNKAFk3Iv7WQjnVPg9TgUfJQQHvBn4A/Bb4dkQ8XLZZOSIe6MAhmlkPG4oY5vhl/XELlw2IpKXJ2esnlkVbAq8CNpW0VERcSE7e+llJ74qIZ1tJtuAFfR6OBI4BNgAuACYAHwTeABwlabmyvYOVmS3QUMUwxy/rjxMuG6ingTOApyUdFRHfBr5BBpPXSloiIn4FfAC4vZUCJC1Zeb42sF1EbENO5noz8PcyLs0HgVfggXvNbOBqjWGOX7YwTrhsgSQtLmnF0q9hMXJKildKel9EfBf4Izkw4HYlYF0YEb8f7J08knYGviBpYnnvs8A/JX0W2Iyc7uIZSe8E7gH29txiZrYwQxHDHL9sIJxwWb8kLUKOhry/pEOBY4GryRnsXyHpiIg4jbxtek9yTBlgfrP6AMvZFTgBuCwiZkeaA4wB3gf8V0T8W9K7gCOAJaMy8auZWV+GIoY5ftlAuUnT+lU6lt5DTrC6NfDhiJgt6UEyWd9B0oci4iRJa0XE3MGWIWlVcvybd0fEteWW7MXJ+ca+ADwAXCbpUnJU5ndFxP2dOUIz62V1xzDHLxsMJ1zWp8adNhHxD0m3AP8GXibp5RFxE/CLElzeIGlSRNzZYlFPAc8A/5G0ONlp9bXkd/MO4ENkoHoEOCsiZrV3ZGY2GgxRDHP8sgHzsBD2Io1AJenlwJNkn4MlyKktlgY+QzaXbwlcGzkZa8tlkUHpzcDLyVumrwRuJJvjfxARv27jcMxslBmqGOb4ZYPhhMv6JGkH4CzgUvIum7eSzeTvAzYFXgm8LSKu6kBZS5N37Uwkx8R5qiw/g+wX8b12yzCz0WWoYpjjlw2UEy57EUmTgUOA8yLiD5L+H7AteRb3MLAj8EREXFFjHfYEPkre3fP3usoxs94z3DHM8cv64oTLnldGVl4C+CnZ7P7eiLi+rPsSsDvw+jo7fSonkd2LnHdsr4i4sa6yzKy3DHcMc/yyBfGwEFad92ts5Ez1BwJzgZ0royF/BLgQWL/m6jwM/A2Y6mBlZgMxgmLYwzh+WT/cwjXKVTqX7ggcDPwfOU7NX4AzgYuBMyPiX8NYTTOzPjmGWbdwC9coVwLVm8gxY74GrAIcUQbuOxJ4C3CoJA8hYmYjjmOYdQt/AUchSauTU1zcWUZTXpXsczCevNvm7WXTO4ADgBUiYt7Q19TM7MUcw6wbuYVrlJH0UuASYAqwTFm8LDnVxfHAzhFxV7ml+mPAHRHxx2GprJlZE8cw61ZOuEYRSZPIoHRSRPw4Ih4tq75FjlVzb0T8U9Ibyab5qyPi2WGprJlZE8cw62buND+KSDoQ2CQijiyTum5MjrT8L/LOmoOAyeSl5i9FxIXDVlkzsyaOYdbN3IdrdLkdeHdpat+LHK/mFeQdPW+OiHeXW6gVEQ837v4ZxvqamVU5hlnX8iXF0eVa4MfAF8k+D98EtgFOBsZJWiwiHomIhyHv/hmuipqZ9cExzLqWLymOQpJWiIiHKq9fD3we2DMi7h2+mpmZLZxjmHUjt3CNQo1AJWlRSTuTZ4cnOFCZWTdwDLNu5IRrlJK0KLAF8CHgk+5cambdxDHMuo0vKY5iJWCtGBH3uXOpmXUbxzDrJk64zMzMzGrmS4pmZmZmNXPCZWZmZlYzJ1xmZmZmNXPCZWZmZlYzJ1xmZmZmNXPCZWZmZlYzJ1xmZmZmNfv/TpjN315p29MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "myfnames=[\"BM25\", \"TF-IDF\",'revienum','overall_rate','pos_reviewLen', 'State', 'Sentiment of neg_review', 'Sentense transformer', 'VBERT']\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt, numpy as np\n",
    "\n",
    "fig, (ax0, ax1) = plt.subplots(1, 2, figsize=(10, 6))\n",
    "\n",
    "ax0.bar(np.arange(len(myfnames)), [abs(x) for x in ca_pipe_all[1].model.to_dict()['Linear']['weights']])\n",
    "ax0.set_xticks(np.arange(len(myfnames)))\n",
    "ax0.set_xticklabels(myfnames, rotation=45, ha='right')\n",
    "ax0.set_title(\"Coordinate Ascent\\n Feature Weights\")\n",
    "ax0.set_yscale('log')\n",
    "\n",
    "ax1.bar(np.arange(len(myfnames)), rf.feature_importances_)\n",
    "ax1.set_xticks(np.arange(len(myfnames)))\n",
    "ax1.set_xticklabels(myfnames, rotation=45, ha='right')\n",
    "ax1.set_title(\"Random Forests\\n Feature Importance\")\n",
    "\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Interaction!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input a query: hiking in Michigan\n",
      "[2022-12-17 21:38:51,388][onir_pt][DEBUG] using GPU (deterministic)\n",
      "[2022-12-17 21:38:51,392][onir_pt][DEBUG] [starting] batches\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "batches:   0%|          | 0/13 [14ms<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-12-17 21:38:51,702][onir_pt][DEBUG] [finished] batches: [309ms] [13it] [42.13it/s]\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 0\n",
      "Document 8934: Oval Beach, Saugatuck, Michigan, One of the Best Beaches and Shorelines in the World Michigan\n",
      "Rate: 4.5 Review Number: 1085.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g42683-d142430-Reviews-or20-Oval_Beach-Saugatuck_Allegan_County_Michigan.html\n",
      "Oval Beach in Saugatuck, Michigan has been rated by Cond, Nast Traveler as one of the 25 best shorelines in the world, while MTV has it ranked as one of the country's top five beaches. National Geographic Traveler takes it a step further, rating it as one of the top two freshwater beaches in the country. Those rankings come thanks to the area's combination of towering sand dunes, white sand beach, beautiful views, seclusion, and services, such as life guards and concessions. Oval beach can best be reached via North America's only hand-pulled chain ferry boat, which crosses the Kalamazoo River from nearby Saugatuck. A climb of 282 steps then takes visitors to Mount Baldhead, where they are rewarded with a splendid view of the harbor and Lake Michigan. From there, visitors descend onto the grassy coastal sands and the stunning shoreline. Along with the customary beach activities of swimming and sunbathing, visitors can also take a freewheeling ride on the dune schooner'' - a converted pickup truck that traverses some of the towering nearby sand hills. Visitors can also book some time on an old-fashioned sternwheeler shuttle that tours the river and the Lake Michigan shoreline, taking in a variety of sights, such as lavish area homes and cottages, wetlands, forests, and dunes, as well as the Ghost Town of Singapore that now lies buried near the mouth of the Kalamazoo River. There's also a WWII Duck'' that provides ferry service between the towns of Saugatuck and Douglas. Biking, hiking, strolling a winding boardwalk, and picnicking are also popular local activities, and there are a number of scenic city parks adorned by Victorian-style gazebos along the riverside. That makes the setting ideal for simply relaxing or watching some of the port's 900 yachts and sailboats glide by. The area is also a haven for anglers, who can charter boats in search of salmon, pike, trout, and perch. And anyone who wants to make a bigger splash can rent anything from kayaks, to jet skis, to sailboats and yachts. Horseback riding and equestrian training are also available in the area. Midwest Living magazine ranked the Saugatuck-Douglas area #5 on its 2007 list of Best Small-Town Getaways, and the two towns provide a variety of sights, shopping and activities for visitors. Together, they are known as The Art Coast of Michigan,'' thanks to the number of artists who have taken up residence and operate studios and galleries in the area. In addition to such small-town, rural charm, visitors will also find a number of unique shops, fine dining and lodging, professional summer theater, jazz music, and some outstanding golfing, including an 18-hole championship course designed by Arnold Palmer. In Douglas, history buffs can take a tour of a floating, maritime museum, the S.S. Keewatin - a 350-foot passenger steamship once sailed the Great Lakes - or visit the award-winning Historical Museum for a look back in time at the area's earliest settlers. The Saugatuck Dunes State Park and the Allegan State Forest are also nearby, as well as a tour of a local vineyard. The area is vintage, small-town Americas at its best - a combination of scenic, beautiful beaches, 19th-century architecture, and 21st-century shops and activities.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 1\n",
      "Document 13436: Wisconsin Year Round State Park Wisconsin\n",
      "Rate: 4.0 Review Number: 133.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g28972-d207065-Reviews-Potawatomi_State_Park-Wisconsin.html\n",
      "Featuring miles of beaches next to the shores of Lake Michigan is a park that is filled with pine trees, wildlife, and lots of fun for the entire family. Whether it is summer fun you are after, days lounging on the beach, or winter sports like cross country skiing, this park has it all. The park is located two miles south of Sheboygan on County Highway V. It is convenient to Highway 43, so you will feel miles away from civilization hiking through the park, even though you are just minutes from a major Wisconsin highway. The park features over 130 family campground spaces, 52 of which have electrical hookups. All campsites are equipped with a fire ring and a picnic table. Sites are designed to accommodate six people or a family with two guests. Vehicles can park near the site on the paved drive. Off-site parking is available if more vehicles are needed. Kids will love the rental camping space that features a canvas and pole tepee. If you forget or you do not have your own equipment, you can enjoy the thrill of primitive, rustic camping. The grounds are open year round. During the warm season, toilets, showers and laundry facilities are all available. There is no water or sewer hookups available at the camp sites, but you can fill up with fresh water at the station located in the park. Group campsites are available and two of them are large enough for approximately 50 people. Picnic shelters are also available for reservations. The park is typically most crowded for camping before July 4th and following Labor Day. Most people enjoy camping when it is a bit cooler, so they avoid the hottest part of the Wisconsin summer. About 10% of the sites are available on the first come first serve basis, so you may want to make a reservation to guarantee your space. This park features a variety of wildlife that can be view just by strolling along the trails. There are several organized nature hikes that are posted on the parks build board announcing the times and the schedule. You can also view the Teacher's Guide online since many of these hikes are booked for students. The Sanderling Nature Center is a great place for nature lovers to visit and is on the north part of the park overlooking Lake Michigan. It is one of the park's most popular attractions. Here you can see displays of park animals, wildflowers, Great Lakes fishing, and the history of the park. There are live animal and plant displays during the season, and the center frequently gets new exhibits. The center has an auditorium for special shows and programs. The Kohler Andraw State park invites guests to visit, swim in the lake, lay on the beach, hike the nature trails, camp, and enjoy the sights and sounds of nature. There is no need to drive all day to get to a great state park. This park is right in Sheboygan and offers the best that nature has.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 2\n",
      "Document 8944: Tamarack Lodge Michigan\n",
      "Rate: 4.5 Review Number: 241.0\n",
      "https://www.tripadvisor.com/Hotel_Review-g42758-d625155-Reviews-Tamarack_Lodge-Traverse_City_Grand_Traverse_County_Michigan.html\n",
      "Tamarack Lodge in Traverse City, Michigan is a winter destination as well as a summer vacation spot. The lodge is placed near the northern point of Michigan, offering a forested and lake location. Visitors are often drawn in summer to the white sandy beach. The lodge is typical of the Midwest. Tamarack Lodge is a condo resort too, allowing guests to spend several weeks at the resort on the East Grand Traverse Bay. The lodge is a mixture of wood ceiling and walls and stone. Vintage lighting and plumbing are part of the experience.  Resort features include direct water views, Lake Michigan shoreline, beachside walking, bonfire pits, picnic areas, outdoor shower, full service bard, swimming dock, boat and jetski rentals, as well as bike racks. Offsite amenities include boat parking, storage, RV parking and storage, tennis and fitness club, spa, and meeting facilities. Amenities that are being worked on as of 2011 include an indoor pool with kiddie pool, locker facilities, hot tub, fitness center, meeting rooms, and video arcade. Gas log fireplace, 42 inch plasma TV, combination safes, wireless internet, full kitchens, granite countertops, and stainless steel appliances are some of the condo amenities along with housekeeping services.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 3\n",
      "Document 13397: Scenic Pathway Along Miwaukee Lakefront Wisconsin\n",
      "Rate: 4.5 Review Number: 52.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g60097-d147148-Reviews-Lakefront_Trail-Milwaukee_Wisconsin.html\n",
      "Milwaukee's Lakefront Trail may be one of the best features of downtown Milwaukee. The trail used to encompass various other pathways through the city, but those trails are now considered separate and known as the Oak Leaf Trail. The Lakefront Trail runs along Lake Michigan and passes the Milwaukee Art Museum, Veteran's Park, and the McKinley Marina. During the summer, the trail is jammed with bikers, inline skaters, joggers, and walkers. It is a popular place for families to enjoy the lakefront, and often people step off the path for picnicking, kite-flying, and other recreational gatherings. The trail runs from the north end of town near the University of Wisconsin - Milwaukee and Shorewood, south through the city down to the suburban areas of St. Francis, Cudahy, and South Milwaukee. Walking, driving, or biking along the trail offers magnificent views of Lake Michigan and the Milwaukee shoreline. Parking is available in a variety of places, often free of charge. On weekends during warm weather, it can be difficult to find parking spaces in the downtown area, or near the established parks and swimming areas of the trail. It may be advisable to park a block or two west of the lake and stroll east to the path. Occasionally, area businesses allow parking for the trail provided you visit the location before or after your hike or bike ride. Along the trail are various options for dining, recreation, and culture. The Alterra coffee house lies just across the street by the northern downtown portion of the trail. The Milwaukee Art Museum is reachable via the trail, and some walk the trail to reach the Summerfest grounds for concerts and ethnic festivals. Futher south are a variety of county parks and recreation areas. Visitors can stop for a bite to eat or a cool beverage while biking the trail or hiking along the shorefront. Trail visitors can choose from a variety of accomodations located just minutes from the trail. The Knickerbocker Hotel is a unique, historic option that offers comfort, class, and delicious dining, all in one place. On Milwaukee's east side, a few blocks from the trail are famous favorites including E.E. Sane Thai food, Bartolotta's Lake Park Bistro, The Lake Park Bistro, and Coast Restaurant. Many of these restaurants offers award winning dining experiences with some of Milwaukee's most delicious food. Visitors have the option of renting recreational equipment for traveling the trail like bikes and skates. This ensures everyone has a chance to enjoy the numerous opportuniteis the trail provides. Riding along Milwaukee's lakefront is a right of passage for many Wisconsin residents. People come from as far away as Illinois to enjoy the lakefront during the summer months. While many areas similar this in other cities are commercialized and packed with retail establishments, Milwaukee chooses to under-utilize the lakefront, citing its desire to keep the area natural and accessible to everyone. Families from all over the city enjoy the beachfront areas of the lake.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 4\n",
      "Document 7886: Bobby's Bike Hike in Chicago, Illinois Illinois\n",
      "Rate: 5.0 Review Number: 4028.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g35805-d659394-Reviews-Bobby_s_Bike_Hike_Chicago-Chicago_Illinois.html\n",
      "Chicago, Illinois is the home of Bobby's Bike Hike, just named the best bicycling city in the United States. The bike hike began in the summer of 2002 and since that time has offered biking tourists the opportunity to view the sights and sounds of Chicago, from a totally unique perspective. There are several types of bicycles available for rental that will accommodate any fitness level, and informative tour guides that take guests all around the city's most popular sites and provide an entertaining commentary. The easiest bike tour offered is the Lakefront Neighborhood Tour. The bike tour is designed for those who have a moderate fitness level and is available every day of the week, with the exception of Sunday. The tour takes guests past beaches, quiet neighborhoods and the city park. Highlights include the Gold Coast Mansion, Playboy Mansion, Oprah Winfrey's home, Lincoln Park Zoo and the Lakefront. The tour is open daily Monday through Friday, and a Saturday tour has been added as well. Reservations are recommended; the tour is the length of seven miles long and lasts for a period of three hours. The Green Chicago Tour is a romp through lakefront parks and past many of the city's major tourist attractions. Tours are offered at 10 a.m. daily, Monday through Friday. Guests will cycle past Soldiers Field, Buckingham Fountain, Navy Pier, Millennium Park and Shedd Aquarium. Tour guides will take visitors past the lakefront, where the 2016 Olympic Games will be held. Guests who go on this tour can expect it to be relatively easy and quite enjoyable, the length is seven miles and three hours long, and again reservations are strongly encouraged. During the fall season, Bobby's Bike Hike offers a downtown tour called the Fall Fireworks Night Tour. Guides will take cyclists past Millennium Park, Museum Campus and Buckingham Fountain. The tour commences every evening at 7 p.m., Monday through Friday. Guests will get the chance to view the splendor of the city in the evening and on Wednesday and Saturday's, get to experience the fireworks off of Navy Pier. Reservations are recommended and the tour is perfect for people of all ages and fitness levels, it encompasses eight miles in length and takes around three hours to complete. On the last Friday of every month, thousands of bikers gather to take over the streets of downtown Chicago. The event is known as the Chicago Critical Mass Ride and begins 5:30 p.m., regardless of the weather or season. The ride begins at Daley Plaza, Dearborn and Washington, the ride is free and extremely fun for people of all ages. Bobby's Bike Hike offers tourists a unique chance to view the city from a memorable perspective and is an interesting way to discover the Windy City.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 5\n",
      "Document 8870: Elk Ridge Golf Club Michigan\n",
      "Rate: 4.0 Review Number: 4.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g29564-d10730551-Reviews-Elk_Ridge_Golf_Course-Atlanta_Michigan.html\n",
      "Elk Ridge Golf Club is located in Atlanta - not Georgia, but Michigan - the state's Elk Capital,'' about midway between Gaylord and Alpena in Northeastern Michigan As such, the course is one with nature and some of the largest elk herds in the state roam the fairways. Golfers should also not be surprised to encounter wild turkeys, bobcats, eagles and bear. The Jerry Matthews-designed course opened in 1991, and was laid out so as not to interfere with the wetlands, duck marshes and native wildlife. The course is what golf in Northern Michigan is all about, and features the customary rolling fairways, marshes, wetlands, fast, undulating greens, and an abundance of trees that golfers have come to expect. There are also good elevation changes and elevated tees, providing for the fun of those big drives downhill as well as some stunning views of the area. The course also features a number of doglegs, some strategic bunkering, as well as water in the form of lakes, wetlands, ponds, and streams all coming into play. The result is a good variety of well-groomed, contoured holes sculpted out of the Northern Michigan woodlands. The 18-hole, par-72 course covers 400 acres of land and plays to 7,058 yards from the longest tees, with a USGA rating of 74.7 and a slope rating of 143 on Bent grass. That 400 acres represents a sprawling amount of acreage, which allowed the course designers to make golfers feel secluded from the world, even when the course is packed. Some of the more noteworthy holes include No. 10, a par-3 that plays to only 164 yards from the Gold tees. This is one of the best elevated tees on the course, and the drop from tee to green is more than 100 feet. That will make the hole play even shorter, but golfers must be wary of the bunker that guards the right side of the green. It's said to shaped like a pig, but is a bear to get out of. No. 16 offers golfers more stunning views of Northern Michigan from an elevated tee. The par-4 hole plays to 381 yards from the back tees, and doglegs left around water and wetlands. Golfers can eliminate more of the dogleg by hitting a longer drive, but the approach shot that follows plays back uphill into an elevated green. This hole was voted one of The Best 18 Holes You Can Play'' by Golf Digest in 1997 Elk Ridge has received a number of accolades since opening, including being rated by Golf Digest in 1992 as the second-best new public golf course in the U.S.  That same publication also ranked the course #47 in 1996-97 among the Top 75 Upscale Public Courses in the country. Practice facilities at Elk Ridge include a driving range and putting green, and rental clubs are available. The course also features a distinctive clubhouse, designed to resemble Indian teepees, and features authentic Indian artifacts on display. The clubhouse also offers a restaurant and snack bar.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 6\n",
      "Document 8879: Grand Beach Resort Hotel Michigan\n",
      "Rate: 4.0 Review Number: 482.0\n",
      "https://www.tripadvisor.com/Hotel_Review-g42758-d280935-Reviews-Grand_Beach_Resort_Hotel-Traverse_City_Grand_Traverse_County_Michigan.html\n",
      "Located on 600 feet of waterfront in Traverse City, MI, the Grand Beach Resort Hotel is a popular destination for those seeking to enjoy water sports on Lake Michigan, enjoy building sand castles during family vacations, have a romantic getaway, or relax while watching the natural surroundings and colorful sunsets of Northern Michigan.   The hotel's list of amenities and features includes something for every guest. There is an indoor, heated pool and Jacuzzi where guests can swim and relax. Every guest receives a complimentary breakfast buffet with a large number of food choices. This is served in the lounge area, which offers scenic views of the water along with outdoor seating. Right outside the door of the hotel guests will find the sugar sand beaches that line the shores of Traverse Bay. This private beach comes provided with beach and lounge chairs on which to enjoy the view and sunshine. Business travelers can reserve the hotel's meeting space for group events. Guest rooms at the Grand Beach Resort Hotel include premium cable channels, DVD players, coffee makers, high speed wireless internet access, microwaves, and refrigerators. Some rooms also include Jacuzzi bathtubs, private balconies that overlook the bay. In addition, some rooms on the first floor step right out onto the beach.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 7\n",
      "Document 8851: Black Lake Golf Course Michigan\n",
      "Rate: 4.5 Review Number: 44.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g42550-d214528-Reviews-Black_Lake_Golf_Club-Onaway_Michigan.html\n",
      "Black Lake Golf Club is a Rees Jones design, and the latest addition to the UAW Walter and May Reuther Family Education Center located in Northern Michigan. The course is situated in heavily wooded lands on the southeast side of Black Lake, which is one of the largest inland lakes in the state. The course was designed by Rees Jones, who had never before worked in the hotbed of golf that is Northern Michigan. The result is a natural, 300-acre layout with a classic, old-style design that has become a trademark for Jones. One of the most distinctive features of the course is its approachable greens. All of them have been designed with ramps'' in the front so that short shots and bump-and-run approaches can all reach the putting surface, so long as they avoid the plentiful bunkers. Most of the greens themselves are designed with two tiers (or more), but the undulations are rather understated. The rest of the course makes use of the natural contours of the land, which Jones described this way: We were trying to build an old style, classic course. You'll notice we don't have mounds or moguls. We just have sweeps and natural grade. We strive for holes blending with the natural terrain. There is nothing artificial or contrived at Black Lake. The 18-hole, par-72 course plays to 7,030 yards from the tips, with a USGA rating of 74.3 and a slope rating of 140 on Pennlinks and Providence fairways and Providence greens, making it a challenge for even the best golfers from that length. When advising first-time players, local management offered three tips: one, be long and straight off the tee, otherwise errant shots will be lost in the woods; two, iron shots can be a bit loose, since the greens are so approachable; and three, don't over-read the greens as long as putts are on the correct tier. Some of the most noteworthy holes on the course include No. 6, which is one of the signature holes. This par-5 plays to 515 yards from the longest tees and is a great risk/reward hole. The hole plays from an elevated tee to a wide fairway and, for players who've hit a good drive, can be reached in two. The green is guarded by water on the front, left and back, however, so it requires an accurate approach shot. Since opening in 2000, Black Lake has received much acclaim, including being ranked #34 among America's Top 100 Public Courses by Golf Digest. Practice facilities at Black Lake are said to be unrivaled'' and include a 350-yard grass driving range, as well as four chipping and putting greens and four practice bunkers. The course also features a clubhouse with locker rooms, lounge and dining room. The Education Center also features other recreational facilities, such as a gym with two full-sized basketball courts, an Olympic-size indoor pool, an exercise and weight room, table tennis and pool tables, sauna, walking and biking trails, softball and soccer fields, and beaches and a boat launch ramp.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 8\n",
      "Document 8884: Grand Traverse Resort and Spa Michigan\n",
      "Rate: 3.5 Review Number: 626.0\n",
      "https://www.tripadvisor.com/Hotel_Review-g29529-d95128-Reviews-Grand_Traverse_Resort_And_Spa-Acme_Grand_Traverse_County_Michigan.html\n",
      "The Grand Traverse Resort and Spa is located in Acme, Michigan close to Traverse City. This area of Michigan is a premier resort section for residents in the state, as well as travelers who are looking for something a little different. Northern Michigan has warm summers without the humidity of southern states. Fishing, boating, sailing, swimming, and many other outdoor activities can be found around the Grand Traverse Resort and Spa.   The resort is set on 900 acres of property with full service spa, 600 rooms, suites and condos. They have 86,500 square feet of function space for meetings, events, and weddings. On the 900 acres is a championship golf course designed by Jack Nicklaus. The course offers 54- holes of various terrain and challenges to keep visitors entertained. Additionally the hotel has a restaurant, seasonal recreation area, indoor and outdoor tennis courts, swimming pool, and children's center. A shopping boutique offers jewelry and art. The tower is part of the hotel with 17 stories of glass and 186 large guest rooms. The rooms all have en-suite bathrooms, telephones, televisions, mini bar, internet, fridges, coffee makers, and top of the line linens. The hotel is a green hotel.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 9\n",
      "Document 13260: Bong State Recreation Area  Wisconsin\n",
      "Rate: 4.0 Review Number: 50.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g59997-d147217-Reviews-Bong_State_Recreation_Area-Kansasville_Wisconsin.html\n",
      "This park in Southeastern Wisconsin was originally supposed to be a jet fighter base. It is named for Major Richard I Bong, a Wisconsin native who was a pilot during World War II. The state purchased the land in 1974 and is soon became the first recreation area in Wisconsin. The park is over 4,500 acres and includes savanna, wetlands, grasslands, and scattered woodlands. It covers lands mostly in the state's Lake Michigan Coastal Ecological landscape but some of it is on the western edge of the Southeast Glacial Plains. The area is packed with things to do all year long and is a popular destination for field trips and family trips. Bong is considered an area and not a state park because it offers activities that are not traditionally found in a state park. There are locations within the space where you can fly model airplanes, hot air balloons, rockets, and hang gliders. There is training space for hunting and sled dogs, as well as falcons. There are trails for all-terrain vehicles and horse trails, and hunting is permitted in season. Each of the areas is sectioned off for its appointed activity. The area is a popular destination for field trips and families with children. There are naturalist guided programs that fit the Wisconsin Model Academic Standards. The programs are easily incorporated into the typical Wisconsin state school curriculum. Visiting children's groups can play hide and seek, act as animal detectives, find and identify insects, go pond peeking, or test water quality. Each of these activities is based on grade level. There are also resources available for educators to enhance the hands-on lessons once they have returned to class. Children are not the only ones who benefit from the presence of the area's naturalists. Guides hike, hold educational chats around the campfire, and answer questions. This is a great way to enhance your visit to the nature center and your afternoon hike. There are year round programs, evening programs, guided nature hikes, and the nature center exhibit. Guests are also welcome to conduct self-guided tours. The tours feature signs and brochures to make the tour interesting and educational. The area features over 16 miles of hiking trails, nearly 15 miles of cross-country ski rails, 13 miles of horse trails, over ten miles of off-road bike trails and snowshoeing trails, seven miles of ATV trails, and nearly two miles of nature trails. Those wishing to ride horses on the trails need a valid state trail pass which can be purchased at the park. Horse trails are typically open all year, but may close if the land is very wet. The Molinaro Center offers visitors exhibits about the history of the area and the wildlife in the park. There is also a solarium that features an exhibit on butterfly rearing. The area also has a boat launch and a fishing pier. The 200-foot beach has a bathhouse, play equipment, volleyball court, and horseshoe pits. Bong State Recreation Area is accessible from Milwaukee, Madison, and Chicago. It is a great place for the entire family to visit.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 10\n",
      "Document 8125: Indiana Dunes State Park Beach, Michigan City, Indiana Indiana\n",
      "Rate: 4.5 Review Number: 99.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g37315-d9874447-Reviews-Michigan_City_Beach-Michigan_City_Indiana.html\n",
      "Indiana Dunes State Park Beach, Michigan City, Indiana, is part of a National Lakeshore that is operated and protected by the National Parks Service of the Department of Interior of the USA. Of the 19 sections of the park, 3 of them are directly accessed here at Michigan City. The park is on the shore of Lake Michigan, the 3rd largest of the Great Lakes and the only one of the lakes that is entirely within the boundaries of the United States. The Central Beach parking lot is one of those access points found at the end of Central Avenue which is off of US Highway 12. The parking lot has room for 100 vehicles. The beach here on Lake Michigan is open from 7 AM to dusk, 7 days a week from April 16 through November 14. There are restrooms here as well as an information board listing activities occurring throughout the park. Despite being a lake there are waves and rip currents in the water, of which swimmers should beware. There are no lifeguards stationed here. Indiana Dunes State Park Beach, Michigan City, Indiana, is also the access point to the River Trail at Heron Rookery, which is found at North Country Rd, 600 East, near US 20. The site is a nesting ground for the Great Blue Heron which have more than 100 nests located here. In mid-March the birds can be seen initiating annual nest repairs. By May the nests are blocked by the tree foliage although the birds can still be seen feeding in shallow waters until the fall when they migrate to warmer southern climates. From late March through early June, the wildflowers here can be seen in bloom. There is also fishing permitted here along the Little Calumet River. The area is also open from 7 AM until dusk. There are no restrooms or other comforts in this section of the park. The parking lot is small but it does have sufficient space for oversized vehicles like RVs to turn around in order to exit. The third and final entrance to Indiana Dunes State Park Beach, Michigan City, Indiana, is at Kemil Beach, on East State Park Road (or Kemil Rd) east of Highway 12. The area has the same hours as the other parts of the park accessible from Michigan City. There is a parking lot here with a billboard of information on activities occurring throughout the park. There are restrooms here that are open from April 2 through November 14. Pets are allowed in this area of the park between Labor Day and Memorial Day. Permit holders may engage in smelt fishing in this area of the park in the spring months only.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 11\n",
      "Document 8923: Michigan's Adventure Michigan\n",
      "Rate: 4.0 Review Number: 1006.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g42499-d265969-Reviews-Michigan_s_Adventure-Muskegon_Muskegon_County_Michigan.html\n",
      "Originally called Deer Park, Michigan's Adventure opened in 1956 with a number of carnival rides and games. Today, the park has over 50 rides, including seven roller coasters. The park is actually two in one - along with the regular amusement parks, there is also a section with water rides called WildWater Adventure. Michigan's Adventure is owned and operated by Cedar Fair, the same company who owns Cedar Point. The roller coasters are a main attraction to the park. One of the most popular is the Corkscrew, which opened in 1979 as one of the most popular roller coasters in the United States. The corkscrew has two inversions, and reaches speeds of 45 miles per hour. There's also Zach's Zoomer, a wooden roller coaster for children, as well as Wolverine Wildcat, a wooden roller coaster that has a track that goes through a dark tunnel and reaches speeds of up to 55 miles per hour. Thunderhawk, opened in 2008, became the first suspended looping roller coaster in Michigan, and this coaster takes riders on five loops and drops them 85 feet. In 1999, Michigan's Adventure also opened a wild mouse style coaster called Mad Mouse, as well as the Big Dipper, a steel coaster for families. The park's flagship roller coaster, however, is called Shivering Timbers. This wooden coaster opened in 1998 and reaches speeds of up to 65 miles per hour. There are a number of drops during this ride, and is it currently the only roller coaster in the world that has a trick track,'' which throws riders from side to side, not just up and down. There's also a large percentage of airtime on the long ride. After the park opened Shivering Timbers, it was given a number of awards. Since then, it has been ranked as one of the best wooden roller coasters in the United States. Michigan's Adventure has a number of non-coaster rides as well. At the amusement park, guests can find airplanes, a carousel, bumper cars, a flying elephant ride, a flying trapeze, go-karts, motorcycles, and more. There are also a number of water park rides and attractions, including a wave pool, Speed Splashers coats, a family water play area, the Ridge Rider body slide, a number of tube slides, and more. Along with offering rides, the park hosts a number of events every year. This includes Rock the Coast, a Christian music festival held annually in May. They also have picnic areas to hold events for guests, as well as school days and scout days. Michigan's Adventure opens annually at the end of May and remains open through September. During the bulk of the summer, the park is open from 11:00 AM to 9:00 PM, with the water park area open daily from noon to 7:00 PM. There may be shorter hours on some days, and some rides close when the weather is bad. General admission costs $26.00, with special rates for two-day passes and groups of 25 people or more. They also have a number of items available for rental, including strollers, wheelchairs, wagons, scooters, and inner tubes. For more information, guests can call 231-766-3377.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 12\n",
      "Document 13430: Veterans Park Wisconsin\n",
      "Rate: 4.5 Review Number: 160.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g60097-d147149-Reviews-Veteran_s_Park-Milwaukee_Wisconsin.html\n",
      "The Milwaukee County parks system prides itself on providing recreation to residents of Milwaukee County. All over the city and county there is greenspace that has been renovated to include trails, boat docks, playgrounds, picnic areas, and covered shelters. The parks offer a place for families to gather, sports to be played, and the natural of beauty of this city by Lake Michigan to be enjoyed. One of the standout members of this park sysem is Veteran's Park in downtown Milwaukee. Situated dirctly on the shore of Lake Michigan, the park is busy on spring and summer weekends, and has also played host to various events throughout the years. When the Harley-Davidson motorcycle company hosted its 100th anniversary celebration, the festivities, including a performance by Elton John, was held in Veteran's Park. Located on the park's south end is the Milwaukee Art Museum. Just a few steps away from the park and the museum are Discovery World children's museum, and the Henry Maier Fairgrounds, which is the site of Summerfest, Milwaukee's annual music festival, as well as multiple ethnic festivals throughout the summer. The north end of the park features a large marina with a lagoon lying between. Those sailing on Lake Michigan have a perfect view of Veteran's Park, and park goers have a mostly uninterrupted view of the lake. The lakefront may be Milwaukee's best feature. Many of Milwaukee's most eclectic neighborhoods lie on the lake, and some are within walking distance of the park. There are organized boat tours that travel up and down the shoreline, most of which start in the Milwaukee River and sail out to the lake. Those who are looking for a decent walk around the lake can start at the south end of Veteran's park and follow the paths up to the marina. These walks may inspire people to practice fitness the views are so enticing. You can continue north away from the park to Bradford Beach. You can also find free parking at the beach and walk down to the park. One breezy spring afernoons, you will find a number of local residents flying kites. There is a kite stand in the park where you can find replacement parts and new kites. Not only are kite enthusiasts and fitness walkers in the park, you will also find bikers, rollerbladers, and joggers. Many people pack a picnic lunch and relax in the grass by the lake. The area can be inspirational for local artists and on a calm day you may find a variety of people painting and sketching the lakefront views. Some even bring easels to the park, taking advantage of the natural surroundings. Public restrooms can be found at the top of the hill where the veteran's memorial building is located. This is also a great place to cool off after a hot summer stroll. Take a few minutes to perfuse the awards and medals that are featured in the building's displays. Milwaukee's lakefront is one of the best features of Wisconsin. The next time you are in the area, take a stroll through Veteran's Park and enjoy the lake.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 13\n",
      "Document 8909: Mackinac Island State Park Michigan\n",
      "Rate: 4.5 Review Number: 1975.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g42423-d615400-Reviews-Mackinac_Island_State_Park-Mackinac_Island_Mackinac_County_Upper_Peninsula_Michigan.html\n",
      "Mackinac Island is located in the Straits of Mackinac, about seven miles east of the Mackinac Bridge. The island was first named a national park in 1875, making it the country's second-oldest behind only Yellowstone National Park. Today, more than 80 percent of the island is still part of the park and intact as a public preserve that's open year-round, free of charge. The Ojibwa and Odawa (Ottawa) Great Lakes Indian Tribes, who have visited Mackinac Island for more than 11,000 years, say this is where life began and hold the island as sacred. Visitors may well agree, as the island and the State Park provide a blend of breathtaking vistas, geological wonders and gems of history and nature - from lush forests, to high limestone bluffs, to sparkling water. The first thing visitors may notice is that, with the exception of government vehicles, automobiles are nowhere to be found on the island. They were banned more than 100 years ago, making for a quieter and cleaner way of life, as the park seeks to preserve and protect its woods, wildlife and natural wonders. Within the park, history is still being uncovered and discovered, and excavations have been carried out since the 1960s. In 1965, the University of Michigan Mackinac Island project excavated part of the Custer Road dump, which had been used by soldiers at Fort Mackinac. During our country's Revolutionary War, the fort was originally located in present day Mackinac City. It was manned by the British and named Fort Michilimackinac, until they closed it, renamed and relocated the fort to the island on a majestic bluff that towers 150 feet above the Straits of Mackinac. Over the course of the fort's history, it has been held by both British Redcoats and American soldiers. For a brief period during the War of 1812, the British retook the fort. Today, it serves as a living piece of history, where visitors can run their hands over wooden palisades, can still hear bugles, rifle retorts, cannon fire, and the tapping of Morse Cose. Visitors can even drill with soldiers, play a Victorian children's game, or dance to the beat of a 19th-century tune. Another park historical project took place in 2002, when the Wawashkamo Restoration and Preservation fund sponsored an archaeological survey of Wawashkamo Golf Course. The land was the site of the historic battle for Mackinac Island in 1814, and the survey was conducted by a team from the Center for Historic and Military Archaeology. The majority of Mackinac Island State Park is covered by lush forests and wildflowers. There are also a number of majestic natural limestone formations, such as Arch Rock, Sugar Loaf and Skull Cave. Arch Rock is particularly renowned, and is a source of much of the island's tourism. In the lore and superstitions of the Ottawa peoples, the rock is a bridge over which departed souls may pass. From a more scientific perspective, the rock is a 149-foot-tall limestone arch formed during the Nipissing post-glacial period. The State Park also features five historic buildings in the downtown area that allow visitors to further experience 18th-century French-Canadian and early American architecture and living.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 14\n",
      "Document 8921: Michigan Historical Museum, Lansing, Michigan, Flagship of the Michigan Historical Museum System Michigan\n",
      "Rate: 3.5 Review Number: 13.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g42545-d8809256-Reviews-Kellie_s_Consignments-Okemos_Ingham_County_Michigan.html\n",
      "The Michigan Historical Museum brings Michigan's history to life for visitors, from prehistoric times through the late twentieth century. Located in the Michigan Library and Historical Center in Lansing, the museum is the flagship of the Michigan Historical Museum System's 12 sites. The museum features five levels, twenty-six permanent galleries and a special exhibit gallery to convey the story of Michigan's past. Among its most striking visual features are a dramatic four-story atrium and a topographical relief map of the state that stands three stories high and greets visitors to the museum. The map also serves to inform and remind visitors of the natural forces and resources that have shaped the state's history, and to provide a background for stories about how Michigan's land and water have been used for industry, commerce and recreation. Some of the exhibits visitors to the musuem encounter include: The First People'' tells the story of the Paleo-Indian (ancient Indian'') peoples, who probably ventured into North America by crossing from Asia to Alaska via the Bering Straits. These Paleo-Indians traveled extensively and followed the herd animals they depended upon for food, skins and hides. They are believed to have arrived in Michigan about 12,000 years ago, where they hunted caribou, and possibly mammoths and mastodons as well, with spears. Settling a State'' showcases the story of Michigan's settlers and the state's early growth. Michigan became a territory'' in 1805 and a state in 1837, with Stevens T. Mason its first governor. During the period between 1830 and 1840, no state experienced faster growth than Michigan, thanks to access via the Erie Canal and the lure of cheap land. The Civil War gallery documents the story of the more than 90,000 Michigan men - including whites, Blacks and Native Americans - who served in the Civil War. As a strong anti-slavery state, Michigan responded quickly to the war, and the number of soldiers who served accounted for 23% of the state's male population. Though no Civil War battles took place on Michigan soil, these soldiers from infantry, cavalry, artillery, engineering, and sharpshooter regiments fought in virtually every major battle of the war. The Mining gallery features an entrance with a towering mine headframe formed of timber, and a massive piece of float copper (copper that is found loose in the soil). Inside the gallery are exhibits about Michigan's copper and iron mining, other natural resources, as well as a history of the mining communities from the state's Upper Peninsula. The Lumbering exhibit document's Michigan's history as the nation's leading producer of lumber from 1869 to 1900. Michigan was noted for its virgin stands of white pine trees, which grew tall and straight-grained. The history also demonstrates how lumbering changed with the coming of the railroads, making it possible to cut trees far from rivers, which were the primary mode of transport prior to that. The Growing Up in Michigan, 1880-1895'' gallery showcases early life in the state, as seen through the eyes of five young people who grew up during the latter part of the 19th century. Visitors can examine actual diaries and reminiscences, and exhibits such as a one-room school house, living quarters and an outdoor shed. The Farm and Factory, 1860-1900'' exhibit shows how the predominantly rural state moved toward becoming industrialized. Via those and other exhibits, the Michigan Historical Museum brings to life in dramatic visual fashion the history of the state of Michigan.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 15\n",
      "Document 8875: Forest Dunes Golf Club, Roscommon, Michigan - A Secluded, Upscale Golf Course in Northern Michigan Michigan\n",
      "Rate: 4.0 Review Number: 26.0\n",
      "https://www.tripadvisor.com/Hotel_Review-g42655-d4195645-Reviews-or20-Lake_AuSable_Lodge_at_Forest_Dunes_Golf_Club-Roscommon_Michigan.html\n",
      "Forest Dunes is an 18-hole upscale course at the Forest Dunes Golf Club in Roscommon, Michigan. A Tom Weiskopf design, the course opened in 2002 and traditionally ranks on the Top 100 list of most golf publications. It has also been ranked the #2 course in Michigan by Michigan Golf. For those who are unfamiliar with the Roscommon area of Northern Michigan, Forest Dunes is truly off-the-beaten path, as Roscommon has a predominantly rural population of around 25,000. That secluded element gives Forest Dunes a wild, natural feel to it, as the course runs through the pine trees and Birch groves of Northern Michigan's Huron National Forest, along a marsh, and through sandy dunes and grasslands. As Ron Whitten, Architectural Editor at Golf Digest, described the course, Half the holes at Forest Dunes are Augusta National style rolling down majestic corridors of red and jack pines, with elaborate bunkering of gleaming white sand. The remaining holes bring to mind Pine Valley, with dark, exposed native sand littered with scruffy underbrush. The mix is artfully blended on both nines.'' The par-72 course plays to 7,141 yards from the longest tees, with a USGA rating of 74.8 and a slope rating of 142. According to Wieskopf, commenting on his design, Forest Dunes replicates classic style and design and emulates a risk/reward involvement on most every hole. I consider this golf course to be one of the best which I have ever been involved in the U.S.'' Golfers who have played this course have said they'd have a hard time picking a favorite hole, as every hole is said to be unique and memorable. Some of the more noteworthy holes, however, include the par-3 No. 3 hole. Nicknamed Table Top,'' this hole plays to 203 yards from the I (Black) tees, but only 143 from the III (White) tees. What makes it so stunning and challenging is the forced carry over a very large sand bunker/waste area, to a green surrounded by four more uniquely shaped bunkers. The par-5 No. 5 hole is ranked as the most difficult on the course, and will probably challenge most golfers. The Mason Trail,'' as it's named, plays to 602 yards from the back tees, and a respectable 567 from the II (Blue) tees. A sharp dogleg left at the landing area will have golfers deciding if they can cut the corner over trees and sand. From there, the fairway is flanked on both sides by trees and two long bunkers/waste areas on the right, and the green features several undulations. The Point'' is a par-3 and the No. 9 hole that's both visually stunning and intimidating. The hole plays to 203 yards from the back tees and 151 from the Whites. The challenge is the forced carry over a large water hazard to a wide green flanked by five bunkers. The finishing hole is called Eagle Chance'' and is a par-5 that plays to only 531 yards from the back tees and 470 yards from the White tees, hence the name. When golfers first encounter this hole, they may think they're in the desert as they try to stay on the think strip of fairway flanked on all sides by sand and sandy grasslands. As they get closer to the hole, however, they'll encounter a large water hazard that runs along the left and around the green as the putting surface turns out into the water. An interesting feature of Forest Dunes is its 19th hole.'' Unlike most comparable names, which typically involve a snack bar/watering hole, this is an actual golf hole. The hole features a green with a bunker at its center, and the purpose of the hole is said to allow golfers one additional hole to settle a score. Practice facilities at Forest Dunes include a dual-ended driving range, bunkered greens and a short-game practice area. The facility also features a spacious Adirondack-style clubhouse with men's and women's locker rooms, as well as a bar, casual and formal dining, as well as spacious decks.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 16\n",
      "Document 8867: Eagle Eye Golf Club Michigan\n",
      "Rate: 4.5 Review Number: 23.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g41968-d4022302-Reviews-Eagle_Eye_Golf_Club-Bath_Clinton_County_Michigan.html\n",
      "Eagle Eye Golf Club is a public course and part of the Hawk Hollow Community, which features 45 holes of championship-caliber golf, as well as a nine-hole walking course and 18-hole putting course. The Eagle Eye course opened in 2003 and is situated just a few minutes from the campus of Michigan State University in East Lansing, Michigan. The 18-hole, par-72 course was designed by Chris Lutzke, in collaboration with legendary golf course architect Pete Dye, and is a rolling, links-style layout. The course plays to a whopping 7,318 yards from the longest tees, with a USGA rating of 76.4 and a slope rating of 145. The course offers golfers five different tees from which to choose, however, and the White tees play to a mere 5,928 yards. The course has been described as visually spectacular and features the traditional links-course elements of fescue mounding and abundant sand bunkers. The landing areas are generous, but there are also numerous water hazards to contend with on 13 of the 18 holes. The greens tend to be very large and have been described as true-rolling.'' Golfers who have played and reviewed the course have described it as very difficult, and many a golf ball has been lost to the water that's almost everywhere on this course. Some reviewers have even said that the course asks you for a pint of blood,'' but also added that the par-3 holes on Eagle Eye will keep golfers interested. Hole No. 2 is one of those par-3s, a very short hole at 112 yards from the Whites, and 171 yards from the back tees. This is a fairly straight hole, but the catch is carrying one of the longest bunkers that golfers are ever likely to encounter on a par-3. Another noteworthy par-3 is No. 5, which trades sand for water. The hole plays longer at 146 yards from the White tees and 210 from the longest tees, and water runs the length of the hole on the left and the hole tends to tilt toward it, running away from the hills on the right. Hole No. 15 is a par-4 that plays to only 394 yards from the back tees, but water starts on both sides of the fairway and follows the right side of the hole up to the green. Adding to the challenge, the hole sits atop a hill that can be deceiving. Anything short will roll down the sloping crest, perhaps even beyond the original lie. The signature hole at Eagle Eye is the par-3 17th, which is modeled after the famous island green at Pete Dye's Stadium Course at TPC Sawgrass. The hole plays to only 146 yards from the longest tee, but the green sits in the middle of water and steady nerves and the right club selection are mandatory on this hole. Since opening Eagle Eye has gotten some rave reviews and honors, including being named the 5th-best new course in America by Golf Digest in 2005. Michigan Golf Magazine also rated it the 4th-best course in the state in 2006. Practice facilities at Eagle Eye include a two-tiered driving range and two undulating putting greens. The course also features a luxurious, 65,000-square-foot clubhouse with locker rooms, a full-service pro shop and restaurant and lounge.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 17\n",
      "Document 8964: Tullymore Golf Club Michigan\n",
      "Rate: 4.0 Review Number: 75.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g42733-d214520-Reviews-Tullymore_Golf_Club-Stanwood_Mecosta_County_Michigan.html\n",
      "Tullymore Golf Course is one of two courses found at the Resorts of Tullymore and St. Ives in Standwood, Michigan. Collectively, they have been called the highest-rated pair of golf courses of any Michigan resort. Tullymore was designed by Jim Engh and Jerry Matthews and features a unique layout that incorporates elements of European courses and traditional American designs. One of the most noteworthy elements of the course is some dramatic bunkering that is unlike anything most golfers have encountered. Soft, white sand is often surrounded by intricate, contoured mounding and the bunkers weave throughout the fairways as well as the green complexes. Such hazard are sure to affect shot and club selection, and may create more of a challenge than the many pines and hardwood trees that line the fairways. There are also some daunting water hazards, including Shinglebolt Creek, found on the course, as well as some subtle ups and downs as the tranquill course winds through 800 acres of woodlands. The course has been described by Golf Digest critic Ron Whitten as having an art deco flavor with its greens and fairways that follow precise and parallel zigzags, its artictically shaped bunkers and it symmetrically repetitive mounding. That architectural artistry is only part of what will fascinate you during your round.'' Whitten also had this to say: Tullymore is a bedazzling golf course, a unique style of golf design that's imposed upon the landscape without being an imposition.'' The course plays to 7,148 yards from the Silver tees, with one of the highest slope ratings in the Midwest at 148. From the Whites, the course only plays to 5,550 yards, so most golfers will choose something in between and play either the Black or Blue tees. Some of the more noteworthy holes include No. 2, rated the most difficult on the course. This par-4 hole plays fairly straight, but is a bit lengthy at 456 yards from the back tees, but only 397 from the Blues. What makes this hole so challenging is the water that runs up the left side for most of the hole, the undulating fairway, and the signature bunker that looms upon the right about 130 yards from the green. Hole No. 7 is par-3 that plays to only 148 yards from the Blue tees, but it's said to have one of the testiest greens on the course. The putting surface is 47 yards deep, and also features some dramatic undulations, which makes bogey on this hole a very frequent occurrence. Another great hole is the par-3 12th, which golf writers have said resembles the famous Dell Hole at Lahinch Golf Club in Ireland. The holes plays to 257 yards from the back tees, but only 157 yards from the Blues. What makes it so daunting, however, is the carry over water to a green tucked behind some mounding that can obscure the target. The finishing hole is a par-5 that plays to 478 yards from the Blue tees, and is also one of the most photogenic to be found. The hole wraps around a lake, and has been called by Golf Chicago as one of the greatest finishing holes in Michigan.'' Practice facilities at Tullymore include a generous driving range, and a short-game area for pitching and putting. Free range balls are included with every round of golf. The course offer features a new clubhouse with a Pro Shop, as well as several options for dining, including the Tap Room at Tullymore and Devine's Fine Dining,\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 18\n",
      "Document 8946: The Binder Park Zoo in Battle Creek, Michigan Michigan\n",
      "Rate: 4.5 Review Number: 545.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g41969-d630592-Reviews-Binder_Park_Zoo-Battle_Creek_Calhoun_County_Michigan.html\n",
      "The Binder Park Zoo, located outside the city of Battle Creek, Michigan, is home to 140 species of animals. First opening its doors in 1977, the zoo is situated on 433 acres of natural forest and wetlands and is accredited by the Association of Zoos and Aquariums and a member of the World Association of Zoos and Aquariums. A wide variety of exhibits makes the zoo an ideal place to take the family for a learning adventure. The Wild Africa exhibit features animals that live on African savannas, which are flat grasslands. The realistic 50-acre environment includes zebras, giraffes, impalas, ostriches and many other species native to the country. Visitors can take part in an interactive component of the exhibit by feeding giraffes from a special, tall deck, called a giraffe deck. Positioned near the zoo's entrance, the Smith Snow Leopard exhibit introduces visitors to the unusual cats in a realistic, state-of-the-art setting. Conservation and education are key elements of the exhibit. The best time of year to see the cats is in the fall when they are most active. The hand-painted, carved wooden Binder Conservation Carousel features two chariots and 36 species of animals. Installed at the zoo in spring 2007, the ride has become a popular attraction for visitors. Zebras, giraffes and snow leopards are among the animals carved into the structure, as well as three horses, as a tribute, painted in the Michigan State University, University of Michigan and Western Michigan University school colors. All profits from the carousel are used for the zoo's conservation programs. Families can choose from a variety of activities. Education and conservation opportunities include camps, an Overnight Safari and Zoo Snooze programs. The Family Overnight Safari allows guest to camp in the Wild Africa exhibit. Along with a special, after-hours tour of the exhibit, families enjoy dinner and an evening snack, and stay overnight at the zoo in a tent from home, or in one of the structures on site. Breakfast in the morning completes the experience, which is a great bonding opportunity. Sunrise on Savanna provides visitors the chance to have breakfast on the giraffe deck. The breakfast includes lectures on African wildlife by the informative zoo staff. The Zookeeper for a Day program gives guests a glimpse of what it is like to work with the animals in the zoo. It allows those interested to take a hands-on approach to visiting the zoo. Also available are teacher and home school programs, birthday parties, environmental tours and field trips. The programs are educational and interactive and a great way to create memories that will last a lifetime. The Binder Park Zoo has become one of the top cultural attractions in the Battle Creek, Michigan area. The zoo is informative and interactive, with family friendly programs and state-of-the-art exhibits and attractions. The diversity of opportunities at the zoo makes it appealing to visitors throughout the seasons.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 19\n",
      "Document 8847: Arcadia Bluffs Golf Club Michigan\n",
      "Rate: 4.5 Review Number: 235.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g29558-d580627-Reviews-Arcadia_Bluffs_Golf_Club-Arcadia_Manistee_County_Michigan.html\n",
      "Arcadia Bluffs Golf Club opened in 1999 as a daily-fee course and, despite its location off the beaten path in Northern Michigan, it has received raves as one of the most visually appealing golf courses in the country - perhaps even the world. The links-style course is carved out of 245 acres of windswept land atop the bluffs overlooking Lake Michigan. From its highest point, the course drops 225 feet, and also features 3,100 feet of shore frontage along the lake. It's been argued that Arcadia Bluffs is not a true links-style course, but more of a seaside links course that features rolling terrain and windswept fescue grasses. Designed by the team of Warren Henderson and Rick Smith, the course is said to evoke memories of courses such as Lahinch and Ballybunion, while contributing some breathtaking views. The par-72/73, 18-hole course plays to 7,300 yards from the championship tees, and a more manageable 6,702 from the Black tees, with a USGA rating of 72.8 and a slope rating of 147. The course features some of the widest fairways golfers will ever find on a challenging course, along with 120 total bunkers, including 50 sod-walled bunkers, lots of native grasses, and large greens that average 8,000 square feet. The real challenge of this course tends to show up from 50 yards on in, with greens that are very undulating and sod-wall bunkers that can be so deep that they make the famous road hole'' bunker in Scotland look tame by comparison. In addition, the terrain is not flat and it's often a challenge to find a level lie. And there are also the wind off Lake Michigan to contend with, which, depending on which way it's blowing, can make club selection another challenge. Like some of the European links courses, which have been described at times as quirky,'' Arcadia Bluffs follows a similar non-traditional routing on the front nine. The course begins with a par-5 hole that plays to 519 yards from the longest tees, and ends with a lengthy par-3 of 203 yards from the Championship tees and 176 yards from the Blacks. The par-5 11th hole is particularly scenic and challenging. The hole plays downhill but to a hefty 633 yards from the back tees, and a respectable 594 from the Blacks. The fairway narrows dramatically once past the landing area, and golfers will often have to contend with an incoming wind as the hole plays directly down toward the water. Another noteworthy hole, and one of the most difficult par-3s on the course is the 13th. The hole plays dramatically downhill and has a forced carry over rugged rough to a green set against the backdrop of Lake Michigan. The winds and the perception that this hole is longer than it looks will have many tees shots finding either the deep bunker in front or the dramatic slopes to the left. GPS-equipped golf carts are available, as well as a caddy program, to assist golfers with distance and club selection. The course has received some prestigious awards, including being ranked #10 by Golf Digest among the Top 100 Public Courses, 2009-2010, and a #7 ranking in 2008 from Golf World Magazine among the Top 50 Public Courses. Practice facilities include a full range and separate short-game facility. There's also a Nantucket, coastal estate-style clubhouse with breathtaking views of the course and Lake Michigan, as well as a Golf Shop and the Restaurant at Arcadia Bluffs for full-service dining.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 20\n",
      "Document 8071: W Chicago Lakeshore Illinois\n",
      "Rate: 4.0 Review Number: 2284.0\n",
      "https://www.tripadvisor.com/Hotel_Review-g35805-d225108-Reviews-W_Chicago_Lakeshore-Chicago_Illinois.html\n",
      "This luxury resort in Chicago offers overnight accommodations and a spa. Guests are treated to city and lake views, Bliss bathroom amenities, down pills, and featherbeds. The 520 guest rooms come in a variety of different styles to suit the needs of all guests. Choose from the most affordable Wonderful Room, the slightly upgraded Spectacular Room, or upgrade to one of the suites like the Fabulous, the Marvelous, or the Extreme Wow. This luxury suite provides lake views from every angle at a height of more than 30 stories high. With 1,100-square feet of space, this suite offers everything you need and more. The W Chicago Lakeshore is located near the city's best attractions including the Field Museum, the John Hancock Observatory, Chicago's best known universities, the McCormick Place Convention Center, Navy Pier, the Theatre District, State Street, Lake Michigan, Ontario Beach, the Loop, and shopping on the Magnificent Mile. Guests can shop at Nordstrom, the Water Tower palace, 900 North Michigan, and right in the hotel at the W Hotels the Store. It is also within walking distance of a number of corporate offices providing the perfect accommodations for business travelers. The W offers convenient accommodations near Chicago's best attractions. \n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 21\n",
      "Document 13424: Travel Lake Michigan with Speed and Comfort Wisconsin\n",
      "Rate: 4.0 Review Number: 326.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g60097-d667301-Reviews-Lake_Express-Milwaukee_Wisconsin.html\n",
      "The Lake Express ferry passes over Lake Michigan between Wisconsin and Michigan. It is a two and a half hour ride, allowing travelers to make the trip without worrying about driving through the harried traffic of Chicago. In addition to avoiding the long drive, you get to relax and enjoy the trip across the lake. Instead of focusing on traffic, you can kick back, enjoy your family and friends, and arrive in comfort. The ferry offers a variety of daily trips, and tickets can be purchased in person, by phone, or online. Best of all, travelers can transport their car along on their journey so when they reach their destination, all they need do is drive off the ferry and be on their way. The ferry can hold 46 vehicles, as well as twelve motorcycles. There are also Hertz rental car facilities located at the terminals on both sides of the lake, so if you would rather rent a car during your trip, it is easy and convenient to do so. The Lake Express terminals are newly built and offer easy highway access and ample parking. You can leave your vehicle for long and short periods of time, without worrying about the safety and security of your car. The ferry is designed for comfort and offers light breakfast meals, salads, and warm sandwiches, as well as coffee, soft drinks, wine, and beer. Quite frankly, the ferry is more comfortable than some of your better airline travel. Seating is wide and comfortable, and riders can relax and stretch their legs, or move throughout the cabin at their leisure Premier seats are also available, allowing passengers wider seating, with larger areas of space around them. Complimentary beverages and newspapers are provided to those riding in the Premeir Class Cabin. The ferry is great for those traveling on business from one location to another. Cell phones and other electronic devices are permitted during passage. There is also free wireless internet connections available at the terminals, as well as a variety of amenities including travel guides and food service.The ferry is a smoke-free environment to ensure the comfort of all guests. In both the Michigan and Wisconsin locations, there are a variety of activies one can enjoy once they have used the ferry to travel. Within minutes of the terminal in Milwaukee are various restaurants, entertainment venues, and miles of beautiful lakefront. The Muskegon, Michigan terminal features sandy beaches, and historical attractions. Biking, wine trails, shoping, and night life are all near the terminal. Many residents of the respective cities travel on the ferry just to enjoy the attractions and recreation for a day trip. Whether it is the excitement and fun of the city of Milwaukee you seek, or you are looking to get away from all of that for a quiet stroll through Michigan wine country, the Lake Express ferry can get you there in comfort and with speed. Kick back, relax, and transport you and your car to your destination, stress free.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 22\n",
      "Document 8959: The Townsend Hotel Michigan\n",
      "Rate: 4.5 Review Number: 1471.0\n",
      "https://www.tripadvisor.com/Hotel_Review-g41996-d89959-Reviews-The_Townsend_Hotel-Birmingham_Michigan.html\n",
      "Designed as a reflection of European elegance, the lobby of the Townsend Hotel features a glittering crystal chandelier, cherry wood paneled walls, and sprays of fresh flowers. The guest rooms and suites are similarly well appointed, with fine furnishings, luxury beds and linens, and marble bathrooms featuring Bulgari personal amenities. It may come as little surprise to learn that this is Michigan's only hotel to receive both a Mobil 4-star and a AAA 4-Diamond rating.  Facilities on the premises include a concierge service, a 24-hour fitness center, and a state-of-the-art business center. For gatherings of five to 500 guests, there is the 5,000-square-foot Townsend Ballroom, complemented by the 2,700-square-foot Regency Room. The hotel's boardrooms are also available for meetings, and a full range of audio/visual equipment can be provided along with expert staff assistance. The Townsend Hotel is particularly proud of the Rugby Grille, named by Zagat's Restaurant Guide as One of the Top Ten Best Restaurants in the United States. It offers innovative American-Continental cuisine, plus an international selection of fine wines and spirits, in a cherry-paneled dining with a cozy fireplace. Lunch and dinner are served daily, breakfast is offered on weekdays, and Saturdays and Sundays feature a special weekend brunch menu. Also on site is an urban chic bar and nightclub called The Corner. Happy hour drink specials, music and dancing, and a cigar area are a few of its attractions. The Townsend Hotel is located 40 minutes from Detroit Metropolitan Airport at 100 Townsend Street, Birmingham, Michigan 48009.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 23\n",
      "Document 13275: Cream City Charm in the Midwest Wisconsin\n",
      "Rate: 4.0 Review Number: 5.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g29590-d4706312-Reviews-Almelund_Mercantile-Almelund_Minnesota.html\n",
      "Milwaukee has a reputation for being a blue collar, hard-working, friendly town. Midwest charm is in plentiful supply in this city by Lake Michigan. The city offers a wide variety of dining options, numerous luxury accomodations, outdoor recreation, arts and entertainment, museums, and the comfort of friendly hometown people. While Milwaukee may not be the number one destination for vacationers in the United States, those who find themselves there for business, pleasure, or just passing through are always surprised at the charm the city has to offer. Sometimes known for its array of restaurants, it has been a long time since Milwaukee featured on steakhouses. Dining options include sushi establishments like Sake Tumi, organic, local creations at Roots, lakeside dining at Coast, and hip bistro dining at Envoy. Of course, it is the Midwest, so steak restaurants are in no short supply, even with the modernization of the city's dining landscape. Sandford is one of the city's best, nestled into a quiet neighborhood on the east side of town, barely recognizable as different than the homes surrounding it. If you are staying in Milwaukee, there are near-limitless options for your stay. The Knickerbocker on the lake hotel is part owner-occupied, part hotel featuring comfortable, homey suites. Another boutique hotel in the area is Hotel Metro, a modern take on hospitable accomodations. Bike fans will love The Iron Horse hotel, located conveniently near the Harley-Davidson museum. The dycor of the hotel reflects the industrialized style of the Harley's Milwaukee heritage. Located right on Lake Michigan, Milwaukee offers visitors and residents a chance to bike, hike, stroll, and skate right by the water. The city has made an effort to not over-commercialize its landscape, so you can enjoy the greenspace near the lake. A visit to the Milwaukee Art Museum would not be complete without a stroll around the grounds overlooking Lake Michigan. Throughout Milwaukee County are public parks that offer visitors and residents a chance to relax, play, and exercise. The art district of Milwaukee is packed with theatres, museums, and galleries. Various locations around town also feature a chance for visitors to the city to experience the rich art community Milwaukee has to offer. Most of these options are within walking distances of one another, as well as the area restaurants and nightlife. During the festival season, visitors can attend an ethnic celebration at the Henry Maier Festival Park. If you are in town around July 4th, check out a show at Summerfest, the city's annual music festival. One entrance fee will get you access to more than five stages of musical performances, entertainment, vendors selling handicrafts, and delicious local cuisine. Milwaukee provides visitors with a hometown feel that can only be found in the Midwest. The beauty of the lake, the culture of the museums and entertainment, and the delicous cuisine built around the carnivorous appetites of Midwesterners will have you wanting to return to the city again and again. Whether you are in town for business or pleasure, you will never lack for things to do.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 24\n",
      "Document 8060: The Westin Michigan Avenue Chicago Illinois\n",
      "Rate: 4.0 Review Number: 3066.0\n",
      "https://www.tripadvisor.com/Hotel_Review-g35805-d87657-Reviews-The_Westin_Michigan_Avenue_Chicago-Chicago_Illinois.html\n",
      "The Westin Michigan Avenue Chicago, Chicago, IL hotel is in an upscale neighborhood near the business district. Locals call the location the Magnificent Mile, which is also across from Bloomingdales and other Water Tower Place shops. Lake Michigan at Oak Street Beach is a short drive away. The Westin is known for their services for business travelers with meeting space, state of the art business equipment and services, and their workout gym.  The Westin has a variety of different rooms such as the traditional room, deluxe lake view, deluxe Michigan Avenue, and Executive suite. Each room runs about 350 square feet with Heavenly Beds from the Westin trademark. Working space, televisions, beautiful décor, Heavenly Shower products and facilities, and telephones can be found in the rooms. Dining is available at two locations: Café a la Carte and the Grill on the Alley. Café a la Carte is open from 6am to 12pm for breakfast and lunch. It is a coffee shop serving light cuisine, with Starbucks coffee. The Grill on the Alley is a full service restaurant for fine dining on American cuisine. This restaurant is in the Hall of Fame by Nations Restaurant News.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 25\n",
      "Document 8904: Lakewood Shores Resort, The Gailes Course, Oscoda, Michigan - The Look and Feel of the Famous Seaside Golf Courses of Scotland Michigan\n",
      "Rate: 3.5 Review Number: 51.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g42559-d219135-Reviews-Lakewood_Shores_Resort_Blackshire_Golf_Course-Oscoda_Michigan.html\n",
      "The Gailes Course at Lakewood Shores Resort in Northeastern Michigan is one of three distinctly different courses found at the resort. The Gailes Course, which opened in 1992, was designed by Kevin Aldridge and features rolling fairways, pot bunkers, tall, fescue-covered mounds and heather roughs, which create the look and feel of the famous seaside courses of Scotland. The Gailes is said be to an old-world'' golf experience - a true Scottish Links design with meandering berms, sod-faced bunkers and occasionally strong winds blowing in off Lake Huron. The fairways at The Gailes are generous and nicely sculpted and, because the grass is cut so short, particularly close to the greens, golfers will find their putters the club of choice even many yards from the actual putting surfaces. The greens themselves tend to be huge, but are fast and undulating. Encountering a crown or swale on the putting surface is not uncommon. The 18-hole, par-72 course plays to 6,954 yards from the longest tees, with a USGA rating of 75.0 and a slope rating of 138. Even the much shorter Blue (6,393 yards) and White tees (6,073 yards) can prove a challenge for first-time visitors to the Gailes. That's because Lakewood Shores has chosen not to make use of a pin-placement system - an intentional design feature that forces golfers to adhere to the time-honored tradition of calculating distance for themselves. Some of the more noteworthy holes include No. 5, the #1-handicap hole. The par-4 plays to 498 yards from the back tees, but a more manageable 408 from the Blues. Difficulties can be encountered with the creek that crosses the fairway about 180 yards from the tee, as well as the fescue-covered mounds on both sides of the fairway. The green is designed for a bump-and-run approach, but golfers must be wary of the pot bunker on the right and a grassy swale on the left. Hole No. 10, Over the Hill,'' is a short par-4 that captures the essence of links golf. Hidden hazards abound and the hole demands one of the best tees shots of the round. There's a forced carry off the tee that requires a long iron to a landing area surrounded by a quagmire of mounding, deep sod-faced bunkers and knee-high fescue. The approach shot that follows plays to a slightly hidden green guarded by bunkers on the left. The Gailes has received a number of awards and accolades since opening, including being named the #1 Resort Course in Michigan by the Detroit News Golf Guide, 1997, and being ranked the #1 Public Resort Course in Michigan by Golf Digest in 1996. Practice facilities at The Gailes include a driving range with 15 tees, two putting greens and the Wee Links, a pitch-and-putt course with holes ranging from 50 to 105 yards. The course also features a full-service clubhouse with both casual and fine dining at the Scotland Cove restaurant. Lodging is also available in the resort's 152 rooms, and other activities include a private beach on Cedar Lake for swimming, jet skiing, canoeing and fishing. The resort is also just minutes away from Lake Huron.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 26\n",
      "Document 8017: Sheraton Chicago Hotel and Towers Illinois\n",
      "Rate: 4.0 Review Number: 5790.0\n",
      "https://www.tripadvisor.com/Hotel_Review-g35805-d87638-Reviews-Sheraton_Grand_Chicago-Chicago_Illinois.html\n",
      "With views of the Chicago River and Lake Michigan, the Sheraton Chicago Hotel and Towers puts guests in the heart of Chicago and offers everything needed for a comfortable stay.  All the 1,249 renovated guestrooms afford views of the river, the lake, or the Chicago skyline. In addition, the Sheraton offers guests many updated amenities, such as rich wood furnishings, windows that open, in-room safes, flat-screen TVs, the Sheraton Sweet Sleeper Bed, and bathrobes. Guests can choose between a standard room, club room, an array of suites, or the Presidential Suite. This suite includes an iMac, library, Bose sound system, and a full luxury bathroom. For those needing event or meeting space, this hotel houses 37 meeting areas, the largest of which measures 3,716 square feet. In addition, event staff, planners, and caterers are ready to assist with any event planning needs. Just 12 miles from Midway Airport and 18 miles from O'Hare International Airport, The Sheraton Chicago Hotel and Towers is located close to many sites and transportation options. This hotel is within walking distance of Michigan Avenue, Navy Pier, and Millennium Park. Also nearby, tourists can enjoy the Art Institute of Chicago, Alder Planetarium, Financial District, McCormick Place Convention Center, Buckingham Fountain, Navy Pier, Soldier Field, and Wrigley Field.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 27\n",
      "Document 8881: Grand Traverse Mall Michigan\n",
      "Rate: 4.0 Review Number: 33.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g42758-d5772876-Reviews-Grand_Traverse_Mall-Traverse_City_Grand_Traverse_County_Michigan.html\n",
      "Open seven days a week, the Grand Traverse Mall is one of the most popular shopping centers in northwestern Michigan. The mall features over 100 stores, as well as a movie theatre, food court, and sit-down restaurants. There are also many events held at this mall every year, and visitors from other parts of Michigan or out of state don't have to travel far to find accommodations because the its convenient location. Grand Traverse Mall personal offer a number of amenities to help shoppers make the most of their time at these stores. There are on-site ATMs, play areas for children, lockers for those staying at the mall all day, stroller and electric scooter rental offices, wheel chair services, and family restrooms. In addition, the mall runs a taxi service to and from Traverse City locations, offers auto jump-start service for those experiencing car problems, and has safety escorts around the mall and through the parking lot. There are four anchor department stores at the Grand Traverse Mall - T.J. Maxx, Macy's, J.C. Penney, and Target. From there, guests can explore any of the four zones (A, B, C, and D), which feature stores in a number of categories. Some of the clothing stores at this mall include Abercrombie & Fitch, Hot Topic, Pacific Sun, Lane Bryant, Charlotte Russe, American Eagle Outfitters, and Aeropostale. There are also a number of jewelry stores found at the Grand Traverse Mall, as well as stores that sell shoes, electronics, toys, children's apparel, and items for the home. Shoppers at this mall can, in addition to visiting the stores, take advantage of the service-orientated shops at this mall, including the nail salons, hairstylists, accountants, eye care centers, cell phone providers, and photo development services. Between the food court and the sit-down options, shoppers have over a dozen eating options. For quick meals and snacks, visitor's can try Mrs. Field's Cookies, Subway, Auntie Anne's Soft Pretzels, or one of the many fast food places. There are also more up-scale restaurants like Garfield's and the Jonathon B Pub. One of the unique features of this mall is its carousel. Located in the food court, this antique ride is popular among both children and adults. Ride tokens only cost one dollar, and residents who frequently shop at the mall can purchase frequent rider tickets in order to earn free rides. Even people who don't like to shop might find themselves at this location, as the Grand Traverse Mall has a fairly large move theatre. They have nine screens, with tickets costing a range of prices depending on the time of the movie and the age of the moviegoer. Typically, there are five to nine different movie options, and all of the theatres have disabled seating availability, as well as devices for those who need help hearing. The Grand Traverse Mall has a number of programs that shoppers can join, include The Club, a free mailing list that sends members discount information and coupons, as well as has a monthly shopping spree prize. They also sell gift cards, get involved with local community organizations for sponsor events, and have fundraising programs for Michigan schools. To learn more about the mall and all of their programs, shoppers can call 231-922-0077.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 28\n",
      "Document 8939: River Town Crossings Michigan\n",
      "Rate: 4.5 Review Number: 92.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g42257-d3825478-Reviews-RiverTown_Crossings-Grandville_Kent_County_Michigan.html\n",
      "River Town Crossings is located in Grandville, Michigan which is in the Grand Rapids area. River Town is currently operated by General Growth Properties. They are one of the larger real estate investment companies in the United States offering outdoor and indoor shopping centers. River Town Crossings is a relatively new mall, being opened in 1999. It has 5 anchor tenants with 1.3 million square feet and two floors. The mall opened with Sears, Hudson's, Khol's, Younkers, and JC Penney as the anchors. Barnes and Noble was also a junior anchor. Gaylan's, which is now Dick's Sporting Goods and Old Navy, opened a few months after the mall. With 120 stores, nine restaurants, and a 20 screen Celebration Cinema the mall is quite popular. There are 6,000 parking spaces at the mall for visitors, as well as some bus stop locations. Today the anchor stores mentioned above have grown to a few more including Jack Loeks Theatres, Great Lakes Home Furnishings, and Macy's. Hudson's went through a buyout, eventually becoming the Macy's brand. Mall hours are Sunday 12pm to 6pm and Monday through Saturday 10am to 9pm. These hours can vary with holidays. The department stores can also set their own hours since they have outside entrances from the mall. During the year one can expect to see numerous sales events, mall walkers in the morning, and seasonal events. River Town Crossings has numerous dining and entertainment options. There is a carousel on the upper level near the food court for children to enjoy. Also for children is the Children's Soft Play Area. This is located in the lower level, with plenty of soft places to climb and have fun. For dining outside the food court there is Max and Erma's, Olga's Kitchen, Olive Garden, On the Border, Panera Bread, Red Robin, and TGI Friday's. Shopping locations are 2 Fast Jewelers, Abercrombie and Fitch, Aerie, Aeropostale, Lane Bryant, LensCrafters, MasterCuts, Aldo, Maurillo's, American Eagle Outfitters, Men's Warehouse, Motherhood Maternity, Banana Republic, New York and Company, Bare Essentials, NYS Collection, Barnes and Nobles, Bath and Body Works, Belts and Buckles, Body Shop, Brookstone, Pacific Sunwear, Build a Bear, Bryon Bank, and Payless Shoe Source. Also at the mall are Pearle Vision, Perfumania, Calendar Club, Campus Den, Red Eagle Massage, Regis Salon, Cellairis, Rogers and Hollands, Centennial Wireless, Rudolph and Me Ornaments, Champs, Rue 21, Charlotte Russe, Shi by Journey's, and CJ Banks. No matter what a shopper might want there will be someplace for every shopper. Not only are there several great stores and restaurants, there are plenty of other eateries as well. In the food court are A&W, Auntie Anne's, Cajun Cafe, China Maxx, Subway, Crazy Candies, Surf City Squeeze, Dairy Queen, Orange Julius, and Sweets and Snacks. Also at River Town Crossings are Jerkeis Jerky Factory, Taco Bell, Kilwin's, Maui Wowi Smoothies, Villa Pizza, and Wetzel's Pretzels. River Town Crossings is one of the largest malls in the state of Michigan offering plenty of shopping and eatery options for the young and older generations.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 29\n",
      "Document 8044: The Loop Neighborhood in Chicago, Illinois Illinois\n",
      "Rate: 4.5 Review Number: 1256.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g35805-d109711-Reviews-The_Loop-Chicago_Illinois.html\n",
      "The Loop is a term used to identify the historic area in downtown Chicago. The actual term the Loop'' has different meanings, but most commonly applies to the area surrounded by the loop train circuit. The Loop is a community area delineated by the University of Chicago as number 32, as the area spread, the term loop has been used to describe the entire downtown Chicago area. The Loop neighborhood is dominated by high-rises and historic buildings. The most notable structure is the Home Insurance Building, which was considered to be the first skyscraper; the building was demolished in 1931. The Chicago Board of Trade Building, and the Sears Tower, which is the tallest building in America, can also been found in the loop. Some of these historical buildings were instrumental in the construction of high-rise buildings years later. The neighborhood general boundaries are to the north, the Chicago River, on the south Congress Park, and Michigan Avenue to the east. The area South of Congress is identified as the South Loop'', and the name should not be confused with the south portion of the Loop neighborhood. The Loop Proper is a five-by-seven block area within the circuit of the Elevated Transit Authority train. New Eastside, is a multi-use district bordered by Michigan Avenue, the Chicago River, Randolph Street and Lakeshore Drive. Most of the structures are built over what was once the rail line of the Illinois Central Railroad yards. The neighborhood has a triple-level street system dissected by Columbus Drive. The Printers Row neighborhood used to be known as Printing House Row, and is located in the south portion of the Loop. The main street of the neighborhood is Dearborn Street, which is where the annual Printers Row Book Fair is held. Many of the former office buildings have been converted into residential lofts and a portion of Printers Row has been declared an official Landmark District. The South Loop is mainly composed of the area south of Congress Parkway, and is located east of the Chicago River. The southern boundary of the South Loop is under debate while the phrase South Loop'', is used to describe the area including Roosevelt Road and farther south to 18th Street and Cermac Road. Also found in the South Loop is the National Vietnam Veterans Art Museum. The West Loop is mostly referred to as West Loop Gate and is a portion of the West Side. While the Historic Michigan Boulevard District is a section of Michigan Avenue located across from Grant Park and Millennium Park.  Chicago's Lakefront is mostly a recreational area that stretches from the north to the south and provides entertainment and other pastimes. The Loop continues to be a growing environment with a thriving entertainment industry, delectable restaurants, high-end boutiques and many other venues that people find interesting and inviting.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 30\n",
      "Document 5239: Caladesi Island State Park, Dunedin, Florida Florida\n",
      "Rate: 4.5 Review Number: 1361.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g34187-d126855-Reviews-Caladesi_Island_State_Park-Dunedin_Florida.html\n",
      "Caladesi Island State Park, Dunedin, Florida, is located off of the Gulf Coast in the western waters of the Sunshine State.'' As an island it is not accessible by car; it can only be reached by private boat or by ferry. The ferry leaves every hour (starting at 10 AM) from Honeymoon Island State Park, in Dunedin, which is located near Tampa, in the southern part of the State. There is an admission charged for the ferry although children under 4 years of age ride for free. No pets are allowed. The park is open 365 days a year from 8 AM to sunset. There is no camping on the Island. Visitors by ferry are given four hours to visit on the Island before they are expected to return to the mainland. It was during a hurricane, in 1921, that the Island was separated from Honeymoon Island which it had been part of previously. Some people have been known to walk to the Island from Clearwater Beach. It is a one hour walk through shallow water. Water and electric hook ups are available for visitors that arrive on their own boats through the Dunedin Causeway Channel (about 1 mile from Marker 14) to the 108-slip marina. Boats are permitted to remain docked overnight with passengers aboard. In the cooler months, on a monthly basis, the State Park Rangers, (who run the facility), host a complimentary breakfast and talk on the deck of the Cafy Caladesi. The Rangers also host guided nature hikes on every Sunday at 1 PM starting out from the Ranger Station. The hike lasts about one hour. In the Mangrove salt water forest there is a marked trail for kayaking; boats can be rented at the marina on the Island. Caladesi Island State Park, Dunedin, Florida, is on an undeveloped Island which has been rated as having the best beaches in The United States. Its 3 miles of white sand beach is a main attraction to some, while other visitors prefer to wander the nature trails. Picnic pavilions can be reserved for a fee. There is also a snack bar and gift shop in the park. Fishing is permitted to anyone in compliance with Florida saltwater fishing regulations. Setting fires, alcohol, firearms and fireworks are all prohibited in the park. It is not uncommon to see dolphins in shallow waters near the island, as well as a variety of birds at home around and on the Island. Small mammals like armadillos and rabbits as well as some snakes also inhabit the park. Caladesi Island State Park, Dunedin, Florida, has been inhabited by only a handful of people over the years. Myrtle Scharrer is said to have been the only person born into a home on Caladesi Island, in 1895. In 1946 she deeded the 157 acres as a park, to the Audobon Society and the Clearwater Marine Science Center.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 31\n",
      "Document 8849: Belle Isle Michigan\n",
      "Rate: 4.0 Review Number: 1090.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g42139-d272245-Reviews-Belle_Isle_Park-Detroit_Michigan.html\n",
      "Belle Isle is an island park in Detroit, Michigan. At just under 1,000 acres, it is the largest park in the country to cover a complete island, and this island is the third largest island in the Detroit River. The 2.42 square mile island is home to a number of recreational spots managed by the Detroit Recreation Department, including the Anna Scripps Whitcomb Conservatory and a small golf course. Belle Isle was originally colonized by French settlers in the 1700s and was at one point a private estate. Once known as Hog Island, the island got its current name in 1845. In the 1880s, at the height of the movement to beautify urban landscapes with formally planned parks, Belle Isle was landscaped by Frederick Law Olmsted. Olmsted, a well-known urban landscaper, was also the designer of Central Park and the Arnold Arboretum in Boston. Between the time that the island was landscaped and the mid-1900s, a number of buildings and structures were added to the island. In 1887, architects John Donaldson and Henry Meier designed and built a Queen Anne style casino building on the west end of the island. When the casino was destroyed by fire, the city chose Albert Kahn to design its replacement. The Renaissance style brick and terra cotta building was built in 1908, and is still used for weddings and events today. Kahn also designed and built the Anna Scripps Whitcomb Conservatory, the oldest public conservatory in the country. For years, Belle Isle was home to a herd of fallow deer, a children's zoo and an aquarium, but the city closed the animal facilities due to budget problems. The island still offers many recreational opportunities. Belle Isle is connected to the city of Detroit by the MacArthur Bridge. It is home to a number of leisure and entertainment amenities. They include the Detroit Botanical Gardens and the Anna Scripps Whitcomb Conservatory, which houses one of the country's most extensive collections of orchids. In addition, it is home to the a yacht club, a boat club, a municipal golf course, a Coast Guard post, and the Dossin Great Lakes Museum. There is a half-mile long swimming beach, and a Nature Center, as well as several miles of hiking trails, including wheelchair accessible hiking trails. Visitors to the island can also take advantage of picnic tables, a playground, baseball fields, basketball courts and tennis courts. Finally, the island is the site of a street race circuit that hosts races in the IndyCar Series and the ALMS. The James Scott Fountain located on Belle Isle was designed by New York architect Cass Gilbert. The fountain was built in 1925 with a special bequest from infamous gambler, James Scott, who left $500,000 to the city with instructions to use the money to build a fountain on Belle Isle. There was only one condition - that a life-size statue of the donor be built beside the new fountain. The statue of James Scott sits in a chair facing beside the fountain, looking out toward the city. Belle Isle Park is open daily from 10:00 AM to 5:00 PM. Admission is free.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 32\n",
      "Document 8898: John Ball Zoological Garden in Grand Rapids, Michigan Michigan\n",
      "Rate: 4.0 Review Number: 765.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g42256-d656374-Reviews-John_Ball_Zoo-Grand_Rapids_Kent_County_Michigan.html\n",
      "The John Ball Zoological Garden, located at 1300 W. Fulton St. in Grand Rapids, Michigan, features more than 1,100 animals from Africa, Asia, South America, Australia and the United States. The zoo features many exciting and educational exhibits, including the Peter C. and Pat Cook Mokomboso Valley Chimpanzee Exhibit; the South American Boardwalk; the North American Trek; the Treasurer of the Tropics building; the Van Andel Living Shores Aquarium, Sting Ray Lagoon and the Budgie Aviary. About one-third of an acre is devoted to the zoo's very own family of chimpanzees. The Mokomboso Valley Chimpanzee Exhibit, one of the largest chimp exhibits in the nation, is a natural environment that is home to these inquisitive, resourceful endangered species. The chimps enjoy an indoor enclosure on rainy, cold or hot days. Educational literature about these vibrant animals is available at the display. The Mayan, Inca and Aztec civilizations are highlighted in the South American Boardwalk exhibit. Guests can talk to researchers at Camp Rio and learn about capybaras, the largest rodents on the globe. Wolves, guanacos and screamers also make the boardwalk a spectacular place to visit and learn. Meanwhile, the North American Trek features grizzly bears, wolverines, river otters and a variety of other animals indigenous to the U.S. Showcasing night animals, as well as a variety of reptiles, amphibians and small primates, Treasures of the Tropics is home to the only Komodo dragon in Michigan. It also houses a black-footed cat, dart frogs, caimans, fruit bats and insects. The Van Andel Living Shores Aquarium offers insight into the world of aquatic creatures living inside a Michigan stream, the Patagonian coast and the northwestern coast of the United States. Penguins, and a variety of fish and invertebrates, make this exhibit enchanting for visitors. Guests can interact with wildlife by touching a sea urchin or a sea star. A 5,000-gallon outdoor pool that allows visitors to see and touch sting rays and sharks as they swim by, Sting Ray Lagoon is the only exhibit of its kind in Michigan. The Far Side of the World Trail complements the exhibit, with the opportunity for visitors to venture out on the open seas for discovery and fun. Opened in April 2008, Frogs! opens up a world of about 80 animals from 15 frog and toad species. The bright colored display with interactive graphics allows children and adults to explore these vibrant creatures together. The goal of the exhibit is to foster a strong respect for the animals, which are in danger of disappearing from the planet forever. The Budgie Aviary, an interactive exhibit situated near the Children's Zoo waterfall, is filled with vividly colored budgies, birds that are indigenous to Australia. Feed sticks can be purchased to allow the birds to eat out of visitors' hands. It is open during the summer months. Opened in June 2008, Bissell's Lions of Lake Manyara is one of the largest lion exhibits in the United States. Lake Manyara in Tanzania, named a biosphere by the United Nations due to its diversity of people and animals, is among a small number of places in the world known to have tree-climbing lions. The exhibit has a climate-controlled climbing tree and special points for panoramic viewing, including replica Tanzanian building. A four-story, 300-foot zipline allows visitors to ride, while securely belted, over the zoo. Camel rides, swan paddle boats, turtle exhibits and a sky trail rope course, featuring two climbing levels for those 7 years old and older, make visits the zoo memorable and engaging for guests of all ages. The Wild Gift Shop provides an array of souvenirs from which to choose, including stuffed animals, books and collectible items. Gifts of the Wild, another gift store, offers a broader variety of items for children and adults, including T-shirts. The grounds also are the perfect location for birthday parties and special events. The zoo has seasonal hours and is open every day, except Christmas Day. Single and double stroller rentals, as well as free wheelchair rentals, are available on a first-come, first-served basis. Group discounts are provided only for summer season, and must be ordered in advance.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 33\n",
      "Document 8922: Michigan State Capitol, Lansing, Michigan, a National Historic Landmark and the Seat of Michigan's Government Michigan\n",
      "Rate: 4.5 Review Number: 345.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g42391-d282093-Reviews-Michigan_State_Capitol-Lansing_Ingham_County_Michigan.html\n",
      "The Michigan State Capitol building is a National Historic Landmark, and has served as the seat of Michigan's state government since the building was completed in 1878 at a cost of $1.2 million. The building was designed by Elijah E. Myers, and its successful completion established his reputation as the leading capitol-builder in the post-Civil War era and one of the country's most renowned architects of public facilities. Myers was one of the first architects to model his designs on the national capitol in Washington, D.C. His Neoclassical Renaissance'' - or Italianate'' - design for the Michigan Capitol set the standard for state capitols for decades. The name of his design is Tuebor, which translates to I will defend.'' The design also provided the inspiration for a national trend after the Civil War to fireproof buildings. The Capitol was contructed to not only house a state government that was growing in both complexity and size, but also to serve as a memento of the war itself. Once the building was dedicated and occupied, the interior embellishment continued over the course of several years. This embellisment included the extensive use of decorative and architectural painting, which was so elaborate and skillful that it still ranks as one of the finest examples of the Victorian decorative arts in the United States. From 1987 to 1992, the building underwent an extensive historical renovation. This included restoring the interior, along with its 139 rooms, and the exterior to their original appearance, as well as upgrading and adapting the building to serve as Michigan's permanent state capitol. The building is located at the intersection of Capitol and Michigan Avenues in Michigan's state capital, Lansing. At one time, the building was large enough to house the state's government agencies and departments. Now, the building houses only the legislative and executive branches of Michigan's state government, as well as the ceremonial offices of the state's governor and lieutenant governor. On the ground floor can be found several offices, such as the Secretary of the Senate and the Clerk of the House. The present building is the third to house the Michigan state government. The first state capitol was located in Detroit, which was the original capital of Michigan. The capital was relocated westward to Lansing in 1847 to be more defensible from Brittish troops stationed in Windsor, Ontario, and to develop the western portions of the state. The Capitol building stands 267 feet from the ground to the tip of the spire atop its dome, and is 420 feet, two inches long and 273 feet and 11 inches wide, including its approaches. The Capitol occupies 1.167 acres and has a perimeter of 1,520 feet. There are four stories, two grand staircases in the north and south corridors that go to the top floor. Two of the most distinctive characteristics of the Capitol are its dome (inside and out), and the statue of Civil War Governor Austin Blair that greets visitors at the front of the building. Blair was a Michigan politican who strongly opposed slavery and secession, and a champion of human rights, the effort to ban capital punishment, and the efforts to give women and Black citizens the right to vote. The exterior dome originally matched the light tan color of the building, but was repainted a bright white during the building's renovation. On the inside, the cast-iron dome presides over the eight muses on the ceiling display. On the north wall of the first floor resides a large clock, known as a long-drop clock.'' The clock is at least as old as the building itself, and was restored to working condition in 1990. Another of the building's noteworthy features is its cornerstone, which has the date 1872 on one side and 1878 on the other, marking the start of construction and the building's completion. On the grounds nearby can be found the state tree of Michigan, an Eastern White Pine, a blue spruce called The Freedom Tree'' in memorial to the Vietnam War's MIAs and POWs, and the Dr. Martin Luther King, Jr. tree planted in memory of Martin Luther King, Jr.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 34\n",
      "Document 8970: World's Largest Stove Michigan\n",
      "Rate: 4.5 Review Number: 61.0\n",
      "https://www.tripadvisor.com/Hotel_Review-g42480-d3360645-Reviews-or25-White_River_RV_Park_Campground-Montague_Muskegon_County_Michigan.html\n",
      "Standing 25 feet high and weighing around 15 tons, the world's largest stove is located in Detroit Michigan. This stove measures approximately 30 feet long and 20 feet wide. It was originally created for the Michigan Stove Co.'s 1893 Columbian Exposition, and it was restored and moved multiple times, most recently for the Michigan State Fair in 1998. This stove is designed as a large-scale replica for an old coal or wood burning stove, modeled after the company's Garland kitchen range model. The stove was designed by William J. Keep and is actually carved from oak, but painted with black and silver paint to look like metal. The sculptor is unknown, but most speculate that it was created by either John Tabaczuk or Joachim Jungwirth. It is located within the Michigan State Fairgrounds, and is most popularly viewed during fairground events, so viewing hours and admission costs depend on the event being held. Visitors can also call the fairgrounds in advance to access the area in the off-season. Originally, it stood on a 20-foot-tall platform above a large display of stoves, and today, it stands on a smaller platform and has a plaque talking about the Michigan Stove Company and the history of this large tourist attraction.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 35\n",
      "Document 8938: Red Hawk Golf Course Michigan\n",
      "Rate: 2.0 Review Number: 54.0\n",
      "https://www.tripadvisor.com/Hotel_Review-g42559-d1021520-Reviews-Oscoda_Lakeside_Hotel-Oscoda_Michigan.html\n",
      "Red Hawk Golf Course is an Arthur Hills design set amid 205 acres the Huron National Forest in Northern Michigan. The course, which opened in 1999, is situated in a valley and features many of the hallmarks of Northern Michigan golf, such as natural ridges, ravines, valleys, wetlands and an abundance of trees. Towering pine, oak, maple, birch and beech trees line the fairways and provide both a challenge and a dramatic backdrop. One course feature that tends to surprise golfers is the elevation change found on a course on the east side of the state, an area not typically known for it. Hole No. 3, for example, features an elevation drop of 63 feet from tee to green, and the green is perched above Fountain Creek farther below. The 18-hole, par-71 public course plays to 6,589 yards from the back tees, with a USGA rating of 71.6 and a slope rating of 139. It's a relatively short course, with the White tees playing to only 5,676 yards. The challenge comes from the rolling landscape and undulating greens that feature subtle, yet deceptive, breaks. The landing areas are not as tight as they might appear off the tee, and bunkering on the course is minimal, but has been described as magnificent. When taken all together, the course is a classic design that is said to mirror the great courses designed in the 1920s and 30s. Some of the more noteworthy holes include No. 2, a short par-4 that plays to only 292 yards from the back tees, and a mere 262 from the Whites. Golfers thinking of driving the green may think twice, however, as the hole features seven bunkers, the most of any on the course. The fairway and green are both elevated, guarded on the left by dense trees and the right by deep bunkers. Hole No. 11 is a scenic and demanding par-5 that plays to 509 yards from the back tees and 486 from the Gold. Tee shots should easily carry the large pond just in front and find the large uphill landing area, but it's what comes next that makes this hole tricky. The hole doglegs left from there, and the green is two-tiered and severely slopes from the front to the back. The course has accumulated a number of awards since its opening, including being ranked #40 by Golf Digest in 2007 among the Top Public Golf Courses in America. And Great Lakes Golf Magazine rated Red Hawk the #3 Public Golf Course in Michigan in 2003. Practice facilities at Red Hawk are some of the finest in Northeastern Michigan, and include a driving range with two tiers. The same Bent grass as found on the course is used on the teeing area and the practice fairway. There are also four target greens complete with bunkers. There's also a short-game practice area, with Bent grass green and practice sand bunker, along with another 7,000-square-foot putting green. Lessons are also available. Red Hawk also features a clubhouse with a fully stocked pro shop, as well as the Red Hawk Grill for dining.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 36\n",
      "Document 7943: Homewood Suites by Hilton Chicago Downtown Illinois\n",
      "Rate: 4.0 Review Number: 1035.0\n",
      "https://www.tripadvisor.com/Hotel_Review-g35805-d6985880-Reviews-Homewood_Suites_by_Hilton_Chicago_Downtown_Magnificent_Mile-Chicago_Illinois.html\n",
      "Homewood Suites by Hilton Chicago Downtown is the epitome of Hilton Hotels with luxury, sophistication, and plenty of complimentary amenities. Their two room King suite with separate living and sleeping areas, combined with a full kitchen, was just renovated. They service complimentary hot breakfast daily, complimentary reception Monday through Thursday, free high speed internet, free business services, free fitness center, complimentary shopping service, and complimentary hot drinks in the lobby throughout the day.  The suites have full kitchens, fridges, microwave, stoves, dishwashers, LCD TV, voicemail, and the Suite Dreams bedding. Additionally, the rooms offer telephone, toiletries, hair dryer, iron, ironing board, and available games for rent. Hotel amenities include 3500 square feet meeting space, which is flexible in size to be smaller should one require it. They have an indoor heated pool, fitness center, guest laundry, and catering facilities. The location of the hotel places it near many of Chicago's nightlife and attractions. It is possible to walk to Nordstroms and use North Bridge Mall to arrive on Michigan Avenue. Michigan Avenue is the site of the Chicago aquarium, restaurants, museums, and corporations. It is also possible to rent movies at the hotel for in room fun.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 37\n",
      "Document 8925: Model T Automotive Heritage Complex Michigan\n",
      "Rate: 5.0 Review Number: 662.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g42304-d256939-Reviews-Gilmore_Car_Museum-Hickory_Corners_Barry_County_Michigan.html\n",
      "Detroit has long been known as a major center for car manufacturing and one of the best places in the city to get an idea of the important role the city played is to visit the Model T Automotive Heritage Complex, or T-Plex. The building was threatened with demolition when local zoning laws changed, until in 1997 preservationist Jerald Mitchell stepped in, thus ensuring its survival. Henry Ford was born just outside Detroit in 1863 and after becoming skilled at the repair of steam engines, developed a self-propelled vehicle in 1896. Ford formed his own company in 1901 and began to not only build cars, but also revolutionize the industrial world by paying his workers a fair wage. Ford's company purchased the building in 1904 and the famous Model T was introduced in 1908. Over the course of almost 20 years, over 15 million Model T Fords would be sold. In 2006, the plant was designated as a National Historic Landmark; the building is also a Michigan Historic site. Work continues constantly in an effort to restore and renovate as much of the building as possible - much of it done by volunteers. Almost a dozen committees are involved not only with restoration, but everything from loaning out cars to publishing a regular newsletter. The distinctive red brick building still retains many of its original fittings and details from the period, including windows and fire escapes. At the time of its construction, the building was an excellent example of efficient industrial construction. Ford introduced various measures aimed at preventing fire damage, including three firewalls and a 25,000-gallon water tank that fed a sprinkler system. But it is what is inside the building that attracts history buffs and car enthusiasts - in addition to a collection of Model T Ford cars, visitors also see the once secret and experimental room where the Model T was developed. Several other Ford cars were constructed here as well, including the underrated Ford Model N and the model K, considered a luxury car at the time. In addition to several intact cars, visitors to T-Plex also experience a unique look at how cars were made over a century ago. A major exhibit shows the famous Model T car during different stages of its assembly and explains how cars were manufactured before the invention of the moving assembly line. Other exhibits in the building tell the history of both the building and the autoworkers. The T-Plex complex is located in an area of Detroit that is also home to several other reminders of the city's importance in the car industry. During its heyday, the Piquette Avenue area was the home of the US auto industry and in addition to the T-Plex building there are several other fine examples of industrial buildings that were once the home of such well known names as Cadillac, Dodge and Studebaker. There are also several sites relating to Henry Ford in and around Detroit, including the Henry Ford Museum in the nearby suburb of Dearborn.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 38\n",
      "Document 13325: Hotel Convenient to Shopping and Dining Wisconsin\n",
      "Rate: 4.5 Review Number: 371.0\n",
      "https://www.tripadvisor.com/Hotel_Review-g60072-d72370-Reviews-Best_Western_Park_Oasis_Inn-Mauston_Wisconsin.html\n",
      "The Residence Inn in Milwaukee is located in the city's Marshall Fields building, former home to the corporate offices of the Marshal Fields department store. This old world style makes for a building filled with character and charm. The building was recently renovated and is no an all-suite hotel. Just a short distance away is the Midwest Airlines Convention center, the Bradley Center, and Milwaukee's theatre district. The hotel is connected to the Shops of Grand Avenue, the city's downtown shopping mall. The hotel features flat-screen televisions in each room wireless Internet, a business center, and 24-hour fitness center. Each morning guests are invited to enjoy the hotels breakfast buffet, as well as social hour on Monday, Tuesdays and Wednesdays. The hotel is located right on Milwaukee's famous RiverWalk, providing an area for relaxing and strolling before or after an event. The hotel's full business center is great for those visiting the city on business. There are five stories and more than 130 suites. Three meeting rooms provide 1,900 square feet of meeting space. The hotel embraces its appeal to business stays and offers copy and fax service, network printing, and overnight delivery and pick up to its guests. There is a cash machine on the premises, and both grocery shopping and housekeeping services are available on the daily basis. There is on-site laundry for those who are enjoying a longer stay, and valet dry cleaning for those who need extensive cleaning service. Newspapers are available in the hotel lobby, and local restaurants are happy to deliver meals to the hotel. Nearby are a variety of Milwaukee's best activities. Bradford Beach on the shores of Lake Michigan is about two miles east of the hotel. There are biking and hiking trails nearby, as well as bowling alleys, sailing, tennis and volleyball courts, and waterskiing. The prime location of the city near Lake Michigan means there are always events and activities going on. People visting Milwaukee for business or pleasure will never be bored. The hotel's fitness center features two stairstepper machines, three treadmills, and two stationary bikes. While there may be more extensive fitness centers at other hotels and resorts, a lot of business travelers will find this is just enough to keep their fitness routine in action while traveling. Nearby dining options include the Rock Bottom Restaurant and Brewery, serving pub fare and local microbrews. Mo's IrishPub is open for lunch and dinner and has a menu filled with traditional Irish favorites, as well as standard American entrees. Karl Ratzsch's is one of Milwaukee's most famous German restaurants and is within walking distance of the Residence Inn. There is also an Applebees that can be reached via the Grand Avenue Shops connection, so you never have to go outside. This is quite the bonus during Milwaukee's harsh winters. Whether you are in town for business or pleasure, the Residence Inn by Marriott can provide comfortable accomodations with a number of perks. Consider staying with them the next time you visit Cream City.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 39\n",
      "Document 7957: Hyde Park Neighborhood in Chicago, Illinois Illinois\n",
      "Rate: 4.5 Review Number: 72.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g35805-d169630-Reviews-Hyde_Park-Chicago_Illinois.html\n",
      "The Hyde Park neighborhood is located on the South Side of Chicago, Illinois. The neighborhood is the home of the University of Chicago, Hyde Park Art Center, the Museum of Science and Industry, the Renaissance Society and the Oriental Institute. Hyde Park is the former name of a Township that included many other neighborhoods that had been annexed by the City of Chicago. Founded by Paul Cornell in the 1850's, Hyde Park is near the Illinois Central Railroad, located south of Chicago. The Hyde Park Township was formed in 1861, and extended from 39th to 63rd Streets. The south border of Hyde Park later extended as far as South 138th Street and west to State Street. The township was annexed to the City of Chicago in 1889, and up until that time had been independent of Chicago. During the 1920's, the University of Chicago lured the Chicago Theological Seminary to relocate to Hyde Park. Since that time, four other theological schools and the Divinity School of the seminary have joined the Hyde Park location, making it home to nearly 50% of Chicago's theological schools. In the 1930's, Hyde Park was a prosperous hotel and resort destination with more than 100 hotels, and a dozen elaborate structures located along the lakefront. During the 1940's, following the Great Depression and during the war, some of the hotels began catering to the transient population. Many of these hotels were later converted into apartment and condominium buildings. Because of a thriving artist community, the 57th Street Art Fair began in 1948, still continuing to the present day. Hyde Park has more than 30 playgrounds located in the neighborhood; all of them are located in within residential areas and are maintained by the Chicago Park District. The parks and playgrounds are designed for children of all ages when the park does not have a playground area, it does still have park benches and open fields. The even numbered streets in the Hyde Park neighborhood are almost all residential areas, while the odd numbers and particularly 51st, 53rd, 55th and 57th streets contain a large number of businesses. 53rd Street is Hyde Park's oldest shopping area and is lined with many wallet-friendly restaurants featuring take-out, and smaller businesses located between Woodlawn and Lake Park. A Farmer's Market, open air outdoor market is held on Harper Court during the summer months. Hyde Park is home to a number of places listed on the National Registry of Historical Places such as the Chicago Beach Hotel, the Arthur M. Compton House, the Isadore H. Heller House, the Shoreland Hotel, Charles Hitchcock Hall and many others. With the dining options, churches, diverse cultural history and architectural structures, Hyde Park is one of the most interesting neighborhoods in Chicago, Illinois. Whether shopping, dining, exploring or looking for an upbeat nightlife, the Hyde Park Neighborhood has something to suit the tastes and interests of everyone.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 40\n",
      "Document 8052: The Shops at North Bridge Illinois\n",
      "Rate: 4.0 Review Number: 55.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g35805-d7094424-Reviews-The_Shops_at_North_Bridge-Chicago_Illinois.html\n",
      "Distinctive shopping is available at The Shops at North Bridge in Chicago, Illinois. They are located at 520 N Michigan Ave and Grand and the stores are open Monday through Saturday from 10:00 am to 8:00 pm and Sunday from 11:00 am to 6:00 pm. The hours can change during the holiday so visitors should check with their favorite stores. The Shops at North Bridge encompass nine city blocks and hold fifty specialty stores, twenty restaurants, five hotels and Nordstrom as an anchor store. Nordstrom Michigan Avenue has a complimentary coat check, wheel chairs, strollers and a delivery service for guests' purchases. There is a concierge service located on Level One of The Shops at North Bridge. They can help guests with dinner reservations, gift suggestions, itinerary planning, hotel referrals, package carry out, parking validations, tourist information, transportation assistance, and complimentary wheel chairs. Guests will find that staff is multilingual and they can also help with guide books to sporting events, activities in the city and museums. Parking validation is available with a minimum ten dollar purchase at any of The Shops at North Bridge or Nordstrom. The validation is accepted at the Ohio Grand Wabash parking garage and at 516 N. Rush. A parking ticket will also be validated if the guest has spent three hours or less at The Shops at North Bridge. Children are also taken care of at The Shops at North Bridge. There are family restrooms and lounge areas as well as a nursing room. Children's menus are offered in the restaurants and guests can get complimentary baby wipes and baby equipment. On site activities include a trip to The Lego Store where they have an in store play area. Other complimentary activities include an appearance by Sanrio's Hello Kitty every other Saturday from 2:00 pm to 4:00 pm and Chicago Marriott Downtown Magnificent Mile which offers a game room and coloring books at the front desk. Birthday parties can be held at the Lego Store and Sanrio's Hello Kitty Store. Some of the stores that are located at The Shops at North Bridge include: A/X Armani Exchange, BOSS Hugo Boss, Chicago Harley Davidson, Kenneth Cole, Sephora, Tommy Bahama, Vosges Haut-Chocolat, Local Charm Jewelry Store or the Spa at Nordstrom's. Some of the food stores at The Shops at North Bridge are the ESPN Zone, Potbelly Sandwich Works, Tuscany Cafy, Joe's Seafood, and Prime Steak & Stone Crab. As if the above stores aren't enough, visitors to Chicago can find Adele Dallas Orr, Bose, Brookstone, Buca di Beppo, Big Bowl, The Body Shop, C.D. Peacock, Candyality, Cesar Russ Photographic Stories, Chicago Sports, Conrad Chicago, Cosi, Crocs, Erwin Pearl, Fans Edge, Gold Leaf Tobacco & Accessories, Handbag 101, Hilligoss Galleries of Fine Art, Hilton Garden Inn Chicago Downtown, Homewood Suites by Hilton, Jolie Fleur Flower Boutique, LOFT, Louis Vuitton, Lucky Brand Jeans, Marbles: The Brain Store, Max Orient, NYS Collections, Optica, Papyrus, Relish, Room and Board, Rosetta Stone, Royal Beauty, SABON, Seacret Spa, SoHo, Spirit of the City and many other specialty stores.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 41\n",
      "Document 13389: Relaxing Site for Outdoor Recreation Wisconsin\n",
      "Rate: 3.0 Review Number: 28.0\n",
      "https://www.tripadvisor.com/Hotel_Review-g30316-d2046068-Reviews-Harbor_Inn_Motel-Algoma_Wisconsin.html\n",
      "The largest of the four Madison, Wisconsin lakes is Lake Mendota. The lake borders Madison, Middleton, Maple Bluff, Westport, and Shorewood Hills. The University of Wisconsin is situated along the southside of the Lake. The lake provides students with the ability to conduct research and participate in lake sports that other colleges may not have access to. Students also congregate along the lakefront for recreation and socializing. A good portion of the lake is surrounded by expensive homes and condominium communities. There are also natural areas and parks that are government owned, including the James Madison Park. Student housing and the Student Union also occupies space along the lakefront. Retail establishments, including hotels and restaurants have taken advantage of the lakefront views and built successful business by Lake Mendota. Seasonal sports are popular on the lake, and the warm temperatures bring boaters out onto the lake. There are numerous launch sites, as well as two marinas. Other lake activities include water-skiing, fishing, wakeboarding, canoeing, tubing, wind-surfing, kayaking, and sailing. Winter sports are also enjoyed on the lake and include ice fishing, ice boating, cross country skiing, and ice hockey. Because of its proximity to the college, Lake Mendota is considered one of the countries most studied lakes. The remote sensor buoy on the lake is part of the Global Lake Ecological Observatory Network. To some, Lake Mendota is best known for a 1979 stunt carried out by a party of the University's student government. After being elected into power, this tongue in cheek political faction honored'' a campaign promise by bringing the Statue of Liberty to Madison. The built a faux statue that appeared to have sunken into the lake. This served to infuriate several student groups, but it did not stop the Pail and Shovel party from winning a second election, prompting them to build and sink an even larger Lady Liberty statue. While many complained about the cost to students for this prank, it goes to show how willing people can be to fall for a variety of political stunts. Often the most absurd tactics display the irony of the way in which people think and vote. Since the pranks, Lake Mendota has been relatively peaceful. It is a weekend destination for people all over the state of Wisconsin. Bicycle paths lead riders along the shores, and hiking trails allow visitors to view the natural habitat of the lake and its surroundings up close. The lake is 42 feet deep and over 9,000 square acres large. Numerous fish specied occupy the lake including bluegill, lake sturgeon, walleye, bass, muskie, and pike. Fishing restrictions include size minimums on walleye, bass, and pike. This ensure the ecosystem of the lake will continue to flourish and offer exciting fishing opportunities to all who visit the lake. Just minutes from the Wisconsin state capital, Lake Mendota provides a respite from the urban culture of Madison. Enjoying a day at Lake Mendota is a popular pasttime for those living in Wisconsin.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 42\n",
      "Document 7979: Millennium Knickerbocker Hotel Chicago Illinois\n",
      "Rate: 4.0 Review Number: 1649.0\n",
      "https://www.tripadvisor.com/Hotel_Review-g35805-d87648-Reviews-Millennium_Knickerbocker_Hotel_Chicago-Chicago_Illinois.html\n",
      "The Millennium Knickerbocker Hotel in Chicago, Illinois offers guests comfortable accommodations near any of the city's best attractions. Each of the 306 guestrooms feature high-definition flat screen televisions, fully stocked mini bars, 400 thread count linens, spa-like bathrooms, granite countertops, rain showers, environmentally conscious water systems, ironing equipment, down comforters and bathrobes and slippers. The hotel is located right on Michigan Avenue, often called Chicago's Magnificent Miles because it is loaded with shops, department stores, cafes and restaurants. Guests have access to the M Bar inside of the hotel, offering more than 50 martini varieties and wines from which to choose. Dining is available onsite at the Nix restaurant. The hotel is a little over a half hour from the airport. Guests enjoy access to 18,000 square feet of meeting and banquet space. The space can be used for parties, meetings, receptions or seminars. The Crystal Ballroom is one of the most famous meeting areas in Chicago and offers a balcony, crystal chandeliers and a domed ceiling. Guests of the hotel have access to a variety of hotel amenities including valet parking, concierge service, a notary public, an ATM machine and a fitness center. The hotel is just steps from Navy Pier, Lake Michigan, the John Hancock Center, the Shedd Aquarium, the Gold Coast Neighborhood, the Museum of Contemporary Art and the Field Musem. \n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 43\n",
      "Document 13273: Convenient Hotel near Lake Michigan Wisconsin\n",
      "Rate: 4.0 Review Number: 31.0\n",
      "https://www.tripadvisor.com/Hotel_Review-g60200-d101054-Reviews-The_Harborview-Port_Washington_Wisconsin.html\n",
      "This Hyatt Regency in Milwaukee is recently renovated and nearby some of Milwaukee's best attractions. It is connected to the Midwest Airlines Center, so those attending or organizing a convention will not have to deal with inclement weather, finding additional parking, or meandering around town searching for their destination. It is also connected to the Grand Avenue Mall, so if you are in town on business or pleasure, you can reach shopping and entertainment without ever stepping outside. Two blocks away is Milwaukee's Bradley Center, the Performing Arts Center, the Entertainment District, and Milwaukee's River Walk. The hotel's convenient location makes it a popular choice among travelers. The amenities of the hotel include wirless high speed Internet, video checkout, messaging systems, concierge services, a full service business center, and a variety of dining and entertainment options. There is also a complimentary fitness center on-site, as well as nearby golf and tennis courts. Just a short drive away is the Harley-Davidson museum and headquarters, Lake Michigan, Milwaukee Breweries, and Miller Park. Bistro 333 and Bar 333 are located within the hotel and offer guests dining and social options that will make their stay more enjoyable and relaxing. Polaris is Milwaukee's only revolving rootop restaurant and lounge and provides a variety of American fare. The Pilsner Palace embraces Milwaukee's brewing tradition and serves tasty American dishes. The Knuckles Sports Bar is a great place to meet friends and watch the big game. The hotel honors the Goldl Passport program, which means you will receive reward points for your stay. You can also earn points by booking events at the hotel. The points are redeemable for travel rewards and meeting credits, which means with each stay, future stays will cost you less. Hyatt Regency vacation packages come in four varities. The Explore package offers guests a chance to stay in luxury accomodations and enjoy area attractions at a discounted rate. The Amour package is all about romance. The Balance package is great for getting away and restoring health, and the Awake package is great for those who want a night away followed by a hearty, delicious breakfast. The Hyatt Regency Milwaukee is a place to plan your next event. They offer personalized service and luxury amenities for your guests. You can book your next meeting, large or small, or your wedding at the Hyatt. Whether it is for business or pleasure, the Hyatt's experienced team of event planner will provide the services you need to ensure your event is a success. Accomodations include 483 guestrooms that include 19 suites, and a combination of king parlors and queen double rooms. Each room has cable TV, voice mail, electronic security locks, bathroom amenities, AM/FM alarm clocks, hair dryers, and in-room Yoga serice. The hotel provides covered parking for its guests, a great benefit to a hotel in downtown Milwaukee. If you are looking for a classy hotel near Milwaukee's best attractions, consider staying in the heart of downtown at the Hyatt Regency Milwaukee.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 44\n",
      "Document 8049: The Raffaello Hotel Illinois\n",
      "Rate: 4.0 Review Number: 2461.0\n",
      "https://www.tripadvisor.com/Hotel_Review-g35805-d87635-Reviews-Gale_Chicago-Chicago_Illinois.html\n",
      "The Raffaello Hotel is located at 201 East Delaware Place in Chicago, Illinois. The facility which can be found in downtown Chicago has a no smoking policy. The hotel is minutes away from the Magnificent Mile, theaters, shopping and an observatory. These are just a few of the nearby attractions that when combined with the amenities of the hotel make it an ideal place to stay in the Chicago area.  Amenities at the Raffaello include personalized service combined with modern technology to make the stay more enjoyable. The facility offers room service, Windy City Massage pedicure and manicure and massage treatments. Those on business trips can take advantage of the business center which is open twenty-four hours a day. The hotel has a fitness center open twenty-four hours a day as well. Valet parking, DVD players, room safes and bath robes are all provided at the facility. The Raffaello Hotel also provides a convention center and meeting rooms to suit events of various sizes. There are even electric machines for shoe shining in the facility. Those wishing to obtain additional information about the rates, accommodations, amenities and services at the Raffaello Hotel in Chicago, Illinois should contact the facility directly.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 45\n",
      "Document 7938: Hilton Chicago Illinois\n",
      "Rate: 4.0 Review Number: 6681.0\n",
      "https://www.tripadvisor.com/Hotel_Review-g35805-d87590-Reviews-Hilton_Chicago-Chicago_Illinois.html\n",
      "The Hilton Chicago is located near Michigan Avenue Mile close to Grant Park, Museum Campus, and Lake Michigan. The Hilton is considered a city landmark. It is also the closest hotel to Chicago's event center and McCormick Place Convention Center. From the hotel visitors can enjoy many of Chicago's more notable locations like Shedd Aquarium, Field Museum, Soldier Field, and Adler Planetarium.  Hilton Chicago has 1544 rooms which combine historic décor with modern facilities. They offer high speed internet access, flat screen television, telephone, bath amenities, and writing desk in the room. The suites also have a large living area and wet bar to add to the comforts one can experience. Each room is designed with cool colors,and wood furniture to accentuate the beauty and comfort offered. Amenities at the hotel include audio/ visual equipment, business center, business phone service, cellular phone rental, fax, dictation machines, printer, meeting rooms, ATM, barber shop, beauty salon, concierge, and multilingual staff. The Hilton Chicago has three places to dine including Kitty O'Sheas. They serve lunch and dinner with a casual atmosphere. Their cuisine is typical American with fish and chips as their main attraction. Lakeside Green Lounge is open for cocktails and dinner. The Pavilion is their fine dining establishment.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 46\n",
      "Document 7895: Chicago Place Illinois\n",
      "Rate: 4.0 Review Number: 655.0\n",
      "https://www.tripadvisor.com/Hotel_Review-g36141-d90943-Reviews-Hyatt_Place_Chicago_Hoffman_Estates-Hoffman_Estates_Illinois.html\n",
      "Chicago Place is a 608 foot building containing offices, shops and restaurants and is located at North Michigan Avenue in Chicago, Illinois. Opening hours are 10am to 7pm Monday to Friday, 10am to 6pm on Saturdays and 12pm to 5pm on Sundays. Chicago Place is managed by Grubb & Ellis Property Solutions Worldwide. The 49 story tower was opened in 1991 with Saks Fifth Avenue as its anchor store, but number of stores has reduced since then with much of the area converted to office space. Above the complex is a further 50 floors containing luxury condominiums. The building was erected during a period of great building investment between 1988 and 2001 with other arrivals being the 900 N. Michigan Avenue building, Crate & Barrel, 676 N. Michigan Avenue and the 600 N. Michigan Avenue building The mall has numerous stores and restaurants located on several levels with the anchor store being Saks Fifth Avenue, which is on level one. The other stores featured on level one are Talbots and Talbots Woman. There are no shops on level two but on level three shoppers will find art and gifts at Russian Creations, whilst fashion items for taller women can be found at Tall Girl. The next floor up has just one store, which is Abby & Yanni Fine Furs & Outwear where visitors will find furs and outwear. The next set of stores are located on level eight, which is also home to the food court. The three shops on this level are a gift vendor called Love From Chicago, Luxury Nails a nail salon for men and women and Time-Us, which sells clocks and watches and does repairs too. There are a few places serving food for visitors at the eight floor food court. Hot snacks and sandwiches can be purchased from Subway, whilst Mexican food can be obtained from Taco Bell Express. Other eateries at the food court include Chicago Dog & Grill, Little Tokyo Express, Panda Express and Great Steak & Potato. North Michigan Avenue is part of what is known as the Magnificent Mile and features many things for visitors to do with many restaurants serving American, Italian, Indian, Brazilian and other national delicacies. A very popular attraction is Chicago's First Lady Architectural Tours, which provides a view of the city's building via a river cruise. There are many museums in the area too including Chicago Children's Museum, Museum of Contemporary Art, McCormick Freedom Museum and Loyola University Museum of Art with the latter two both in North Michigan Avenue. Visitors seeking a little pampering can find Elizabeth Arden Red Door Salon & Spa, Renew Spa, Tiffani Kim Institute and Spa NORDSTROM a short distance from Chicago Place. There are restrooms available at Chicago Place and elsewhere in North Michigan Avenue including at The 900 Shops, Water Tower Place and The Shops at North Bridge. There are numerous places to park in the area including at G.O Parking Inc and Standard Parking.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 47\n",
      "Document 7921: Fairfield Inn & Suites Chicago Downtown Illinois\n",
      "Rate: 4.5 Review Number: 2075.0\n",
      "https://www.tripadvisor.com/Hotel_Review-g35805-d224036-Reviews-Fairfield_Inn_Suites_by_Marriott_Chicago_Downtown_Magnificent_Mile-Chicago_Illinois.html\n",
      "Just one block from Chicago's famous Magnificent Mile on Chicago Avenue, the Fairfield Inn & Suites Chicago Downtown is newly renovated and is considered Fairfield's flagship hotel in Chicago. Its convenient location puts guests close to businesses, shopping, tourist attractions, and entertainment. With a modern lobby and various amenities, the hotel seeks to welcome every guest and meet their travel needs.   The 185 guest rooms (including 26 suites) located on 15 floors feature upgraded contemporary furnishings with the intent to allow its guests to relax and spread out. Each room includes complimentary high speed wireless internet access, 37-inch flat screen LCD televisions, high definition cable channels, and a large work area with an ergonomic chair. The suites come with microwaves, refrigerators, and two televisions. In addition, each guest receives a complimentary continental breakfast buffet on the second floor. The Fairfield Inn & Suites Chicago Downtown offers numerous packages for various types of travelers. For example, students can receive a special rate or tourists can book a package including tickets to local attractions. The Fairfield Inn & Suites Chicago Downtown also offers shopping packages and parking packages, Business travelers can reserve the hotel's meeting room, which accommodates up to 12 people and has a selection of audio/visual equipment for use.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 48\n",
      "Document 7917: Eclectic Architecture and Abundant Shopping Make the Magnificent Mile in Chicago a Must See Illinois\n",
      "Rate: 4.5 Review Number: 16.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g35805-d570278-Reviews-Vosges_Haut_Chocolat-Chicago_Illinois.html\n",
      "The Magnificent Mile in Chicago has become one of the biggest shopping areas in the United States with all of the big brand-name stores; it is flocked to by 22 million visitors annually. The Mile'' runs from the northern strip of Michigan Avenue below Lake Shore Drive to the Chicago River to the south. The name of the Magnificent Mile is said to have been coined by Arthur Rubloff, real estate developer, when launching his plan for construction and revitalization of the area after World War II. However, many of the architectural sites date from the 1920's.  Between 1920 and 1929, a series of buildings were completed as part of the execution of the Paris on the Prairie plan, designed by Daniel Burnham in1909. Burnham, who received his big break as an architect after the 1871 Great Chicago Fire and the massive reconstruction effort that ensued, later established himself as the designer of the homes of the wealthy Chicagoans. Though the architectural tastes of Burnham have been questioned by many, by the time of his death in 1912, his architectural firm was the largest in the country. Three years before his death he wrote his proposed plan for the city in essay form, advocating for wide boulevards and fountains, advocating Greek inspired architecture and city support for development. Michigan Avenue was indeed widened into a broad boulevard and the Bascule Bridge, built to connect the Avenue with Pine Street on the opposite bank of the Chicago River, was modeled after the Parisian Ponte Alexandre III. The Ponte Alexandre III is widely considered to be the most opulent and beautiful bridge in Paris. The Chicago Bridge was adorned with nymph and cherub statues and old Europe-inspired, street lamps. The bridge was followed by other large buildings throughout the 1920's as mentioned above. The oldest structure is the gothic-style Old Water Tower which. made of limestone block, managed to survive the Great Fire, dating back to 1869. This landmark which is one of the oldest in the city has become symbolic of the Magnificent Mile in Chicago and of the city itself. In 1969 it was named the first Water Landmark by the American Water Works Association. From the 1970's onwards construction filled out what remained of open spaces on the skyline and this once residential area became the shopping hub that it is today, lined with skyscrapers of diverse architectural styles and periods. One of the last big skyscrapers, the park tower, contains a hotel on the first 30 floors and then upwards fills with 37-floors of residential housing. Single family homes as well as condos and townhouses can be found along the Magnificent Mile and on surrounding blocks. Aside from the scores of shops and half a dozen hotels, diverse restaurants abound and all of the creature comforts are readily available in this busy district of the Windy City.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Rank: 49\n",
      "Document 7988: Northwestern University In Evanston, Illinois Illinois\n",
      "Rate: 4.5 Review Number: 389.0\n",
      "https://www.tripadvisor.com/Attraction_Review-g35972-d144239-Reviews-Northwestern_University-Evanston_Illinois.html\n",
      "Northwestern University is located in Evanston, Illinois and was founded in 1851, by the Methodists and opened its doors in 1855. At the time the faculty consisted of two teachers and ten students. The University derives its name from the nine founders desire to serve all the citizens that occupied of the former Northwest Territory.  These included Ohio, Indiana, Illinois, Minnesota, Wisconsin and Michigan. The original campus contained only one building, which was called Old Hall''. The first permanent building University Hall was built in 1869, and a campus was built in Chicago in 1920, for its graduate programs of law, medicine, and business. The Evanston College For Ladies merged with Northwestern University in 1873, and famed suffragist Frances Willard became the first dean of women. Northwestern University went co-ed in 1869, at the urging of Dean Erastus Haven, the first female student arrived in 1874. Northwestern University founding charter made it permanently paying property taxes, this continues to cause difficulties with the City Of Evanston government. In 1892, the schools colors of black and gold were replaced with purple, because it was thought the previous colors were used by too many other universities. While the color remains as royal purple, the University has also adopted white as its de facto color. The alma mater of the University is Hail to purple, Hail to White'', and both are listed in the guidelines. As of 1933, Northwestern University and its rival, the University of Chicago, united. It was concluded by the Deans of both schools to be in the best interest of finances and survival. The first-ever NCAA Men's Division/Basketball Championship games were held at Northwestern University. A program of African Studies was founded by anthropologist Melville J. Herskovits in 1948, this program was the first of its kind at any American educational institution. Northwestern University is nationally respected for its graduate programs in Law, Chemistry, Business, Education, Journalism, Music and many other areas. It was elected to the Association of American Universities in 1917. Northwestern University also boasts some very famous alumni including; Adlai Stevenson, Illinois Governor Rod Blagojevich, Academy-award winning actor Charlton Heston, Ann-Margret, Warren Beatty, David Schwimmer. Northwestern University's School of Medill (Journalism) is one of the top journalism, integrated marketing and media school in the United States and has produced famous journalists, political activists and nine Pulitzer Prize Laureates and to this day, continues to enjoy a reputation founded on excellence.\n"
     ]
    }
   ],
   "source": [
    "x = input(\"Input a query: \")\n",
    "df = ca_pipe_all.search(x)\n",
    "for idx in df.index:\n",
    "    result_id = int(df['docid'][idx])\n",
    "    print('\\n')\n",
    "    print('\\n')\n",
    "    print('Rank: '+str(df['rank'][idx]))\n",
    "    print('Document '+ str(result_id) + ': '+ dataset['Name'][result_id], dataset['State'][result_id])\n",
    "    print(\"Rate: \"+ str(dataset['overall_rate'][result_id]), \"Review Number: \"+ str(dataset['review_num'][result_id]))\n",
    "    print(dataset['url'][result_id])\n",
    "    print(dataset['Description'][result_id])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}