notebooks/Ch10_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Chapter 10. Faster decision making with machine learning andPySpark\n",
    "====\n",
    "### Mastering Large Datasets with Python by JT Wolohan \n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decision Trees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark import SparkContext\n",
    "from pyspark.sql import SparkSession\n",
    "from functools import reduce\n",
    "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
    "from pyspark.ml.classification import DecisionTreeClassifier\n",
    "from pyspark.ml.evaluation import BinaryClassificationEvaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def string_to_index(df, label):\n",
    "     return StringIndexer(inputCol=label,\n",
    "                          outputCol=\"i-\"+label).fit(df) \\\n",
    "                                               .transform(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "spark = SparkSession.builder \\\n",
    "           .master(\"local\") \\\n",
    "           .appName(\"Decision Trees\") \\\n",
    "           .getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = spark.read.csv(\"../Ch10/mushrooms.data\", header=True, inferSchema=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "categories = ['cap-shape', 'cap-surface', 'cap-color']\n",
    "df = reduce(string_to_index, categories, df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = VectorAssembler(inputCols=[\"i-cap-shape\",\"i-cap-surface\", \"i-cap-color\"],\n",
    "                     outputCol=\"features\").transform(df)\n",
    "\n",
    "df = StringIndexer(inputCol='edible?', outputCol='label').fit(df).transform(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree = DecisionTreeClassifier()\n",
    "model = tree.fit(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DecisionTreeClassificationModel (uid=DecisionTreeClassifier_06513451d79b) of depth 5 with 29 nodes\n",
      "  If (feature 1 in {2.0,3.0})\n",
      "   If (feature 2 in {0.0,2.0,4.0,6.0,7.0})\n",
      "    If (feature 2 in {0.0,2.0,7.0})\n",
      "     If (feature 0 in {0.0,1.0,2.0,4.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 0 not in {0.0,1.0,2.0,4.0})\n",
      "      Predict: 1.0\n",
      "    Else (feature 2 not in {0.0,2.0,7.0})\n",
      "     If (feature 2 in {6.0})\n",
      "      Predict: 1.0\n",
      "     Else (feature 2 not in {6.0})\n",
      "      Predict: 0.0\n",
      "   Else (feature 2 not in {0.0,2.0,4.0,6.0,7.0})\n",
      "    If (feature 2 in {3.0})\n",
      "     Predict: 1.0\n",
      "    Else (feature 2 not in {3.0})\n",
      "     Predict: 0.0\n",
      "  Else (feature 1 not in {2.0,3.0})\n",
      "   If (feature 0 in {3.0,5.0})\n",
      "    If (feature 2 in {0.0,1.0,3.0})\n",
      "     If (feature 0 in {5.0})\n",
      "      Predict: 1.0\n",
      "     Else (feature 0 not in {5.0})\n",
      "      Predict: 0.0\n",
      "    Else (feature 2 not in {0.0,1.0,3.0})\n",
      "     If (feature 2 in {5.0,6.0})\n",
      "      Predict: 1.0\n",
      "     Else (feature 2 not in {5.0,6.0})\n",
      "      If (feature 0 in {5.0})\n",
      "       Predict: 1.0\n",
      "      Else (feature 0 not in {5.0})\n",
      "       Predict: 0.0\n",
      "   Else (feature 0 not in {3.0,5.0})\n",
      "    If (feature 0 in {2.0})\n",
      "     If (feature 2 in {1.0,4.0,5.0,6.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 2 not in {1.0,4.0,5.0,6.0})\n",
      "      Predict: 1.0\n",
      "    Else (feature 0 not in {2.0})\n",
      "     If (feature 2 in {8.0,9.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 2 not in {8.0,9.0})\n",
      "      Predict: 1.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(model.toDebugString)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decision Tree AUC: 0.6333\n"
     ]
    }
   ],
   "source": [
    "bce = BinaryClassificationEvaluator()\n",
    "\n",
    "auc = bce.evaluate(model.transform(df))\n",
    "print(\"Decision Tree AUC: {:0.4f}\".format(auc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark import SparkContext\n",
    "from pyspark.sql import SparkSession\n",
    "from functools import reduce\n",
    "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
    "from pyspark.ml.classification import RandomForestClassifier\n",
    "from pyspark.ml.evaluation import BinaryClassificationEvaluator\n",
    "from pyspark.ml.tuning import CrossValidator, ParamGridBuilder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "spark = SparkSession.builder \\\n",
    "           .master(\"local\") \\\n",
    "           .appName(\"Random Forests\") \\\n",
    "           .getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "bce = BinaryClassificationEvaluator()\n",
    "forest = RandomForestClassifier()\n",
    "df = spark.read.csv(\"../Ch10/mushrooms.data\", header=True, inferSchema=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "categories = df.columns\n",
    "categories.pop(categories.index('edible?'))\n",
    "df = reduce(string_to_index, categories, df)\n",
    "indexes = [\"i-\"+c for c in categories]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = VectorAssembler(inputCols=indexes,\n",
    "                     outputCol=\"features\").transform(df)\n",
    "df = StringIndexer(inputCol='edible?',\n",
    "                   outputCol='label').fit(df).transform(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid = ParamGridBuilder().addGrid(forest.maxDepth, [0, 2]).build()\n",
    "cv = CrossValidator(estimator=forest, estimatorParamMaps=grid,\n",
    "                        evaluator=bce,numFolds=10,\n",
    "                        parallelism=4)\n",
    "cv_model = cv.fit(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest AUC: 0.9950\n"
     ]
    }
   ],
   "source": [
    "area_under_curve = bce.evaluate(cv_model.transform(df))\n",
    "print(\"Random Forest AUC: {:0.4f}\".format(area_under_curve))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestClassificationModel (uid=RandomForestClassifier_3715b1717fde) with 20 trees\n",
      "  Tree 0 (weight 1.0):\n",
      "    If (feature 7 in {0.0})\n",
      "     If (feature 11 in {0.0,2.0,3.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 11 not in {0.0,2.0,3.0})\n",
      "      Predict: 1.0\n",
      "    Else (feature 7 not in {0.0})\n",
      "     If (feature 1 in {2.0,3.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 1 not in {2.0,3.0})\n",
      "      Predict: 1.0\n",
      "  Tree 1 (weight 1.0):\n",
      "    If (feature 19 in {1.0,2.0,4.0,5.0,6.0,7.0,8.0})\n",
      "     If (feature 4 in {0.0,4.0,5.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 4 not in {0.0,4.0,5.0})\n",
      "      Predict: 1.0\n",
      "    Else (feature 19 not in {1.0,2.0,4.0,5.0,6.0,7.0,8.0})\n",
      "     If (feature 21 in {6.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 21 not in {6.0})\n",
      "      Predict: 1.0\n",
      "  Tree 2 (weight 1.0):\n",
      "    If (feature 11 in {0.0,2.0,3.0})\n",
      "     Predict: 0.0\n",
      "    Else (feature 11 not in {0.0,2.0,3.0})\n",
      "     If (feature 20 in {2.0,3.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 20 not in {2.0,3.0})\n",
      "      Predict: 1.0\n",
      "  Tree 3 (weight 1.0):\n",
      "    If (feature 19 in {1.0,2.0,4.0,5.0,6.0,7.0,8.0})\n",
      "     If (feature 7 in {0.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 7 not in {0.0})\n",
      "      Predict: 1.0\n",
      "    Else (feature 19 not in {1.0,2.0,4.0,5.0,6.0,7.0,8.0})\n",
      "     If (feature 20 in {2.0,3.0,5.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 20 not in {2.0,3.0,5.0})\n",
      "      Predict: 1.0\n",
      "  Tree 4 (weight 1.0):\n",
      "    If (feature 8 in {2.0,3.0,6.0,7.0,8.0,9.0,10.0,11.0})\n",
      "     If (feature 4 in {0.0,4.0,5.0,8.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 4 not in {0.0,4.0,5.0,8.0})\n",
      "      Predict: 1.0\n",
      "    Else (feature 8 not in {2.0,3.0,6.0,7.0,8.0,9.0,10.0,11.0})\n",
      "     If (feature 19 in {1.0,2.0,4.0,6.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 19 not in {1.0,2.0,4.0,6.0})\n",
      "      Predict: 1.0\n",
      "  Tree 5 (weight 1.0):\n",
      "    If (feature 19 in {1.0,2.0,4.0,5.0,6.0,7.0,8.0})\n",
      "     Predict: 0.0\n",
      "    Else (feature 19 not in {1.0,2.0,4.0,5.0,6.0,7.0,8.0})\n",
      "     If (feature 20 in {2.0,3.0,5.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 20 not in {2.0,3.0,5.0})\n",
      "      Predict: 1.0\n",
      "  Tree 6 (weight 1.0):\n",
      "    If (feature 8 in {2.0,3.0,6.0,7.0,8.0,9.0,10.0,11.0})\n",
      "     Predict: 0.0\n",
      "    Else (feature 8 not in {2.0,3.0,6.0,7.0,8.0,9.0,10.0,11.0})\n",
      "     If (feature 18 in {0.0,3.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 18 not in {0.0,3.0})\n",
      "      Predict: 1.0\n",
      "  Tree 7 (weight 1.0):\n",
      "    If (feature 20 in {1.0,2.0,3.0,4.0,5.0})\n",
      "     If (feature 18 in {0.0,1.0,3.0,4.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 18 not in {0.0,1.0,3.0,4.0})\n",
      "      Predict: 1.0\n",
      "    Else (feature 20 not in {1.0,2.0,3.0,4.0,5.0})\n",
      "     If (feature 3 in {1.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 3 not in {1.0})\n",
      "      Predict: 1.0\n",
      "  Tree 8 (weight 1.0):\n",
      "    If (feature 19 in {1.0,2.0,4.0,5.0,6.0,7.0,8.0})\n",
      "     Predict: 0.0\n",
      "    Else (feature 19 not in {1.0,2.0,4.0,5.0,6.0,7.0,8.0})\n",
      "     If (feature 17 in {1.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 17 not in {1.0})\n",
      "      Predict: 1.0\n",
      "  Tree 9 (weight 1.0):\n",
      "    If (feature 7 in {0.0})\n",
      "     If (feature 19 in {0.0,1.0,2.0,5.0,7.0,8.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 19 not in {0.0,1.0,2.0,5.0,7.0,8.0})\n",
      "      Predict: 1.0\n",
      "    Else (feature 7 not in {0.0})\n",
      "     If (feature 1 in {2.0,3.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 1 not in {2.0,3.0})\n",
      "      Predict: 1.0\n",
      "  Tree 10 (weight 1.0):\n",
      "    If (feature 4 in {0.0,4.0,5.0,8.0})\n",
      "     Predict: 0.0\n",
      "    Else (feature 4 not in {0.0,4.0,5.0,8.0})\n",
      "     Predict: 1.0\n",
      "  Tree 11 (weight 1.0):\n",
      "    If (feature 19 in {1.0,2.0,4.0,5.0,6.0,7.0,8.0})\n",
      "     Predict: 0.0\n",
      "    Else (feature 19 not in {1.0,2.0,4.0,5.0,6.0,7.0,8.0})\n",
      "     If (feature 6 in {1.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 6 not in {1.0})\n",
      "      Predict: 1.0\n",
      "  Tree 12 (weight 1.0):\n",
      "    If (feature 12 in {0.0,2.0,3.0})\n",
      "     If (feature 4 in {0.0,4.0,5.0,8.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 4 not in {0.0,4.0,5.0,8.0})\n",
      "      Predict: 1.0\n",
      "    Else (feature 12 not in {0.0,2.0,3.0})\n",
      "     If (feature 18 in {0.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 18 not in {0.0})\n",
      "      Predict: 1.0\n",
      "  Tree 13 (weight 1.0):\n",
      "    If (feature 4 in {0.0,4.0,5.0,8.0})\n",
      "     If (feature 17 in {2.0})\n",
      "      Predict: 1.0\n",
      "     Else (feature 17 not in {2.0})\n",
      "      Predict: 0.0\n",
      "    Else (feature 4 not in {0.0,4.0,5.0,8.0})\n",
      "     Predict: 1.0\n",
      "  Tree 14 (weight 1.0):\n",
      "    If (feature 18 in {0.0,3.0,4.0})\n",
      "     If (feature 4 in {0.0,4.0,5.0,8.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 4 not in {0.0,4.0,5.0,8.0})\n",
      "      Predict: 1.0\n",
      "    Else (feature 18 not in {0.0,3.0,4.0})\n",
      "     If (feature 10 in {2.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 10 not in {2.0})\n",
      "      Predict: 1.0\n",
      "  Tree 15 (weight 1.0):\n",
      "    If (feature 19 in {1.0,2.0,4.0,5.0,6.0,7.0,8.0})\n",
      "     Predict: 0.0\n",
      "    Else (feature 19 not in {1.0,2.0,4.0,5.0,6.0,7.0,8.0})\n",
      "     If (feature 4 in {0.0,8.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 4 not in {0.0,8.0})\n",
      "      Predict: 1.0\n",
      "  Tree 16 (weight 1.0):\n",
      "    If (feature 4 in {0.0,4.0,5.0,8.0})\n",
      "     Predict: 0.0\n",
      "    Else (feature 4 not in {0.0,4.0,5.0,8.0})\n",
      "     Predict: 1.0\n",
      "  Tree 17 (weight 1.0):\n",
      "    If (feature 11 in {0.0,2.0,3.0})\n",
      "     If (feature 8 in {1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 8 not in {1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0})\n",
      "      Predict: 1.0\n",
      "    Else (feature 11 not in {0.0,2.0,3.0})\n",
      "     If (feature 4 in {0.0,8.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 4 not in {0.0,8.0})\n",
      "      Predict: 1.0\n",
      "  Tree 18 (weight 1.0):\n",
      "    If (feature 8 in {2.0,3.0,6.0,7.0,8.0,9.0,10.0,11.0})\n",
      "     If (feature 7 in {0.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 7 not in {0.0})\n",
      "      Predict: 1.0\n",
      "    Else (feature 8 not in {2.0,3.0,6.0,7.0,8.0,9.0,10.0,11.0})\n",
      "     If (feature 19 in {1.0,2.0,4.0,6.0})\n",
      "      Predict: 0.0\n",
      "     Else (feature 19 not in {1.0,2.0,4.0,6.0})\n",
      "      Predict: 1.0\n",
      "  Tree 19 (weight 1.0):\n",
      "    If (feature 18 in {0.0,3.0,4.0})\n",
      "     Predict: 0.0\n",
      "    Else (feature 18 not in {0.0,3.0,4.0})\n",
      "     Predict: 1.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(cv_model.bestModel.toDebugString)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Read for more? Go to chapter 11!](./Ch11_notebook.ipynb)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mldbook",
   "language": "python",
   "name": "mldbook"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}