text-mining-part-3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Summary from part 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We are going to reload the reviews data from csv file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews_df = pd.read_csv('yelp_reviews_clemson_sc.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We are going to load the following libraries to help with the descriptive analysis process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Remove invalid rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "reviews_df = reviews_df.dropna()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Remove unnecessary symbols and phrases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_reviews_df = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for i in range(0,20):\n",
    "    print(reviews_df.restaurant[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_reviews_df['Review'] = reviews_df.Review.str[2:-2]\n",
    "clean_reviews_df['author'] = reviews_df.author.str[2:-2]\n",
    "clean_reviews_df['date'] = reviews_df.date.str[12:22].str.lstrip(' ').str.rstrip('\\\\n')\n",
    "clean_reviews_df['rating'] = reviews_df.rating.str[2:5]\n",
    "clean_reviews_df['restaurant'] = reviews_df.restaurant.str.replace(\"', '\", \" \").str[2:-2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_reviews_df.head(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_reviews_df['score'] = clean_reviews_df.rating.astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_reviews_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_reviews_df['review_length'] = clean_reviews_df['Review'].apply(lambda x: len(x) - x.count(' '))\n",
    "clean_reviews_df['Month'], clean_reviews_df['Day'], clean_reviews_df['Year'] = clean_reviews_df['date'].str.split('/').str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_reviews_df.head(15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove restaurants with too few reviews ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "restaurants = clean_reviews_df.groupby('restaurant').rating.count()\n",
    "restaurants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "restaurants = restaurants[restaurants > 40]\n",
    "restaurants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_reviews_df['restaurant'] = clean_reviews_df.restaurant.str.lstrip(\"one, '\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(restaurants.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "restaurant_df = clean_reviews_df[clean_reviews_df.restaurant.isin(list(restaurants.index))]\n",
    "restaurant_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### First analysis: Visual review lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hist = sns.FacetGrid(data=restaurant_df, col='rating')\n",
    "hist.map(plt.hist, 'review_length', bins=50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looks interesting, but not quite visible yet. Let's try another view"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.boxplot(x='rating', y='review_length', data=restaurant_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Yet another graphical library: ggplot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!pip install plotnine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import plotnine as p9\n",
    "p9.options.figure_size = (8, 6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Plotting distributions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(p9.ggplot(data=restaurant_df,\n",
    "           mapping=p9.aes(x='restaurant',\n",
    "                          y='score',color='restaurant'))\n",
    "    + p9.geom_jitter(alpha=0.2)\n",
    "    + p9.geom_boxplot(alpha=0.)\n",
    "    + p9.theme_bw()\n",
    "    + p9.theme(axis_text_x = p9.element_text(angle=90))\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### How have the review ratings evolved over time for the restaurants?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "yearly_counts = restaurant_df.groupby(['Year', 'restaurant'])['score'].mean()\n",
    "yearly_counts.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When checking the result of the previous calculation, we actually have both the year and the species_id as a row index. We can reset this index to use both as column variable:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "yearly_counts = yearly_counts.reset_index(name='score')\n",
    "yearly_counts['Year'] = yearly_counts.Year.astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yearly_counts=yearly_counts[yearly_counts.Year > 2007]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(p9.ggplot(data=yearly_counts,\n",
    "           mapping=p9.aes(x='Year',\n",
    "                          y='score'))\n",
    "    + p9.geom_line()\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Unfortunately this does not work, because we plot data for all the species together. We need to tell plotnine to draw a line for each restaurant by modifying the aesthetic function and map the restaurant to the color:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(p9.ggplot(data=yearly_counts,\n",
    "           mapping=p9.aes(x='Year',\n",
    "                          y='score',\n",
    "                          color='restaurant'))\n",
    "    + p9.geom_point()\n",
    "    + p9.geom_line()\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Faceting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(p9.ggplot(data=restaurant_df,\n",
    "           mapping=p9.aes(x='score',\n",
    "                          y='review_length',\n",
    "                          color='Year'))\n",
    "    + p9.geom_point(alpha=0.1)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(p9.ggplot(data=restaurant_df,\n",
    "           mapping=p9.aes(x='score',\n",
    "                          y='review_length',\n",
    "                          color='Year'))\n",
    "    + p9.geom_point(alpha=0.1)\n",
    "    + p9.facet_wrap(\"restaurant\")\n",
    "    + p9.theme(strip_text_x = p9.element_text(size = 4, colour = \"red\"))\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}