lab_pearson_correlation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading training and testing review data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = './data/'\n",
    "\n",
    "# define file path\n",
    "train_file = os.path.join(data_path, 'review_train.csv')\n",
    "\n",
    "# read datasets\n",
    "train_data = pd.read_csv(train_file)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generating business pairs for the Pearson correlation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# notice that the similarity of (item1, item2) is the same as (item2, item1)\n",
    "# reduce the number of pairs in order to reduce the computation\n",
    "\n",
    "businesses = list(set(train_data['bid']))\n",
    "business_pairs = []\n",
    "\n",
    "for i in range(len(businesses)):\n",
    "    for j in range((i + 1), len(businesses)):\n",
    "        business_pairs.append(sorted([businesses[i], businesses[j]]))\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Computing the weight matrix (Pearson similarity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert the review data into a dictionary {(uid, bid): rating}\n",
    "review_dict = {(row[0], row[1]): row[2] for row in train_data.values.tolist()}\n",
    "\n",
    "# map each business id to a list of users rated on that {business_id: list[user_ids]}\n",
    "business_groups = train_data.groupby('bid')\n",
    "business_user_dict = {bid: list(business_groups.get_group(bid)['uid']) for bid in business_groups.groups}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define a function to find common (co-rated) user list for two given businesses\n",
    "\n",
    "def find_co_rated_users(bid_1, bid_2):\n",
    "    \n",
    "    # Hint: utilize the \"business_user_dict\" to query the user lists\n",
    "    \n",
    "    ### pseudocode ###\n",
    "    # user_list_1 = the users who rated on bid_1\n",
    "    # user_list_2 = the users who rated on bid_2\n",
    "    # co_rated_users = the intersection (user_list_1, user_list_2)\n",
    "    # return co_rated_users\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define a function to compute Pearson correlation\n",
    "\n",
    "import math\n",
    "\n",
    "def compute_pearson_correlation(rating_list_1, rating_list_2):\n",
    "\n",
    "    ### pseudocode ###\n",
    "    # avg_rating_1 = the average rating of rating_list_1 \n",
    "    # avg_rating_2 = the average rating of rating_list_2\n",
    "\n",
    "    # weight_sum, weight_1, weight_2 = 0.0, 0.0, 0.0\n",
    "    # calculate the value for weight_sum (numerator) and weight_1, weight_2 (denominator)\n",
    "        \n",
    "    # return weight_sum / sqrt(weight_1) / sqrt(weight_2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compute the Pearson correlation for each business pair\n",
    "\n",
    "PEARSON_THRED = 0.1 \n",
    "weight_matrix = []\n",
    "\n",
    "for pair in business_pairs:\n",
    "    \n",
    "    bid_1, bid_2 = pair[0], pair[1]\n",
    "    \n",
    "    ### pseudocode ###\n",
    "    \n",
    "    # call find_co_rated_users()\n",
    "    # if the number of co_rated_users <= 1, weight should be 0.0\n",
    "    \n",
    "    # get the rating list of the co-rated users on bid_1 and bid_2\n",
    "    # call compute_pearson_correlation()\n",
    "\n",
    "    if weight > PEARSON_THRED:  # you can set some threshold to filter the low correlated values\n",
    "        weight_matrix.append([bid_1, bid_2, weight])  # (bid_1, bid_2) is in alphabetical order\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write the results to a CSV file\n",
    "\n",
    "weight_matrix_df = pd.DataFrame(weight_matrix, columns=['bid1', 'bid2', 'corr'])\n",
    "weight_matrix_file_path = os.path.join(data_path, 'weight_matrix.csv')\n",
    "weight_matrix_df.to_csv(weight_matrix_file_path, index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}