item_based_cf_solution.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = './data/'\n",
    "\n",
    "# define file path\n",
    "train_file = os.path.join(data_path, 'review_train.csv')\n",
    "test_file = os.path.join(data_path, 'review_test.csv')\n",
    "weight_matrix_file = os.path.join(data_path, 'weight_matrix.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read datasets\n",
    "train_data = pd.read_csv(train_file)\n",
    "test_data = pd.read_csv(test_file)\n",
    "weight_matrix = pd.read_csv(weight_matrix_file)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Item-based CF Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert the review data into a dictionary {(uid, bid): rating}\n",
    "review_dict = {(row[0], row[1]): row[2] for row in train_data.values.tolist()}\n",
    "\n",
    "# map each user_id to a list of business {user_id: list[business_ids]}\n",
    "user_groups = train_data.groupby('uid')\n",
    "user_business_dict = {uid: list(user_groups.get_group(uid)['bid']) for uid in user_groups.groups}\n",
    "\n",
    "# transfer weight_matrix to a dictionary {(bid1, bid2): weight}\n",
    "weight_matrix_dict = {(row[0], row[1]): row[2] for row in weight_matrix.values.tolist()}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find the most N correlated business according to the weight matrix\n",
    "\n",
    "WEIGHT_THRESHOLD = 0.0\n",
    "\n",
    "def find_n_nearest_business(target_bid, neighbor_business, n=3):\n",
    "    \n",
    "    neighbor_business_weight = {}\n",
    "    \n",
    "    for bid in neighbor_business:\n",
    "        b_pair = tuple(sorted([target_bid, bid]))\n",
    "        if weight_matrix_dict.get(b_pair) and weight_matrix_dict[b_pair] > WEIGHT_THRESHOLD:\n",
    "            neighbor_business_weight[bid] = weight_matrix_dict[b_pair]\n",
    "            \n",
    "    sorted_neighbor_business_weight = sorted(neighbor_business_weight.items(), key=lambda kv: kv[1], reverse=True)\n",
    "\n",
    "    if len(sorted_neighbor_business_weight) >= n: \n",
    "        return sorted_neighbor_business_weight[:n]\n",
    "    elif len(sorted_neighbor_business_weight) <= 1:\n",
    "        return []\n",
    "    else:\n",
    "        return sorted_neighbor_business_weight\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compute weighted average over neighborhood set\n",
    "\n",
    "def weighted_average_prediction(target_user, target_business, nearest_business):\n",
    "    \n",
    "    w_list, r_list = [], []\n",
    "\n",
    "    for item in nearest_business:\n",
    "        \n",
    "        bid, weight = item[0], item[1]\n",
    "        rating = review_dict[(target_user, bid)]\n",
    "        w_list.append(weight)\n",
    "        r_list.append(rating)\n",
    "\n",
    "#     print(w_list, r_list)\n",
    "    weighted_sum = sum([w_list[i] * r for i, r in enumerate(r_list)]) if len(r_list) > 0 else 0.0\n",
    "    sum_weight = sum([abs(x) for x in w_list]) if len(w_list) > 0 else 0.0\n",
    "    prediction = weighted_sum / sum_weight if sum_weight != 0.0 else 0.0\n",
    "    \n",
    "    return prediction\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make prediction for each given (user, business) pair in the testing data\n",
    "\n",
    "results = []\n",
    "\n",
    "for pair in test_data.values.tolist():\n",
    "    \n",
    "    target_user, target_business = pair[0], pair[1]\n",
    "    neighbor_business = user_business_dict.get(target_user)\n",
    "    \n",
    "    if neighbor_business:\n",
    "        \n",
    "        nearest_business = find_n_nearest_business(target_business, neighbor_business, n=3)\n",
    "        prediction = weighted_average_prediction(target_user,target_business, nearest_business)\n",
    "        results.append([target_user, target_business, prediction])\n",
    "    \n",
    "    else:  # cannot do the prediction if a user never appears in training data => cold start problem\n",
    "        results.append([target_user, target_business, 0.0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36480\n"
     ]
    }
   ],
   "source": [
    "print(len(results))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "36480"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# finalize the results\n",
    "# fill the missing predictions with average values\n",
    "\n",
    "avg_rating = sum(list(train_data['ratings'])) / len(train_data)\n",
    "\n",
    "def quick_check(x):\n",
    "    if x > 5:\n",
    "        return 5.0\n",
    "    elif x < 1:\n",
    "        return 1.0\n",
    "    else:\n",
    "        return x\n",
    "\n",
    "final_results = []\n",
    "\n",
    "for result in results:\n",
    "    uid, bid, prediction = result[0], result[1], result[2]\n",
    "    prediction = avg_rating if prediction == 0.0 else prediction\n",
    "    prediction = quick_check(prediction)\n",
    "    final_results.append([uid, bid, prediction])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write the results to a CSV file\n",
    "\n",
    "results_df = pd.DataFrame(final_results, columns=['uid', 'bid', 'prediction'])\n",
    "results_file_path = os.path.join(data_path, 'review_prediction.csv')\n",
    "results_df.to_csv(results_file_path, index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE = 1.169495745354903.\n"
     ]
    }
   ],
   "source": [
    "# load ground truth for the testing data\n",
    "\n",
    "test_ground_truth_file = os.path.join(data_path, 'review_test_ground_truth.csv')\n",
    "test_ground_truth_data = pd.read_csv(test_ground_truth_file)\n",
    "\n",
    "evaluation = test_ground_truth_data.merge(results_df, on=['uid', 'bid'])\n",
    "evaluation['delta'] = evaluation['ratings'] - evaluation['prediction']\n",
    "\n",
    "RMSE = (sum(evaluation['delta'] ** 2) / len(evaluation)) ** 0.5\n",
    "print('RMSE = {}.'.format(RMSE))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>bid</th>\n",
       "      <th>ratings</th>\n",
       "      <th>prediction</th>\n",
       "      <th>delta</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HEvyblFw4I-UsMqgPGYY_Q</td>\n",
       "      <td>iA8Ve2sZKN5Vz3mYKrtCaQ</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.631157</td>\n",
       "      <td>-0.631157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HEvyblFw4I-UsMqgPGYY_Q</td>\n",
       "      <td>iA8Ve2sZKN5Vz3mYKrtCaQ</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.631157</td>\n",
       "      <td>-0.631157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>HEvyblFw4I-UsMqgPGYY_Q</td>\n",
       "      <td>iA8Ve2sZKN5Vz3mYKrtCaQ</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.631157</td>\n",
       "      <td>-2.631157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>HEvyblFw4I-UsMqgPGYY_Q</td>\n",
       "      <td>iA8Ve2sZKN5Vz3mYKrtCaQ</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.631157</td>\n",
       "      <td>-2.631157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DNNkLmbwfI0ufKGqQfmvKQ</td>\n",
       "      <td>faPVqws-x-5k2CQKDNtHxw</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.426148</td>\n",
       "      <td>0.573852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>F_5_UNX-wrAFCXuAkBZRDw</td>\n",
       "      <td>UPIYuRaZvknINOd1w8kqRQ</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.260522</td>\n",
       "      <td>-0.260522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>ldqh2aWLTW6D2RHDCj_2TA</td>\n",
       "      <td>DHUAQ4pzH9KKzGZDm1jZLg</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.666667</td>\n",
       "      <td>-0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>f17-l69K0G7WAeTmPHtptw</td>\n",
       "      <td>HYKTKG3X7jtLe6elxp63JQ</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.764051</td>\n",
       "      <td>0.235949</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>FAjCZoxiGw9HJKueB8YWTg</td>\n",
       "      <td>O7ot_LMlCfLpOP9tBqeNfw</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.541475</td>\n",
       "      <td>0.458525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>leyNDNVu09Ldbg5ujPWMhQ</td>\n",
       "      <td>RESDUcs7fIiihp38-d6_6g</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.562917</td>\n",
       "      <td>-0.562917</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      uid                     bid  ratings  prediction  \\\n",
       "0  HEvyblFw4I-UsMqgPGYY_Q  iA8Ve2sZKN5Vz3mYKrtCaQ      3.0    3.631157   \n",
       "1  HEvyblFw4I-UsMqgPGYY_Q  iA8Ve2sZKN5Vz3mYKrtCaQ      3.0    3.631157   \n",
       "2  HEvyblFw4I-UsMqgPGYY_Q  iA8Ve2sZKN5Vz3mYKrtCaQ      1.0    3.631157   \n",
       "3  HEvyblFw4I-UsMqgPGYY_Q  iA8Ve2sZKN5Vz3mYKrtCaQ      1.0    3.631157   \n",
       "4  DNNkLmbwfI0ufKGqQfmvKQ  faPVqws-x-5k2CQKDNtHxw      5.0    4.426148   \n",
       "5  F_5_UNX-wrAFCXuAkBZRDw  UPIYuRaZvknINOd1w8kqRQ      4.0    4.260522   \n",
       "6  ldqh2aWLTW6D2RHDCj_2TA  DHUAQ4pzH9KKzGZDm1jZLg      3.0    3.666667   \n",
       "7  f17-l69K0G7WAeTmPHtptw  HYKTKG3X7jtLe6elxp63JQ      4.0    3.764051   \n",
       "8  FAjCZoxiGw9HJKueB8YWTg  O7ot_LMlCfLpOP9tBqeNfw      4.0    3.541475   \n",
       "9  leyNDNVu09Ldbg5ujPWMhQ  RESDUcs7fIiihp38-d6_6g      3.0    3.562917   \n",
       "\n",
       "      delta  \n",
       "0 -0.631157  \n",
       "1 -0.631157  \n",
       "2 -2.631157  \n",
       "3 -2.631157  \n",
       "4  0.573852  \n",
       "5 -0.260522  \n",
       "6 -0.666667  \n",
       "7  0.235949  \n",
       "8  0.458525  \n",
       "9 -0.562917  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluation[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: 28302, 1: 8328, 2: 995, 3: 121, 4: 6}\n"
     ]
    }
   ],
   "source": [
    "segments = {0:0, 1:0, 2:0, 3:0, 4:0}\n",
    "\n",
    "for _, row in evaluation.iterrows():\n",
    "    if row['delta'] < 1.0:\n",
    "        segments[0] += 1\n",
    "    elif 1.0 <= row['delta'] < 2.0:\n",
    "        segments[1] += 1\n",
    "    elif 2.0 <= row['delta'] < 3.0:\n",
    "        segments[2] += 1\n",
    "    elif 3.0 <= row['delta'] < 4.0:\n",
    "        segments[3] += 1\n",
    "    elif row['delta'] >= 4.0:\n",
    "        segments[4] += 1\n",
    "\n",
    "print(segments)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = './data'\n",
    "\n",
    "# define file path\n",
    "review_file = os.path.join(data_path, 'review_train.csv')\n",
    "business_file = os.path.join(data_path, 'sub_business.csv')\n",
    "\n",
    "review_data = pd.read_csv(review_file)\n",
    "business_data = pd.read_csv(business_file)\n",
    "\n",
    "reviews_with_location = review_data.merge(business_data, on='bid')\n",
    "reviews_with_location_file_path = os.path.join(data_path, 'reviews_with_location.csv')\n",
    "reviews_with_location.to_csv(reviews_with_location_file_path, index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}