scripts/prediction_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The Prediction Model\n",
    "\n",
    "Using the data stored in `top_posts.csz.gz` this script will aim to make model that can predict the number of upvotes (likes) given all other data in the file excluding `number_of_upvotes`, `total_votes`, and `number_of_downvotes`.\n",
    "\n",
    "This script relies upon the results from the scripts: `optimize_top_words.ipynb` and `prediction_model_ablation.ipynb` for the feature vector, the size `n` for the most popular words, and the `alpha` value that will be assigned to the Ridge Regression Model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gzip\n",
    "import nltk\n",
    "import random\n",
    "import numpy as np\n",
    "from csv import writer\n",
    "from csv import DictReader\n",
    "from datetime import date\n",
    "from datetime import datetime\n",
    "from collections import defaultdict\n",
    "from sklearn.linear_model import Ridge\n",
    "from nltk.tokenize import word_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /Users/dwolfson/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /Users/dwolfson/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Download a few needed packages for the nltk \n",
    "nltk.download('punkt')\n",
    "nltk.download('averaged_perceptron_tagger')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Debug variables\n",
    "date = date.today().strftime(\"%b %d\")\n",
    "baseline = 0\n",
    "pred_mse = 0\n",
    "feature_list = []\n",
    "shuffle = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Most optimal size found for n-length list of popular words\n",
    "n = 600\n",
    "\n",
    "# Most optimal alpha found for Ridge Regression\n",
    "alpha = 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Returns the MSE of a list of preditions & labels\n",
    "def MSE(predictions, labels):\n",
    "    differences = [(x-y)**2 for x,y in zip(predictions,labels)]\n",
    "    return sum(differences) / len(differences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Returns a list the frequencies of a given title's Parts of Speech\n",
    "def parts_of_speech(title):\n",
    "    # Tokenize the words in the title\n",
    "    tokens = word_tokenize(title)\n",
    "    \n",
    "    # Turns each token into a pair with its value and Part of Speech label\n",
    "    # More Info Here: https://realpython.com/nltk-nlp-python/#tagging-parts-of-speech\n",
    "    pos = nltk.pos_tag(tokens)\n",
    "    \n",
    "    # Map the generalized Parts of Speech to their frequency in the title\n",
    "    frequencies = {\n",
    "        \"Adjectives\":0,\n",
    "        \"Nouns\":0,\n",
    "        \"Adverbs\":0,\n",
    "        \"Pronouns\":0,\n",
    "        \"Verbs\":0,\n",
    "        \"Determiners\":0\n",
    "    }\n",
    "    \n",
    "    # Count the frequencies of each Part of Speech generalizing to 7 categories\n",
    "    for pair in pos:\n",
    "        if pair[1].startswith(\"JJ\"):\n",
    "            frequencies[\"Adjectives\"] += 1\n",
    "        elif pair[1].startswith(\"NN\"):\n",
    "            frequencies[\"Nouns\"] += 1\n",
    "        elif pair[1].startswith(\"RB\"):\n",
    "            frequencies[\"Adverbs\"] += 1\n",
    "        elif pair[1].startswith(\"PRP\"):\n",
    "            frequencies[\"Pronouns\"] += 1\n",
    "        elif pair[1].startswith(\"VB\"):\n",
    "            frequencies[\"Verbs\"] += 1\n",
    "        elif pair[1].startswith(\"DT\"):\n",
    "            frequencies[\"Determiners\"] += 1\n",
    "        else:\n",
    "            pass\n",
    "\n",
    "    return list(frequencies.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Returns a one-hot encoding (OHE) of the hour of day, and weekday\n",
    "# OHE allows for encoding a n-length list of binary features in n - 1 space\n",
    "def one_hot_encoding_time(unixtime):\n",
    "    hour = [0] * 23\n",
    "    week = [0] * 6\n",
    "    \n",
    "    # Get the local time of the given unix timestamp\n",
    "    time = datetime.fromtimestamp(int(float(unixtime)))\n",
    "    \n",
    "    # One hot encode the hour (hour 0 is just a list of 0's)\n",
    "    # https://docs.python.org/3/library/datetime.html#datetime.datetime.hour\n",
    "    if time.hour != 0:\n",
    "        hour[time.hour - 1] = 1\n",
    "    \n",
    "    # One hot encode the weekday (day 0 is just a list of 0's)\n",
    "    # https://docs.python.org/3/library/datetime.html#datetime.date.weekday\n",
    "    if time.weekday() != 0:\n",
    "        week[time.weekday() - 1] = 1\n",
    "        \n",
    "    return hour + week\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper function returns a list that represents the presence of popular words\n",
    "def popular_words(title, n, n_popular_words):\n",
    "    words = [0] * n\n",
    "    \n",
    "    for word in word_tokenize(title):\n",
    "        if word in n_popular_words:\n",
    "            words[n_popular_words.index(word)] = 1\n",
    "    \n",
    "    return words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creates a feature vector for a given row of data\n",
    "def feature(datum):\n",
    "    feat = [1]\n",
    "    \n",
    "    n_popular_words = word_popularity[:n]\n",
    "    \n",
    "    # Add a feature for the score (price of awards given)\n",
    "    feat.append(int(datum['score']))\n",
    "    \n",
    "    # Add a feature for the number of comments\n",
    "    feat.append(int(datum['number_of_comments']))\n",
    "    \n",
    "    # Add a feature for character length of title\n",
    "    feat.append(len(datum['title']))\n",
    "    \n",
    "    # Add a feature for word length of title\n",
    "    feat.append(len(word_tokenize(datum['title'])))\n",
    "    \n",
    "    # Add a binary feature for if the content is declared original (OC)\n",
    "    feat.append(1) if \"[oc]\" in datum['title'].lower() else feat.append(0)\n",
    "    \n",
    "    # Add features for the frequencies of generalized Parts of Speech\n",
    "    feat.extend(parts_of_speech(datum['title']))\n",
    "    \n",
    "    # Add features for the one-hot encoding of the Hour and Weekday\n",
    "    feat.extend(one_hot_encoding_time(datum['unixtime']))\n",
    "    \n",
    "    # Add feature list for the presence of any of the n-most popular words\n",
    "    feat.extend(popular_words(datum['title'], n, n_popular_words))\n",
    "    \n",
    "    return feat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Insert debug for features\n",
    "feature_list.append('score')\n",
    "feature_list.append('number_of_comments')\n",
    "feature_list.append('title_length')\n",
    "feature_list.append('title_word_length')\n",
    "feature_list.append('orginal_content')\n",
    "feature_list.append('parts_of_speech')\n",
    "feature_list.append('ohe_hour')\n",
    "feature_list.append('ohe_week')\n",
    "feature_list.append('{}_popular_words'.format(n))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "word_popularity = defaultdict(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Open and store each post as a list of dict elements\n",
    "with gzip.open('../data/top_posts.csv.gz', 'rt') as file:\n",
    "    csv_reader = DictReader(file)\n",
    "    \n",
    "    for row in csv_reader:\n",
    "        data.append(row)\n",
    "        for word in word_tokenize(row['title']):\n",
    "            word_popularity[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_popularity = sorted(word_popularity.items(), key=lambda item: item[1], reverse=True)\n",
    "word_popularity = [pair[0] for pair in word_popularity]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Shuffle the data since it's sorted by subreddit to give a fairer distribution\n",
    "random.shuffle(data)\n",
    "shuffle = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X is the list of all feature vectors\n",
    "X = []\n",
    "# y is the list of labels (correct values)\n",
    "y = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "for datum in data:\n",
    "    X.append(feature(datum))\n",
    "    y.append(int(datum['number_of_upvotes']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "246472"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "246472"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Data Points:  197178\n",
      "Valid Data Points:  24647\n",
      "Test Data Points:   24647\n"
     ]
    }
   ],
   "source": [
    "# Split the datum between training (80%), validation (10%), and test (10%)\n",
    "train = round(len(X) * 0.8)\n",
    "valid = train + round(len(X) * 0.1)\n",
    "tests = train + round(len(X) * 0.1)\n",
    "\n",
    "print(f\"Train Data Points:  {train}\")\n",
    "print(f\"Valid Data Points:  {valid - train}\")\n",
    "print(f\"Test Data Points:   {tests - train}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X[:train]\n",
    "X_valid = X[train:valid]\n",
    "X_tests = X[valid:]\n",
    "\n",
    "y_train = y[:train]\n",
    "y_valid = y[train:valid]\n",
    "y_tests = y[valid:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get a baseline by testing the model against the average label\n",
    "average_likes = np.mean(y)\n",
    "y_avg = [average_likes] * len(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Minimum Number of Upvotes: 0\n",
      "Maximum Number of Upvotes: 430,539\n",
      "Average Number of Upvotes: 14,947.346384173456\n"
     ]
    }
   ],
   "source": [
    "# Get more information about y\n",
    "print(f\"Minimum Number of Upvotes: {format(min(y), ',')}\")\n",
    "print(f\"Maximum Number of Upvotes: {format(max(y), ',')}\")\n",
    "print(f\"Average Number of Upvotes: {format(average_likes, ',')}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Ridge(alpha=1000, fit_intercept=False)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Intialize and fit the model to the training datas\n",
    "model = Ridge(fit_intercept=False, alpha=alpha)\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test model on validation\n",
    "y_valid_pred = model.predict(X_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "510,078,874.7371405\n"
     ]
    }
   ],
   "source": [
    "# Get the MSE from the baseline averages\n",
    "baseline = MSE(y_avg[train:valid], y_valid)\n",
    "print(format(baseline, ','))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "367,630,069.5549524\n"
     ]
    }
   ],
   "source": [
    "# Get the MSE from the validation predictions\n",
    "pred_mse = MSE(y_valid_pred, y_valid)\n",
    "print(format(pred_mse, ','))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Append the debug data from this script to the CSV of MSE records\n",
    "with open('../data/prediction_model_MSE.csv', 'a') as file:\n",
    "    csv_writer = writer(file)\n",
    "    \n",
    "    row = []\n",
    "    row.append(date)\n",
    "    row.append(baseline)\n",
    "    row.append(pred_mse)\n",
    "    row.append('|'.join(feature_list))\n",
    "    row.append(shuffle)\n",
    "\n",
    "    csv_writer.writerow(row) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "357,822,196.4272623\n",
      "501,463,918.97177625\n"
     ]
    }
   ],
   "source": [
    "# Finally run the model on the test set and report the performance (MSE)\n",
    "y_tests_pred = model.predict(X_tests)\n",
    "pred_mse = MSE(y_tests_pred, y_tests)\n",
    "print(format(pred_mse, ','))\n",
    "print(format(MSE(y_avg[valid:], y_tests), ','))"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}