demo_WanderingEarth.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('liulang.csv', names=['Num','nickname','link','img_link','start','eval','votes','data','comments'],skiprows=[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Num</th>\n",
       "      <th>nickname</th>\n",
       "      <th>link</th>\n",
       "      <th>img_link</th>\n",
       "      <th>start</th>\n",
       "      <th>eval</th>\n",
       "      <th>votes</th>\n",
       "      <th>data</th>\n",
       "      <th>comments</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>侠侠</td>\n",
       "      <td>https://www.douban.com/people/40082523/</td>\n",
       "      <td>https://img3.doubanio.com/icon/u40082523-2.jpg</td>\n",
       "      <td>10</td>\n",
       "      <td>很差</td>\n",
       "      <td>35972</td>\n",
       "      <td>\\r\\n                    2019-02-05\\r\\n        ...</td>\n",
       "      <td>真为吴京的演技尴尬，总是摆出一副大义凌然的样子，好奇为什么刘的作品中总有这种傻逼般的圣母存在...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>影志</td>\n",
       "      <td>https://www.douban.com/people/tjz230/</td>\n",
       "      <td>https://img1.doubanio.com/icon/u1005928-127.jpg</td>\n",
       "      <td>40</td>\n",
       "      <td>推荐</td>\n",
       "      <td>66841</td>\n",
       "      <td>\\r\\n                    2019-02-04\\r\\n        ...</td>\n",
       "      <td>电影比预期要更恢弘磅礴，晨昏线过后的永夜、火种计划、让地球流浪、木星推动地球…等等大小设定，...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>嘟嘟熊之父 🧸</td>\n",
       "      <td>https://www.douban.com/people/duduxiongzhifu/</td>\n",
       "      <td>https://img3.doubanio.com/icon/u2201715-15.jpg</td>\n",
       "      <td>10</td>\n",
       "      <td>很差</td>\n",
       "      <td>67180</td>\n",
       "      <td>\\r\\n                    2019-01-28\\r\\n        ...</td>\n",
       "      <td>还能更土更儿戏一点吗？毫无思考仅靠煽动，毫无敬畏仅余妄想。好的科幻片应该首先承认人类的无知，...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>陆支羽</td>\n",
       "      <td>https://www.douban.com/people/luzhiyu/</td>\n",
       "      <td>https://img3.doubanio.com/icon/u2866549-132.jpg</td>\n",
       "      <td>50</td>\n",
       "      <td>力荐</td>\n",
       "      <td>58208</td>\n",
       "      <td>\\r\\n                    2019-01-29\\r\\n        ...</td>\n",
       "      <td>1.终于，轮到我们仰望星空。2.后启示录死亡废墟，赛博朋克地下城，以及烟波浩渺的末日想象，缔...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>沙雕电影</td>\n",
       "      <td>https://www.douban.com/people/185573840/</td>\n",
       "      <td>https://img3.doubanio.com/icon/u185573840-2.jpg</td>\n",
       "      <td>40</td>\n",
       "      <td>推荐</td>\n",
       "      <td>32570</td>\n",
       "      <td>\\r\\n                    2019-02-05\\r\\n        ...</td>\n",
       "      <td>一个悲伤的故事：太阳都要毁灭，地球都要流浪了，我国的校服还是这么丑......</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Num nickname                                           link  \\\n",
       "0    0       侠侠        https://www.douban.com/people/40082523/   \n",
       "1    1       影志          https://www.douban.com/people/tjz230/   \n",
       "2    2  嘟嘟熊之父 🧸  https://www.douban.com/people/duduxiongzhifu/   \n",
       "3    3      陆支羽         https://www.douban.com/people/luzhiyu/   \n",
       "4    4     沙雕电影       https://www.douban.com/people/185573840/   \n",
       "\n",
       "                                          img_link  start eval  votes  \\\n",
       "0   https://img3.doubanio.com/icon/u40082523-2.jpg     10   很差  35972   \n",
       "1  https://img1.doubanio.com/icon/u1005928-127.jpg     40   推荐  66841   \n",
       "2   https://img3.doubanio.com/icon/u2201715-15.jpg     10   很差  67180   \n",
       "3  https://img3.doubanio.com/icon/u2866549-132.jpg     50   力荐  58208   \n",
       "4  https://img3.doubanio.com/icon/u185573840-2.jpg     40   推荐  32570   \n",
       "\n",
       "                                                data  \\\n",
       "0  \\r\\n                    2019-02-05\\r\\n        ...   \n",
       "1  \\r\\n                    2019-02-04\\r\\n        ...   \n",
       "2  \\r\\n                    2019-01-28\\r\\n        ...   \n",
       "3  \\r\\n                    2019-01-29\\r\\n        ...   \n",
       "4  \\r\\n                    2019-02-05\\r\\n        ...   \n",
       "\n",
       "                                            comments  \n",
       "0  真为吴京的演技尴尬，总是摆出一副大义凌然的样子，好奇为什么刘的作品中总有这种傻逼般的圣母存在...  \n",
       "1  电影比预期要更恢弘磅礴，晨昏线过后的永夜、火种计划、让地球流浪、木星推动地球…等等大小设定，...  \n",
       "2  还能更土更儿戏一点吗？毫无思考仅靠煽动，毫无敬畏仅余妄想。好的科幻片应该首先承认人类的无知，...  \n",
       "3  1.终于，轮到我们仰望星空。2.后启示录死亡废墟，赛博朋克地下城，以及烟波浩渺的末日想象，缔...  \n",
       "4            一个悲伤的故事：太阳都要毁灭，地球都要流浪了，我国的校服还是这么丑......  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_label(df):\n",
    "    df[\"sentiment\"] = df[\"start\"].apply(lambda x: 1 if x>30 else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "make_label(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Num</th>\n",
       "      <th>nickname</th>\n",
       "      <th>link</th>\n",
       "      <th>img_link</th>\n",
       "      <th>start</th>\n",
       "      <th>eval</th>\n",
       "      <th>votes</th>\n",
       "      <th>data</th>\n",
       "      <th>comments</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>侠侠</td>\n",
       "      <td>https://www.douban.com/people/40082523/</td>\n",
       "      <td>https://img3.doubanio.com/icon/u40082523-2.jpg</td>\n",
       "      <td>10</td>\n",
       "      <td>很差</td>\n",
       "      <td>35972</td>\n",
       "      <td>\\r\\n                    2019-02-05\\r\\n        ...</td>\n",
       "      <td>真为吴京的演技尴尬，总是摆出一副大义凌然的样子，好奇为什么刘的作品中总有这种傻逼般的圣母存在...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>影志</td>\n",
       "      <td>https://www.douban.com/people/tjz230/</td>\n",
       "      <td>https://img1.doubanio.com/icon/u1005928-127.jpg</td>\n",
       "      <td>40</td>\n",
       "      <td>推荐</td>\n",
       "      <td>66841</td>\n",
       "      <td>\\r\\n                    2019-02-04\\r\\n        ...</td>\n",
       "      <td>电影比预期要更恢弘磅礴，晨昏线过后的永夜、火种计划、让地球流浪、木星推动地球…等等大小设定，...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>嘟嘟熊之父 🧸</td>\n",
       "      <td>https://www.douban.com/people/duduxiongzhifu/</td>\n",
       "      <td>https://img3.doubanio.com/icon/u2201715-15.jpg</td>\n",
       "      <td>10</td>\n",
       "      <td>很差</td>\n",
       "      <td>67180</td>\n",
       "      <td>\\r\\n                    2019-01-28\\r\\n        ...</td>\n",
       "      <td>还能更土更儿戏一点吗？毫无思考仅靠煽动，毫无敬畏仅余妄想。好的科幻片应该首先承认人类的无知，...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>陆支羽</td>\n",
       "      <td>https://www.douban.com/people/luzhiyu/</td>\n",
       "      <td>https://img3.doubanio.com/icon/u2866549-132.jpg</td>\n",
       "      <td>50</td>\n",
       "      <td>力荐</td>\n",
       "      <td>58208</td>\n",
       "      <td>\\r\\n                    2019-01-29\\r\\n        ...</td>\n",
       "      <td>1.终于，轮到我们仰望星空。2.后启示录死亡废墟，赛博朋克地下城，以及烟波浩渺的末日想象，缔...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>沙雕电影</td>\n",
       "      <td>https://www.douban.com/people/185573840/</td>\n",
       "      <td>https://img3.doubanio.com/icon/u185573840-2.jpg</td>\n",
       "      <td>40</td>\n",
       "      <td>推荐</td>\n",
       "      <td>32570</td>\n",
       "      <td>\\r\\n                    2019-02-05\\r\\n        ...</td>\n",
       "      <td>一个悲伤的故事：太阳都要毁灭，地球都要流浪了，我国的校服还是这么丑......</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Num nickname                                           link  \\\n",
       "0    0       侠侠        https://www.douban.com/people/40082523/   \n",
       "1    1       影志          https://www.douban.com/people/tjz230/   \n",
       "2    2  嘟嘟熊之父 🧸  https://www.douban.com/people/duduxiongzhifu/   \n",
       "3    3      陆支羽         https://www.douban.com/people/luzhiyu/   \n",
       "4    4     沙雕电影       https://www.douban.com/people/185573840/   \n",
       "\n",
       "                                          img_link  start eval  votes  \\\n",
       "0   https://img3.doubanio.com/icon/u40082523-2.jpg     10   很差  35972   \n",
       "1  https://img1.doubanio.com/icon/u1005928-127.jpg     40   推荐  66841   \n",
       "2   https://img3.doubanio.com/icon/u2201715-15.jpg     10   很差  67180   \n",
       "3  https://img3.doubanio.com/icon/u2866549-132.jpg     50   力荐  58208   \n",
       "4  https://img3.doubanio.com/icon/u185573840-2.jpg     40   推荐  32570   \n",
       "\n",
       "                                                data  \\\n",
       "0  \\r\\n                    2019-02-05\\r\\n        ...   \n",
       "1  \\r\\n                    2019-02-04\\r\\n        ...   \n",
       "2  \\r\\n                    2019-01-28\\r\\n        ...   \n",
       "3  \\r\\n                    2019-01-29\\r\\n        ...   \n",
       "4  \\r\\n                    2019-02-05\\r\\n        ...   \n",
       "\n",
       "                                            comments  sentiment  \n",
       "0  真为吴京的演技尴尬，总是摆出一副大义凌然的样子，好奇为什么刘的作品中总有这种傻逼般的圣母存在...          0  \n",
       "1  电影比预期要更恢弘磅礴，晨昏线过后的永夜、火种计划、让地球流浪、木星推动地球…等等大小设定，...          1  \n",
       "2  还能更土更儿戏一点吗？毫无思考仅靠煽动，毫无敬畏仅余妄想。好的科幻片应该首先承认人类的无知，...          0  \n",
       "3  1.终于，轮到我们仰望星空。2.后启示录死亡废墟，赛博朋克地下城，以及烟波浩渺的末日想象，缔...          1  \n",
       "4            一个悲伤的故事：太阳都要毁灭，地球都要流浪了，我国的校服还是这么丑......          1  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df[['comments']]\n",
    "y = df.sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(30000, 1)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(30000,)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comments</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>真为吴京的演技尴尬，总是摆出一副大义凌然的样子，好奇为什么刘的作品中总有这种傻逼般的圣母存在...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>电影比预期要更恢弘磅礴，晨昏线过后的永夜、火种计划、让地球流浪、木星推动地球…等等大小设定，...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>还能更土更儿戏一点吗？毫无思考仅靠煽动，毫无敬畏仅余妄想。好的科幻片应该首先承认人类的无知，...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.终于，轮到我们仰望星空。2.后启示录死亡废墟，赛博朋克地下城，以及烟波浩渺的末日想象，缔...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>一个悲伤的故事：太阳都要毁灭，地球都要流浪了，我国的校服还是这么丑......</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            comments\n",
       "0  真为吴京的演技尴尬，总是摆出一副大义凌然的样子，好奇为什么刘的作品中总有这种傻逼般的圣母存在...\n",
       "1  电影比预期要更恢弘磅礴，晨昏线过后的永夜、火种计划、让地球流浪、木星推动地球…等等大小设定，...\n",
       "2  还能更土更儿戏一点吗？毫无思考仅靠煽动，毫无敬畏仅余妄想。好的科幻片应该首先承认人类的无知，...\n",
       "3  1.终于，轮到我们仰望星空。2.后启示录死亡废墟，赛博朋克地下城，以及烟波浩渺的末日想象，缔...\n",
       "4            一个悲伤的故事：太阳都要毁灭，地球都要流浪了，我国的校服还是这么丑......"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import jieba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def chinese_word_cut(mytext):\n",
    "    return \" \".join(jieba.cut(mytext))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "X['cutted_comment'] = X.comments.apply(chinese_word_cut)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    真为 吴京 的 演技 尴尬 ， 总是 摆 出 一副 大义 凌然 的 样子 ， 好奇 为什么 ...\n",
       "1    电影 比 预期 要 更 恢弘 磅礴 ， 晨昏 线 过后 的 永夜 、 火种 计划 、 让 地...\n",
       "2    还 能 更 土 更 儿戏 一点 吗 ？ 毫无 思考 仅靠 煽动 ， 毫无 敬畏 仅余 妄想 ...\n",
       "3    1 . 终于 ， 轮 到 我们 仰望 星空 。 2 . 后 启示录 死亡 废墟 ， 赛博 朋...\n",
       "4    一个 悲伤 的 故事 ： 太阳 都 要 毁灭 ， 地球 都 要 流浪 了 ， 我国 的 校服...\n",
       "Name: cutted_comment, dtype: object"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.cutted_comment[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comments</th>\n",
       "      <th>cutted_comment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>真为吴京的演技尴尬，总是摆出一副大义凌然的样子，好奇为什么刘的作品中总有这种傻逼般的圣母存在...</td>\n",
       "      <td>真为 吴京 的 演技 尴尬 ， 总是 摆 出 一副 大义 凌然 的 样子 ， 好奇 为什么 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>电影比预期要更恢弘磅礴，晨昏线过后的永夜、火种计划、让地球流浪、木星推动地球…等等大小设定，...</td>\n",
       "      <td>电影 比 预期 要 更 恢弘 磅礴 ， 晨昏 线 过后 的 永夜 、 火种 计划 、 让 地...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>还能更土更儿戏一点吗？毫无思考仅靠煽动，毫无敬畏仅余妄想。好的科幻片应该首先承认人类的无知，...</td>\n",
       "      <td>还 能 更 土 更 儿戏 一点 吗 ？ 毫无 思考 仅靠 煽动 ， 毫无 敬畏 仅余 妄想 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.终于，轮到我们仰望星空。2.后启示录死亡废墟，赛博朋克地下城，以及烟波浩渺的末日想象，缔...</td>\n",
       "      <td>1 . 终于 ， 轮 到 我们 仰望 星空 。 2 . 后 启示录 死亡 废墟 ， 赛博 朋...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>一个悲伤的故事：太阳都要毁灭，地球都要流浪了，我国的校服还是这么丑......</td>\n",
       "      <td>一个 悲伤 的 故事 ： 太阳 都 要 毁灭 ， 地球 都 要 流浪 了 ， 我国 的 校服...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            comments  \\\n",
       "0  真为吴京的演技尴尬，总是摆出一副大义凌然的样子，好奇为什么刘的作品中总有这种傻逼般的圣母存在...   \n",
       "1  电影比预期要更恢弘磅礴，晨昏线过后的永夜、火种计划、让地球流浪、木星推动地球…等等大小设定，...   \n",
       "2  还能更土更儿戏一点吗？毫无思考仅靠煽动，毫无敬畏仅余妄想。好的科幻片应该首先承认人类的无知，...   \n",
       "3  1.终于，轮到我们仰望星空。2.后启示录死亡废墟，赛博朋克地下城，以及烟波浩渺的末日想象，缔...   \n",
       "4            一个悲伤的故事：太阳都要毁灭，地球都要流浪了，我国的校服还是这么丑......   \n",
       "\n",
       "                                      cutted_comment  \n",
       "0  真为 吴京 的 演技 尴尬 ， 总是 摆 出 一副 大义 凌然 的 样子 ， 好奇 为什么 ...  \n",
       "1  电影 比 预期 要 更 恢弘 磅礴 ， 晨昏 线 过后 的 永夜 、 火种 计划 、 让 地...  \n",
       "2  还 能 更 土 更 儿戏 一点 吗 ？ 毫无 思考 仅靠 煽动 ， 毫无 敬畏 仅余 妄想 ...  \n",
       "3  1 . 终于 ， 轮 到 我们 仰望 星空 。 2 . 后 启示录 死亡 废墟 ， 赛博 朋...  \n",
       "4  一个 悲伤 的 故事 ： 太阳 都 要 毁灭 ， 地球 都 要 流浪 了 ， 我国 的 校服...  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22500, 2)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comments</th>\n",
       "      <th>cutted_comment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26340</th>\n",
       "      <td>真为吴京的演技尴尬，总是摆出一副大义凌然的样子，好奇为什么刘的作品中总有这种傻逼般的圣母存在...</td>\n",
       "      <td>真为 吴京 的 演技 尴尬 ， 总是 摆 出 一副 大义 凌然 的 样子 ， 好奇 为什么 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3877</th>\n",
       "      <td>充斥着模仿的痕迹，某些镜头直接地心引力同款，另外就是画质极渣的摄影以及播放卡顿。宏大主题配以...</td>\n",
       "      <td>充斥 着 模仿 的 痕迹 ， 某些 镜头 直接 地心引力 同款 ， 另外 就是 画质 极渣 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27081</th>\n",
       "      <td>1.终于，轮到我们仰望星空。2.后启示录死亡废墟，赛博朋克地下城，以及烟波浩渺的末日想象，缔...</td>\n",
       "      <td>1 . 终于 ， 轮 到 我们 仰望 星空 。 2 . 后 启示录 死亡 废墟 ， 赛博 朋...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5440</th>\n",
       "      <td>还能更土更儿戏一点吗？毫无思考仅靠煽动，毫无敬畏仅余妄想。好的科幻片应该首先承认人类的无知，...</td>\n",
       "      <td>还 能 更 土 更 儿戏 一点 吗 ？ 毫无 思考 仅靠 煽动 ， 毫无 敬畏 仅余 妄想 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22164</th>\n",
       "      <td>还能更土更儿戏一点吗？毫无思考仅靠煽动，毫无敬畏仅余妄想。好的科幻片应该首先承认人类的无知，...</td>\n",
       "      <td>还 能 更 土 更 儿戏 一点 吗 ？ 毫无 思考 仅靠 煽动 ， 毫无 敬畏 仅余 妄想 ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                comments  \\\n",
       "26340  真为吴京的演技尴尬，总是摆出一副大义凌然的样子，好奇为什么刘的作品中总有这种傻逼般的圣母存在...   \n",
       "3877   充斥着模仿的痕迹，某些镜头直接地心引力同款，另外就是画质极渣的摄影以及播放卡顿。宏大主题配以...   \n",
       "27081  1.终于，轮到我们仰望星空。2.后启示录死亡废墟，赛博朋克地下城，以及烟波浩渺的末日想象，缔...   \n",
       "5440   还能更土更儿戏一点吗？毫无思考仅靠煽动，毫无敬畏仅余妄想。好的科幻片应该首先承认人类的无知，...   \n",
       "22164  还能更土更儿戏一点吗？毫无思考仅靠煽动，毫无敬畏仅余妄想。好的科幻片应该首先承认人类的无知，...   \n",
       "\n",
       "                                          cutted_comment  \n",
       "26340  真为 吴京 的 演技 尴尬 ， 总是 摆 出 一副 大义 凌然 的 样子 ， 好奇 为什么 ...  \n",
       "3877   充斥 着 模仿 的 痕迹 ， 某些 镜头 直接 地心引力 同款 ， 另外 就是 画质 极渣 ...  \n",
       "27081  1 . 终于 ， 轮 到 我们 仰望 星空 。 2 . 后 启示录 死亡 废墟 ， 赛博 朋...  \n",
       "5440   还 能 更 土 更 儿戏 一点 吗 ？ 毫无 思考 仅靠 煽动 ， 毫无 敬畏 仅余 妄想 ...  \n",
       "22164  还 能 更 土 更 儿戏 一点 吗 ？ 毫无 思考 仅靠 煽动 ， 毫无 敬畏 仅余 妄想 ...  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22500,)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7500, 2)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7500,)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_custom_stopwords(stop_words_file):\n",
    "    with open(stop_words_file) as f:\n",
    "        stopwords = f.read()\n",
    "    stopwords_list = stopwords.split('\\n')\n",
    "    custom_stopwords_list = [i for i in stopwords_list]\n",
    "    return custom_stopwords_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_words_file = \"stopwords.txt\"\n",
    "stopwords = get_custom_stopwords(stop_words_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['如下', '汝', '三番两次', '三番五次', '三天两头', '瑟瑟', '沙沙', '上', '上来', '上去']"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stopwords[-10:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "vect = CountVectorizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "term_matrix = pd.DataFrame(vect.fit_transform(X_train.cutted_comment).toarray(), columns=vect.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>100</th>\n",
       "      <th>30</th>\n",
       "      <th>89</th>\n",
       "      <th>bb</th>\n",
       "      <th>bgm</th>\n",
       "      <th>eason</th>\n",
       "      <th>frost</th>\n",
       "      <th>punk</th>\n",
       "      <th>一下</th>\n",
       "      <th>一丝</th>\n",
       "      <th>...</th>\n",
       "      <th>鞋里</th>\n",
       "      <th>预期</th>\n",
       "      <th>题材</th>\n",
       "      <th>飞船</th>\n",
       "      <th>首先</th>\n",
       "      <th>首部</th>\n",
       "      <th>骄傲</th>\n",
       "      <th>高分</th>\n",
       "      <th>高空</th>\n",
       "      <th>鼓励</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 707 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   100  30  89  bb  bgm  eason  frost  punk  一下  一丝 ...  鞋里  预期  题材  飞船  首先  \\\n",
       "0    0   0   0   0    0      0      0     0   0   0 ...   0   0   1   0   0   \n",
       "1    0   0   0   0    0      0      0     0   0   0 ...   0   0   0   0   0   \n",
       "2    0   0   0   0    0      0      0     0   0   0 ...   0   0   0   0   0   \n",
       "3    0   0   0   0    0      0      0     0   0   0 ...   0   0   0   0   1   \n",
       "4    0   0   0   0    0      0      0     0   0   0 ...   0   0   0   0   1   \n",
       "\n",
       "   首部  骄傲  高分  高空  鼓励  \n",
       "0   0   0   0   0   0  \n",
       "1   0   0   0   0   0  \n",
       "2   0   0   0   0   0  \n",
       "3   0   0   0   0   0  \n",
       "4   0   0   0   0   0  \n",
       "\n",
       "[5 rows x 707 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "term_matrix.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22500, 707)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "term_matrix.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "vect = CountVectorizer(stop_words=frozenset(stopwords))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "term_matrix = pd.DataFrame(vect.fit_transform(X_train.cutted_comment).toarray(), columns=vect.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>100</th>\n",
       "      <th>30</th>\n",
       "      <th>89</th>\n",
       "      <th>bb</th>\n",
       "      <th>bgm</th>\n",
       "      <th>eason</th>\n",
       "      <th>frost</th>\n",
       "      <th>punk</th>\n",
       "      <th>一丝</th>\n",
       "      <th>一分</th>\n",
       "      <th>...</th>\n",
       "      <th>零下</th>\n",
       "      <th>鞋里</th>\n",
       "      <th>预期</th>\n",
       "      <th>题材</th>\n",
       "      <th>飞船</th>\n",
       "      <th>首部</th>\n",
       "      <th>骄傲</th>\n",
       "      <th>高分</th>\n",
       "      <th>高空</th>\n",
       "      <th>鼓励</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 582 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   100  30  89  bb  bgm  eason  frost  punk  一丝  一分 ...  零下  鞋里  预期  题材  飞船  \\\n",
       "0    0   0   0   0    0      0      0     0   0   0 ...   0   0   0   1   0   \n",
       "1    0   0   0   0    0      0      0     0   0   0 ...   0   0   0   0   0   \n",
       "2    0   0   0   0    0      0      0     0   0   0 ...   0   0   0   0   0   \n",
       "3    0   0   0   0    0      0      0     0   0   0 ...   0   0   0   0   0   \n",
       "4    0   0   0   0    0      0      0     0   0   0 ...   0   0   0   0   0   \n",
       "\n",
       "   首部  骄傲  高分  高空  鼓励  \n",
       "0   0   0   0   0   0  \n",
       "1   0   0   0   0   0  \n",
       "2   0   0   0   0   0  \n",
       "3   0   0   0   0   0  \n",
       "4   0   0   0   0   0  \n",
       "\n",
       "[5 rows x 582 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "term_matrix.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_df = 0.8 # 在超过这一比例的文档中出现的关键词（过于平凡），去除掉。\n",
    "min_df = 3 # 在低于这一数量的文档中出现的关键词（过于独特），去除掉。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "vect = CountVectorizer(max_df = max_df,\n",
    "                       min_df = min_df,\n",
    "                       token_pattern=u'(?u)\\\\b[^\\\\d\\\\W]\\\\w+\\\\b',\n",
    "                       stop_words=frozenset(stopwords))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "term_matrix = pd.DataFrame(vect.fit_transform(X_train.cutted_comment).toarray(), columns=vect.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bb</th>\n",
       "      <th>bgm</th>\n",
       "      <th>eason</th>\n",
       "      <th>frost</th>\n",
       "      <th>punk</th>\n",
       "      <th>一丝</th>\n",
       "      <th>一分</th>\n",
       "      <th>一副</th>\n",
       "      <th>一同</th>\n",
       "      <th>一塌糊涂</th>\n",
       "      <th>...</th>\n",
       "      <th>零下</th>\n",
       "      <th>鞋里</th>\n",
       "      <th>预期</th>\n",
       "      <th>题材</th>\n",
       "      <th>飞船</th>\n",
       "      <th>首部</th>\n",
       "      <th>骄傲</th>\n",
       "      <th>高分</th>\n",
       "      <th>高空</th>\n",
       "      <th>鼓励</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 579 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bb  bgm  eason  frost  punk  一丝  一分  一副  一同  一塌糊涂 ...  零下  鞋里  预期  题材  飞船  \\\n",
       "0   0    0      0      0     0   0   0   1   0     0 ...   0   0   0   1   0   \n",
       "1   0    0      0      0     0   0   0   0   0     0 ...   0   0   0   0   0   \n",
       "2   0    0      0      0     0   0   0   0   0     0 ...   0   0   0   0   0   \n",
       "3   0    0      0      0     0   0   0   0   0     0 ...   0   0   0   0   0   \n",
       "4   0    0      0      0     0   0   0   0   0     0 ...   0   0   0   0   0   \n",
       "\n",
       "   首部  骄傲  高分  高空  鼓励  \n",
       "0   0   0   0   0   0  \n",
       "1   0   0   0   0   0  \n",
       "2   0   0   0   0   0  \n",
       "3   0   0   0   0   0  \n",
       "4   0   0   0   0   0  \n",
       "\n",
       "[5 rows x 579 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "term_matrix.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "nb = MultinomialNB()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import make_pipeline\n",
    "pipe = make_pipeline(vect, nb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('countvectorizer',\n",
       "  CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "          dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "          lowercase=True, max_df=0.8, max_features=None, min_df=3,\n",
       "          ngram_range=(1, 1), preprocessor=None,\n",
       "          stop_words=frozenset({'', '咦', '已经', '其余', '往', '除却', '兮', '从优', '▲', '孰料', '才', '移动', '到', '己', '出于', '曾', '趁着', '临', '而且', '得出', '时候', '大多', '［②ａ］', '运用', '等', '可见', '什麽', '只是', '其实', '［②Ｂ］', '任', '谨', '即若', '再次', '皆可', '大量', '联袂', '＞λ', '得到', '无', '背靠背', '周围', '正如', '极其', '不仅仅是', '毕竟', '一一', '窃',...面', '巴', '奇', '挨门挨户', '尽然', '不怕', '［④ｂ］', '７', '［④］', '却不', '况且', '纵', '较比', '将才', '居然', '『', '经常'}),\n",
       "          strip_accents=None, token_pattern='(?u)\\\\b[^\\\\d\\\\W]\\\\w+\\\\b',\n",
       "          tokenizer=None, vocabulary=None)),\n",
       " ('multinomialnb', MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True))]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe.steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.cross_validation import cross_val_score\n",
    "cross_val_score(pipe, X_train.cutted_comment, y_train, cv=5, scoring='accuracy').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('countvectorizer', CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=0.8, max_features=None, min_df=3,\n",
       "        ngram_range=(1, 1), preprocessor=None,\n",
       "        stop_words=...e, vocabulary=None)), ('multinomialnb', MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True))])"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe.fit(X_train.cutted_comment, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 1, ..., 0, 0, 1], dtype=int64)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe.predict(X_test.cutted_comment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = pipe.predict(X_test.cutted_comment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[4805,    0],\n",
       "       [   0, 2695]], dtype=int64)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "from snownlp import SnowNLP\n",
    "def get_sentiment(text):\n",
    "    return SnowNLP(text).sentiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_snownlp = X_test.comments.apply(get_sentiment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_snownlp_normalized = y_pred_snownlp.apply(lambda x: 1 if x>0.5 else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10747    1\n",
       "12573    1\n",
       "29676    1\n",
       "8856     1\n",
       "21098    0\n",
       "Name: comments, dtype: int64"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_snownlp_normalized[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.424"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.accuracy_score(y_test, y_pred_snownlp_normalized)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[16, 72],\n",
       "       [ 0, 37]], dtype=int64)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.confusion_matrix(y_test, y_pred_snownlp_normalized)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}