RNN_data_prep.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "paperIds = pd.read_csv('/home/lisi/impact-prediction/data/paperIds-1975-2005-2015-2.tsv', sep='\\t', names=['paperIds'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this file has a lot of sparse rows so we open it in this way\n",
    "with open('/home/lisi/impact-prediction/data/paperHistories-1975-2005-2015-2.tsv', 'r') as f:\n",
    "    lines = [row.split() for row in f.read().split(\"\\n\") if row]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "paperIds['citations'] = lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "respones = pd.read_csv('/home/lisi/impact-prediction/data/paperResponses-1975-2005-2015-2.tsv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "respones['paperIds'] = paperIds.paperIds.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "paperIds.citations = paperIds.citations.apply(lambda x: x[-5:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "paperIds.citations = paperIds.citations.apply(lambda x: np.nan if len(x) < 5 else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "paperIds = pd.merge(paperIds, respones, how='left', on='paperIds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "paperIds = paperIds.dropna().reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "ids = paperIds.paperIds.values\n",
    "c6 = paperIds.total_citations_in_0.values\n",
    "c7 = paperIds.total_citations_in_1.values\n",
    "c8 = paperIds.total_citations_in_2.values\n",
    "c9 = paperIds.total_citations_in_3.values\n",
    "c10 = paperIds.total_citations_in_4.values\n",
    "c11 = paperIds.total_citations_in_5.values\n",
    "c12 = paperIds.total_citations_in_6.values\n",
    "c13 = paperIds.total_citations_in_7.values\n",
    "c14 = paperIds.total_citations_in_8.values\n",
    "c15 = paperIds.total_citations_in_9.values\n",
    "paperIds = pd.DataFrame(paperIds[\"citations\"].to_list(), columns=['citations_{}'.format(j) for j in range(1,6)])\n",
    "paperIds['paperIds'] = ids\n",
    "paperIds['citations_6'] = c6\n",
    "paperIds['citations_7'] = c7\n",
    "paperIds['citations_8'] = c8\n",
    "paperIds['citations_9'] = c9\n",
    "paperIds['citations_10'] = c10\n",
    "paperIds['citations_11'] = c11\n",
    "paperIds['citations_12'] = c12\n",
    "paperIds['citations_13'] = c13\n",
    "paperIds['citations_14'] = c14\n",
    "paperIds['citations_15'] = c15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we read the citation graph and make a csv file\n",
    "with open('/home/lisi/impact-prediction/data/paperIdToPaperData-1975-2015.json', 'r',encoding='utf-8') as j:\n",
    "        d = j.read().replace('false', '0').replace('true', '1').splitlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# string to dictionary\n",
    "good = []\n",
    "bad = [] # there are some authors that have special characters that utf-8 or utf-16 cant solve\n",
    "for i in d:\n",
    "    try:\n",
    "        good.append(eval(i))\n",
    "    except:\n",
    "        bad.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# manually added the bad rows\n",
    "add_bad = {'id':['ca9ba78b3e268ec9b58ffa69524e6f35c0314fcc', \n",
    "                 '30af66277400354f88d0a5de0feaab9fb5278705', \n",
    "                '61afe58c68b91d20a65bb4166cfceb501745c134', \n",
    "                '6d46189d2040a01ccdb4236b0095361294b1a8bf'],\n",
    "          'year':[2007, 2007, 2006, 2004],\n",
    "          'venue':['','','','']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata = pd.DataFrame.from_dict(good, orient='columns') # list to dataframe\n",
    "metadata = metadata.drop(['citations-in-year', 'authors', 'is-survey'],axis=1)\n",
    "metadata = metadata.append(pd.DataFrame(add_bad), ignore_index=True) # add the bad rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata['venue'].replace('', np.nan, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata = metadata.dropna().reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata = pd.merge(metadata, paperIds, left_on=['id'], right_on=['paperIds'], how='right')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# metadata = metadata.drop(['id', 'paperIds'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "xtest = pd.read_csv('/home/lisi/Desktop/Laci/LaszloModel/test_set_ids.csv')\n",
    "xtrain = pd.read_csv('/home/lisi/Desktop/Laci/LaszloModel/train_set_ids.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = metadata[metadata.id.isin(list(xtest.id.values))].drop(['paperIds'],axis=1)\n",
    "X_train = metadata[metadata.id.isin(list(xtrain.id.values))].drop(['paperIds'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.to_csv('rnn_x_train.csv', index=False)\n",
    "X_test.to_csv('rnn_x_test.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>id</th>\n",
       "      <th>venue</th>\n",
       "      <th>citations_1</th>\n",
       "      <th>citations_2</th>\n",
       "      <th>citations_3</th>\n",
       "      <th>citations_4</th>\n",
       "      <th>citations_5</th>\n",
       "      <th>citations_6</th>\n",
       "      <th>citations_7</th>\n",
       "      <th>citations_8</th>\n",
       "      <th>citations_9</th>\n",
       "      <th>citations_10</th>\n",
       "      <th>citations_11</th>\n",
       "      <th>citations_12</th>\n",
       "      <th>citations_13</th>\n",
       "      <th>citations_14</th>\n",
       "      <th>citations_15</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1988.0</td>\n",
       "      <td>1b0d090a6518235977d0ecf532329fbdb9fafb56</td>\n",
       "      <td>compgeom</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>1997.0</td>\n",
       "      <td>dd50d87c777cecfbe0e2994262b6e08c94984d4f</td>\n",
       "      <td>orl</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>1996.0</td>\n",
       "      <td>1e70d2eb80eba7a3ec384f6d0f41612a10e7e38d</td>\n",
       "      <td>ht</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>2001.0</td>\n",
       "      <td>41c19582e260f268524b87b95b58b2b4d8712fee</td>\n",
       "      <td>cn</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "      <td>12</td>\n",
       "      <td>16</td>\n",
       "      <td>17</td>\n",
       "      <td>20</td>\n",
       "      <td>23</td>\n",
       "      <td>24</td>\n",
       "      <td>25</td>\n",
       "      <td>26</td>\n",
       "      <td>27</td>\n",
       "      <td>28</td>\n",
       "      <td>31</td>\n",
       "      <td>32</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>1997.0</td>\n",
       "      <td>df6ddb7456d80689970e78065684add1eb6bda80</td>\n",
       "      <td>jcmc</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>691475</th>\n",
       "      <td>1997.0</td>\n",
       "      <td>9c7025c8b0cf4bec8877e1b9cd9c03aa21eab7ab</td>\n",
       "      <td>sas</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>691503</th>\n",
       "      <td>1999.0</td>\n",
       "      <td>2da3596196ea3b1a932b02503d44d3f218b38c6e</td>\n",
       "      <td>icip</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>691538</th>\n",
       "      <td>1997.0</td>\n",
       "      <td>4cf4076601ee77c6ce00183e6806591cb759e1cd</td>\n",
       "      <td>arvlsi</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>691559</th>\n",
       "      <td>2000.0</td>\n",
       "      <td>dc7773082466cc2f2d11f69b8d9191df41ac244f</td>\n",
       "      <td>cvpr</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>691561</th>\n",
       "      <td>2001.0</td>\n",
       "      <td>aeb64cdbec6262fb0cf7106d1a6a013cf28388bd</td>\n",
       "      <td>interspeech</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>38483 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          year                                        id        venue  \\\n",
       "12      1988.0  1b0d090a6518235977d0ecf532329fbdb9fafb56     compgeom   \n",
       "39      1997.0  dd50d87c777cecfbe0e2994262b6e08c94984d4f          orl   \n",
       "40      1996.0  1e70d2eb80eba7a3ec384f6d0f41612a10e7e38d           ht   \n",
       "72      2001.0  41c19582e260f268524b87b95b58b2b4d8712fee           cn   \n",
       "86      1997.0  df6ddb7456d80689970e78065684add1eb6bda80         jcmc   \n",
       "...        ...                                       ...          ...   \n",
       "691475  1997.0  9c7025c8b0cf4bec8877e1b9cd9c03aa21eab7ab          sas   \n",
       "691503  1999.0  2da3596196ea3b1a932b02503d44d3f218b38c6e         icip   \n",
       "691538  1997.0  4cf4076601ee77c6ce00183e6806591cb759e1cd       arvlsi   \n",
       "691559  2000.0  dc7773082466cc2f2d11f69b8d9191df41ac244f         cvpr   \n",
       "691561  2001.0  aeb64cdbec6262fb0cf7106d1a6a013cf28388bd  interspeech   \n",
       "\n",
       "       citations_1 citations_2 citations_3 citations_4 citations_5  \\\n",
       "12               0           0           0           0           1   \n",
       "39               0           0           0           1           1   \n",
       "40               6           7           7           7           7   \n",
       "72               6          10          12          16          17   \n",
       "86               3           4           7           9           9   \n",
       "...            ...         ...         ...         ...         ...   \n",
       "691475           2           3           3           3           4   \n",
       "691503           0           0           1           2           2   \n",
       "691538           2           2           2           3           3   \n",
       "691559           0           0           0           2           2   \n",
       "691561           0           1           1           5           6   \n",
       "\n",
       "        citations_6  citations_7  citations_8  citations_9  citations_10  \\\n",
       "12                1            1            1            1             1   \n",
       "39                1            1            1            2             3   \n",
       "40                8            9            9            9             9   \n",
       "72               20           23           24           25            26   \n",
       "86               10           10           10           11            11   \n",
       "...             ...          ...          ...          ...           ...   \n",
       "691475            5            5            5            5             5   \n",
       "691503            3            4            4            4             4   \n",
       "691538            3            3            4            4             4   \n",
       "691559            2            2            2            2             2   \n",
       "691561            7            8            8           10            10   \n",
       "\n",
       "        citations_11  citations_12  citations_13  citations_14  citations_15  \n",
       "12                 1             1             1             1             1  \n",
       "39                 3             3             3             3             3  \n",
       "40                 9             9             9             9             9  \n",
       "72                27            28            31            32            33  \n",
       "86                11            11            11            12            14  \n",
       "...              ...           ...           ...           ...           ...  \n",
       "691475             5             5             5             5             5  \n",
       "691503             4             4             4             5             5  \n",
       "691538             4             5             5             5             5  \n",
       "691559             2             2             2             2             2  \n",
       "691561            10            13            14            15            15  \n",
       "\n",
       "[38483 rows x 18 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}