data-exploration.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Missing data GDSC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdsc_dr = pd.read_csv('data/GDSC/metrics.csv').rename(columns={'Cell_line':'CCL'})\n",
    "gdsc_ge = pd.read_csv('data/GDSC/ge.csv', sep='\\t').rename(columns={'Unnamed: 0':'CCL'}).transpose()\n",
    "new = pd.read_csv('data/GDSC/new.csv').rename(columns={'CELL_LINE_NAME':'CCL'})\n",
    "new['CCL'] = new['CCL'].str.replace('-', '').str.replace(' ','').str.upper()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "707 different CCLs in Pozdeyev's data\n"
     ]
    }
   ],
   "source": [
    "print(str(len(gdsc_dr['CCL'].unique())) + ' different CCLs in Pozdeyev\\'s data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "139 different Drugs in Pozdeyev's data\n"
     ]
    }
   ],
   "source": [
    "print(str(len(gdsc_dr['Drug_name'].unique())) + ' different Drugs in Pozdeyev\\'s data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "808 different CCLs in new data\n"
     ]
    }
   ],
   "source": [
    "print(str(len(new['CCL'].unique())) + ' different CCLs in new data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200 different Drugs in new data\n"
     ]
    }
   ],
   "source": [
    "print(str(len(new['DRUG_ID'].unique())) + ' different Drugs in new data')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deletes the symbols from CCL ID to normalize format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean(s):\n",
    "    new = []\n",
    "    for i in s:\n",
    "        new.append(re.sub(\"[^A-Za-z0-9]\", '', i).upper())\n",
    "    return new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "poz_r = list(gdsc_dr['CCL'].unique())\n",
    "cell_r = list(gdsc_ge.index.to_series().reset_index()[0])\n",
    "gdsc_r = list(pd.Series(new['CCL'].unique()))\n",
    "\n",
    "cell = clean(cell_r)\n",
    "poz = clean(poz_r)\n",
    "gdsc = clean(gdsc_r)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pozdeyev's CCLs missing from CellMinerCDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "miss1 = []\n",
    "for ccl in poz:\n",
    "    if ccl not in cell:\n",
    "        miss1.append(ccl)\n",
    "len(miss1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "New data CCLs missing from CellMinerCDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "miss2 = []\n",
    "for ccl in gdsc:\n",
    "    if ccl not in cell:\n",
    "        miss2.append(ccl)\n",
    "len(miss2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdsc2 = {k:re.sub(\"[^A-Za-z0-9]\", '', k).upper() for k in gdsc_r if re.sub(\"[^A-Za-z0-9]\", '', k).upper() not in miss2}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These rules are made by looking at the 'missing' CCLs that turned out to simply have different names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in miss2:\n",
    "    if i[0:3]=='NCI':\n",
    "        gdsc2[i] = i.lstrip('NCI')\n",
    "    elif i[0:3]=='SNU':\n",
    "        gdsc2[i] = 'NCI' + i\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdsc2['NTERA2CLD1'] = ('NTERASCLD1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "new['CCL'] = new['CCL'].replace(gdsc2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Same is done now for Pozdayev's data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "poz2 = {k:re.sub(\"[^A-Za-z0-9]\", '', k) for k in poz_r if re.sub(\"[^A-Za-z0-9]\", '', k).upper() not in miss1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "missv = ['7860', 'COLO320HSR','KNS81FD', 'EOL1CELL', 'NCIH510A','TT' , 'U266', 'U031']\n",
    "for i, ele in enumerate(miss1):\n",
    "    poz2[ele] = missv[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdsc_dr['CCL'] = gdsc_dr['CCL'].replace(poz2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdsc_dr.to_csv('data/Processed/gdsc_poz_dr.csv', index =False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdsc_ge = gdsc_ge.reset_index()\n",
    "gdsc_ge.columns = gdsc_ge.iloc[0]\n",
    "gdsc_ge = gdsc_ge[1:]\n",
    "cellrep = {k:v for k in cell v in }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CCL</th>\n",
       "      <th>A1BG</th>\n",
       "      <th>A1CF</th>\n",
       "      <th>A2M</th>\n",
       "      <th>A2ML1</th>\n",
       "      <th>A3GALT2</th>\n",
       "      <th>A3GALT2P</th>\n",
       "      <th>A4GALT</th>\n",
       "      <th>A4GNT</th>\n",
       "      <th>AAAS</th>\n",
       "      <th>...</th>\n",
       "      <th>ZW10</th>\n",
       "      <th>ZWILCH</th>\n",
       "      <th>ZWINT</th>\n",
       "      <th>ZXDB</th>\n",
       "      <th>ZXDC</th>\n",
       "      <th>ZYG11A</th>\n",
       "      <th>ZYG11B</th>\n",
       "      <th>ZYX</th>\n",
       "      <th>ZZEF1</th>\n",
       "      <th>ZZZ3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>201T</td>\n",
       "      <td>3.17242</td>\n",
       "      <td>2.93284</td>\n",
       "      <td>3.39045</td>\n",
       "      <td>2.71703</td>\n",
       "      <td>2.67498</td>\n",
       "      <td>2.67498</td>\n",
       "      <td>3.48303</td>\n",
       "      <td>3.1235</td>\n",
       "      <td>4.75917</td>\n",
       "      <td>...</td>\n",
       "      <td>7.26434</td>\n",
       "      <td>6.20224</td>\n",
       "      <td>6.28252</td>\n",
       "      <td>4.04948</td>\n",
       "      <td>4.93531</td>\n",
       "      <td>2.74394</td>\n",
       "      <td>4.77964</td>\n",
       "      <td>4.70476</td>\n",
       "      <td>4.6623</td>\n",
       "      <td>8.80625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>22RV1</td>\n",
       "      <td>3.53794</td>\n",
       "      <td>6.36465</td>\n",
       "      <td>5.33244</td>\n",
       "      <td>2.92303</td>\n",
       "      <td>2.81538</td>\n",
       "      <td>2.81538</td>\n",
       "      <td>3.24112</td>\n",
       "      <td>3.26263</td>\n",
       "      <td>4.72216</td>\n",
       "      <td>...</td>\n",
       "      <td>8.37838</td>\n",
       "      <td>7.52632</td>\n",
       "      <td>9.27417</td>\n",
       "      <td>3.78576</td>\n",
       "      <td>5.20193</td>\n",
       "      <td>2.74257</td>\n",
       "      <td>4.63502</td>\n",
       "      <td>3.94019</td>\n",
       "      <td>4.65866</td>\n",
       "      <td>8.08867</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23132-87</td>\n",
       "      <td>3.37095</td>\n",
       "      <td>6.28488</td>\n",
       "      <td>3.48568</td>\n",
       "      <td>2.83156</td>\n",
       "      <td>2.91337</td>\n",
       "      <td>2.91337</td>\n",
       "      <td>3.31303</td>\n",
       "      <td>3.09653</td>\n",
       "      <td>4.87362</td>\n",
       "      <td>...</td>\n",
       "      <td>8.58629</td>\n",
       "      <td>6.52671</td>\n",
       "      <td>8.02195</td>\n",
       "      <td>3.52058</td>\n",
       "      <td>4.95737</td>\n",
       "      <td>2.63693</td>\n",
       "      <td>4.35012</td>\n",
       "      <td>4.39581</td>\n",
       "      <td>4.59858</td>\n",
       "      <td>7.95115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>42-MG-BA</td>\n",
       "      <td>6.01674</td>\n",
       "      <td>3.19189</td>\n",
       "      <td>3.24948</td>\n",
       "      <td>2.80896</td>\n",
       "      <td>2.6262</td>\n",
       "      <td>2.6262</td>\n",
       "      <td>3.06339</td>\n",
       "      <td>3.35685</td>\n",
       "      <td>4.91996</td>\n",
       "      <td>...</td>\n",
       "      <td>8.57463</td>\n",
       "      <td>7.27485</td>\n",
       "      <td>7.68393</td>\n",
       "      <td>3.47306</td>\n",
       "      <td>5.23711</td>\n",
       "      <td>2.75974</td>\n",
       "      <td>5.51799</td>\n",
       "      <td>4.39196</td>\n",
       "      <td>4.15771</td>\n",
       "      <td>8.94488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>451Lu</td>\n",
       "      <td>5.72524</td>\n",
       "      <td>2.97195</td>\n",
       "      <td>10.7685</td>\n",
       "      <td>2.92855</td>\n",
       "      <td>2.75028</td>\n",
       "      <td>2.75028</td>\n",
       "      <td>3.36712</td>\n",
       "      <td>3.16265</td>\n",
       "      <td>5.14418</td>\n",
       "      <td>...</td>\n",
       "      <td>7.80586</td>\n",
       "      <td>7.94224</td>\n",
       "      <td>9.38534</td>\n",
       "      <td>3.29279</td>\n",
       "      <td>5.53094</td>\n",
       "      <td>2.58216</td>\n",
       "      <td>5.08715</td>\n",
       "      <td>5.19761</td>\n",
       "      <td>4.92217</td>\n",
       "      <td>7.4706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1076</th>\n",
       "      <td>YH-13</td>\n",
       "      <td>5.16786</td>\n",
       "      <td>3.07284</td>\n",
       "      <td>3.04907</td>\n",
       "      <td>3.09807</td>\n",
       "      <td>3.02972</td>\n",
       "      <td>3.02972</td>\n",
       "      <td>2.99009</td>\n",
       "      <td>3.31857</td>\n",
       "      <td>4.56187</td>\n",
       "      <td>...</td>\n",
       "      <td>8.47652</td>\n",
       "      <td>8.04813</td>\n",
       "      <td>8.2353</td>\n",
       "      <td>3.3059</td>\n",
       "      <td>4.74561</td>\n",
       "      <td>2.57763</td>\n",
       "      <td>5.21584</td>\n",
       "      <td>7.00354</td>\n",
       "      <td>4.20382</td>\n",
       "      <td>7.91928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1077</th>\n",
       "      <td>YKG-1</td>\n",
       "      <td>4.71162</td>\n",
       "      <td>3.06794</td>\n",
       "      <td>3.30693</td>\n",
       "      <td>3.03796</td>\n",
       "      <td>2.71344</td>\n",
       "      <td>2.71344</td>\n",
       "      <td>3.1544</td>\n",
       "      <td>2.98892</td>\n",
       "      <td>4.58681</td>\n",
       "      <td>...</td>\n",
       "      <td>6.98386</td>\n",
       "      <td>7.97826</td>\n",
       "      <td>9.15039</td>\n",
       "      <td>3.97241</td>\n",
       "      <td>5.16608</td>\n",
       "      <td>2.81345</td>\n",
       "      <td>5.11719</td>\n",
       "      <td>5.08564</td>\n",
       "      <td>4.91817</td>\n",
       "      <td>8.20834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1078</th>\n",
       "      <td>YMB-1-E</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1079</th>\n",
       "      <td>YT</td>\n",
       "      <td>3.14359</td>\n",
       "      <td>3.77979</td>\n",
       "      <td>5.09516</td>\n",
       "      <td>2.87933</td>\n",
       "      <td>2.74909</td>\n",
       "      <td>2.74909</td>\n",
       "      <td>3.18623</td>\n",
       "      <td>3.35959</td>\n",
       "      <td>4.51172</td>\n",
       "      <td>...</td>\n",
       "      <td>8.46731</td>\n",
       "      <td>8.19677</td>\n",
       "      <td>9.20568</td>\n",
       "      <td>3.51532</td>\n",
       "      <td>5.15234</td>\n",
       "      <td>2.47275</td>\n",
       "      <td>5.04678</td>\n",
       "      <td>6.01671</td>\n",
       "      <td>5.16818</td>\n",
       "      <td>7.06395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1080</th>\n",
       "      <td>ZR-75-30</td>\n",
       "      <td>5.59418</td>\n",
       "      <td>3.03213</td>\n",
       "      <td>3.5622</td>\n",
       "      <td>3.10855</td>\n",
       "      <td>2.76767</td>\n",
       "      <td>2.76767</td>\n",
       "      <td>3.61138</td>\n",
       "      <td>3.24774</td>\n",
       "      <td>4.54791</td>\n",
       "      <td>...</td>\n",
       "      <td>7.72574</td>\n",
       "      <td>7.39858</td>\n",
       "      <td>8.78124</td>\n",
       "      <td>3.57496</td>\n",
       "      <td>5.30155</td>\n",
       "      <td>3.04522</td>\n",
       "      <td>4.95208</td>\n",
       "      <td>4.60416</td>\n",
       "      <td>4.473</td>\n",
       "      <td>7.6756</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1080 rows × 19563 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "0          CCL     A1BG     A1CF      A2M    A2ML1  A3GALT2 A3GALT2P   A4GALT  \\\n",
       "1         201T  3.17242  2.93284  3.39045  2.71703  2.67498  2.67498  3.48303   \n",
       "2        22RV1  3.53794  6.36465  5.33244  2.92303  2.81538  2.81538  3.24112   \n",
       "3     23132-87  3.37095  6.28488  3.48568  2.83156  2.91337  2.91337  3.31303   \n",
       "4     42-MG-BA  6.01674  3.19189  3.24948  2.80896   2.6262   2.6262  3.06339   \n",
       "5        451Lu  5.72524  2.97195  10.7685  2.92855  2.75028  2.75028  3.36712   \n",
       "...        ...      ...      ...      ...      ...      ...      ...      ...   \n",
       "1076     YH-13  5.16786  3.07284  3.04907  3.09807  3.02972  3.02972  2.99009   \n",
       "1077     YKG-1  4.71162  3.06794  3.30693  3.03796  2.71344  2.71344   3.1544   \n",
       "1078   YMB-1-E      NaN      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "1079        YT  3.14359  3.77979  5.09516  2.87933  2.74909  2.74909  3.18623   \n",
       "1080  ZR-75-30  5.59418  3.03213   3.5622  3.10855  2.76767  2.76767  3.61138   \n",
       "\n",
       "0       A4GNT     AAAS  ...     ZW10   ZWILCH    ZWINT     ZXDB     ZXDC  \\\n",
       "1      3.1235  4.75917  ...  7.26434  6.20224  6.28252  4.04948  4.93531   \n",
       "2     3.26263  4.72216  ...  8.37838  7.52632  9.27417  3.78576  5.20193   \n",
       "3     3.09653  4.87362  ...  8.58629  6.52671  8.02195  3.52058  4.95737   \n",
       "4     3.35685  4.91996  ...  8.57463  7.27485  7.68393  3.47306  5.23711   \n",
       "5     3.16265  5.14418  ...  7.80586  7.94224  9.38534  3.29279  5.53094   \n",
       "...       ...      ...  ...      ...      ...      ...      ...      ...   \n",
       "1076  3.31857  4.56187  ...  8.47652  8.04813   8.2353   3.3059  4.74561   \n",
       "1077  2.98892  4.58681  ...  6.98386  7.97826  9.15039  3.97241  5.16608   \n",
       "1078      NaN      NaN  ...      NaN      NaN      NaN      NaN      NaN   \n",
       "1079  3.35959  4.51172  ...  8.46731  8.19677  9.20568  3.51532  5.15234   \n",
       "1080  3.24774  4.54791  ...  7.72574  7.39858  8.78124  3.57496  5.30155   \n",
       "\n",
       "0      ZYG11A   ZYG11B      ZYX    ZZEF1     ZZZ3  \n",
       "1     2.74394  4.77964  4.70476   4.6623  8.80625  \n",
       "2     2.74257  4.63502  3.94019  4.65866  8.08867  \n",
       "3     2.63693  4.35012  4.39581  4.59858  7.95115  \n",
       "4     2.75974  5.51799  4.39196  4.15771  8.94488  \n",
       "5     2.58216  5.08715  5.19761  4.92217   7.4706  \n",
       "...       ...      ...      ...      ...      ...  \n",
       "1076  2.57763  5.21584  7.00354  4.20382  7.91928  \n",
       "1077  2.81345  5.11719  5.08564  4.91817  8.20834  \n",
       "1078      NaN      NaN      NaN      NaN      NaN  \n",
       "1079  2.47275  5.04678  6.01671  5.16818  7.06395  \n",
       "1080  3.04522  4.95208  4.60416    4.473   7.6756  \n",
       "\n",
       "[1080 rows x 19563 columns]"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdsc_ge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdsc_ge.to_csv('data/Processed/gdsc_cell_ge.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "707"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Missing CCLE data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "ccle_ge = pd.read_csv('data/CCLE/ge.csv', sep='\\t').rename(columns={'Unnamed: 0':'CCL'}).set_index('CCL').transpose()\n",
    "ccle_dr = pd.read_csv('data/CCLE/metrics.csv').rename(columns={'Cell_line':'CCL'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "poz = list(ccle_dr['CCL'].unique())\n",
    "cell = list(ccle_ge.index.to_series().str.upper().reset_index()[0])\n",
    "\n",
    "ncell = []\n",
    "for i in cell:\n",
    "    ncell.append(re.sub(\"[^A-Za-z0-9]\", '', i))\n",
    "cell = ncell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "504"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(poz)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['BGC823',\n",
       " 'GLC82',\n",
       " 'KP1N',\n",
       " 'KP1NL',\n",
       " 'M059J',\n",
       " 'MB157',\n",
       " 'OVCAR3',\n",
       " 'SF8657',\n",
       " 'SNUC2B']"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "miss = []\n",
    "for ccl in poz:\n",
    "    if ccl not in cell2:\n",
    "        miss.append(ccl)\n",
    "miss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now I substitute by the correct names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### I look for the following 9 cell lines [here](https://www.expasy.org/) and find what the problem with them is."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "BGC823, GLC82, KP1N and KP1NL are problematic (shown to be derivatives of HeLa and PANC-1) cell lines so I will drop them.\n",
    "\n",
    "MB157 and OVCAR3 are under different names, namely MDAMB157, NIH:OVCAR3\n",
    "\n",
    "_M059J_ wasn't found on cellminercdb, there is a M059K but idk if it's the same, maybe download M059J from [here](https://web.expasy.org/cellosaurus/CVCL_0400). It is on GDSC's cellminercdb dataset.\n",
    "\n",
    "_SF8657_ wasn't found,  can be downloaded from [here] (https://web.expasy.org/cellosaurus/CVCL_K085)\n",
    "\n",
    "_SNUC2B_ can't be found on this cellminercdb dataset but it is on GDSC's, here you find it under the alias SNUC2A according [to] (https://web.expasy.org/cellosaurus/CVCL_1710)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = ccle_dr[~ccle_dr['CCL'].isin(['BGC823', 'GLC82', 'KP1N', 'KP1NL', 'M059J', 'SF8657', 'SNUC2B'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MB157\n",
      "OVCAR3\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Missing CTRP data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 459,
   "metadata": {},
   "outputs": [],
   "source": [
    "ctrp_ge = pd.read_csv('data/CTRP/cell_ge.csv', sep='\\t').rename(columns={'Unnamed: 0':'CCL'}).set_index('CCL').transpose()\n",
    "ctrp_dr = pd.read_csv('data/CTRP/metrics.csv').rename(columns={'Cell_line':'CCL'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 451,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "poz = list(ctrp_dr['CCL'].unique())\n",
    "cell3 = list(ctrp_ge.index.to_series().str.upper().reset_index()[0])\n",
    "\n",
    "ncell = []\n",
    "for i in cell3:\n",
    "    ncell.append(re.sub(\"[^A-Za-z0-9]\", '', i))\n",
    "cell3 = ncell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 453,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "66"
      ]
     },
     "execution_count": 453,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "miss = []\n",
    "for ccl in poz:\n",
    "    if ccl not in cell3:\n",
    "        miss.append(ccl)\n",
    "len(miss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 414,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['2004',\n",
       " 'ASKA',\n",
       " 'BT112',\n",
       " 'BT131',\n",
       " 'BT139',\n",
       " 'BT145',\n",
       " 'BT147',\n",
       " 'BT159',\n",
       " 'BT16',\n",
       " 'BT164',\n",
       " 'BT172',\n",
       " 'BT179',\n",
       " 'BT216',\n",
       " 'BT224',\n",
       " 'BT228',\n",
       " 'BT231',\n",
       " 'BT232',\n",
       " 'BT239',\n",
       " 'BT245',\n",
       " 'BT248',\n",
       " 'BT271',\n",
       " 'BT286',\n",
       " 'BT320',\n",
       " 'BT328',\n",
       " 'BT330',\n",
       " 'BT333',\n",
       " 'BT340',\n",
       " 'BT359',\n",
       " 'BT416',\n",
       " 'BT422',\n",
       " 'BT428',\n",
       " 'BT440',\n",
       " 'BT444',\n",
       " 'BT482',\n",
       " 'BT498',\n",
       " 'BT504',\n",
       " 'H292',\n",
       " 'HS578BST',\n",
       " 'HS888LU',\n",
       " 'HSTS',\n",
       " 'KRIJ',\n",
       " 'PSTS',\n",
       " 'SW982',\n",
       " 'SYO1',\n",
       " 'TM8716',\n",
       " 'TTTHYROID',\n",
       " 'UCH1',\n",
       " 'YAMATO',\n",
       " 'BT187']"
      ]
     },
     "execution_count": 414,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "miss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 415,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CHP126',\n",
       " 'CHP212',\n",
       " 'IMR32',\n",
       " 'KELLY',\n",
       " 'KPNSI9S',\n",
       " 'KPNYN',\n",
       " 'MHHNB11',\n",
       " 'NB1',\n",
       " 'NH6',\n",
       " 'SIMA',\n",
       " 'SKNAS',\n",
       " 'SKNBE2',\n",
       " 'SKNDZ',\n",
       " 'SKNFI',\n",
       " 'SKNSH',\n",
       " 'HUCCT1',\n",
       " 'HUH28',\n",
       " 'SNU1079',\n",
       " 'SNU1196',\n",
       " 'SNU245',\n",
       " 'SNU308',\n",
       " 'SNU478',\n",
       " 'SNU869',\n",
       " 'A673',\n",
       " 'CADOES1',\n",
       " 'CAL78',\n",
       " 'G292CLONEA141B1',\n",
       " 'HOS',\n",
       " 'HS822T',\n",
       " 'HS888T',\n",
       " 'MG63',\n",
       " 'MHHES1',\n",
       " 'RDES',\n",
       " 'SAOS2',\n",
       " 'SJSA1',\n",
       " 'SKES1',\n",
       " 'SKNMC',\n",
       " 'SW1353',\n",
       " 'TC71',\n",
       " 'U2OS',\n",
       " 'AU565',\n",
       " 'BT20',\n",
       " 'BT474',\n",
       " 'BT549',\n",
       " 'CAL120',\n",
       " 'CAL148',\n",
       " 'CAL51',\n",
       " 'CAL851',\n",
       " 'CAMA1',\n",
       " 'DU4475',\n",
       " 'EFM192A',\n",
       " 'HCC1143',\n",
       " 'HCC1395',\n",
       " 'HCC1419',\n",
       " 'HCC1428',\n",
       " 'HCC1500',\n",
       " 'HCC1569',\n",
       " 'HCC1806',\n",
       " 'HCC1937',\n",
       " 'HCC1954',\n",
       " 'HCC202',\n",
       " 'HCC2218',\n",
       " 'HCC38',\n",
       " 'HDQP1',\n",
       " 'HMC18',\n",
       " 'HS578T',\n",
       " 'JIMT1',\n",
       " 'KPL1',\n",
       " 'MCF7',\n",
       " 'MDAMB157',\n",
       " 'MDAMB231',\n",
       " 'MDAMB361',\n",
       " 'MDAMB415',\n",
       " 'MDAMB453',\n",
       " 'MDAMB468',\n",
       " 'SKBR3',\n",
       " 'T47D',\n",
       " 'UACC812',\n",
       " 'ZR751',\n",
       " 'ZR7530',\n",
       " '42MGBA',\n",
       " '8MGBA',\n",
       " 'A172',\n",
       " 'AM38',\n",
       " 'BECKER',\n",
       " 'CAS1',\n",
       " 'CCFSTTG1',\n",
       " 'D283MED',\n",
       " 'DAOY',\n",
       " 'DBTRG05MG',\n",
       " 'DKMG',\n",
       " 'GAMG',\n",
       " 'GB1',\n",
       " 'GI1',\n",
       " 'GMS10',\n",
       " 'GOS3',\n",
       " 'HS683',\n",
       " 'KALS1',\n",
       " 'KG1C',\n",
       " 'KNS42',\n",
       " 'KNS60',\n",
       " 'KNS81',\n",
       " 'LN18',\n",
       " 'LN229',\n",
       " 'M059K',\n",
       " 'NMCG1',\n",
       " 'ONS76',\n",
       " 'SF126',\n",
       " 'SF295',\n",
       " 'SNU1105',\n",
       " 'SNU201',\n",
       " 'SNU466',\n",
       " 'SNU489',\n",
       " 'SNU626',\n",
       " 'SNU738',\n",
       " 'SW1088',\n",
       " 'SW1783',\n",
       " 'T98G',\n",
       " 'TM31',\n",
       " 'U118MG',\n",
       " 'U251MG',\n",
       " 'U87MG',\n",
       " 'YH13',\n",
       " 'YKG1',\n",
       " 'AN3CA',\n",
       " 'COLO684',\n",
       " 'EFE184',\n",
       " 'EN',\n",
       " 'ESS1',\n",
       " 'HEC108',\n",
       " 'HEC151',\n",
       " 'HEC1A',\n",
       " 'HEC1B',\n",
       " 'HEC251',\n",
       " 'HEC265',\n",
       " 'HEC50B',\n",
       " 'HEC59',\n",
       " 'HEC6',\n",
       " 'ISHIKAWAHERAKLIO02ER',\n",
       " 'JHUEM1',\n",
       " 'JHUEM2',\n",
       " 'JHUEM3',\n",
       " 'KLE',\n",
       " 'MFE296',\n",
       " 'MFE319',\n",
       " 'RL952',\n",
       " 'SNGM',\n",
       " 'SNU1077',\n",
       " 'SNU685',\n",
       " '697',\n",
       " 'A3KAW',\n",
       " 'A4FUK',\n",
       " 'ALLSIL',\n",
       " 'AML193',\n",
       " 'AMO1',\n",
       " 'BCP1',\n",
       " 'BDCM',\n",
       " 'BL41',\n",
       " 'BL70',\n",
       " 'BV173',\n",
       " 'CA46',\n",
       " 'CI1',\n",
       " 'CMK',\n",
       " 'CMLT1',\n",
       " 'DAUDI',\n",
       " 'DB',\n",
       " 'DEL',\n",
       " 'DND41',\n",
       " 'DOHH2',\n",
       " 'EB1',\n",
       " 'EHEB',\n",
       " 'EJM',\n",
       " 'EM2',\n",
       " 'EOL1',\n",
       " 'F36P',\n",
       " 'GA10',\n",
       " 'GDM1',\n",
       " 'GRANTA519',\n",
       " 'HDLM2',\n",
       " 'HDMYZ',\n",
       " 'HEL',\n",
       " 'HEL9217',\n",
       " 'HH',\n",
       " 'HL60',\n",
       " 'HPBALL',\n",
       " 'HS611T',\n",
       " 'HT',\n",
       " 'HUNS1',\n",
       " 'HUT78',\n",
       " 'JEKO1',\n",
       " 'JJN3',\n",
       " 'JK1',\n",
       " 'JM1',\n",
       " 'JURKAT',\n",
       " 'JURLMK1',\n",
       " 'JVM2',\n",
       " 'JVM3',\n",
       " 'K562',\n",
       " 'KARPAS299',\n",
       " 'KARPAS422',\n",
       " 'KARPAS620',\n",
       " 'KASUMI1',\n",
       " 'KASUMI2',\n",
       " 'KCL22',\n",
       " 'KE37',\n",
       " 'KE97',\n",
       " 'KHM1B',\n",
       " 'KIJK',\n",
       " 'KMH2',\n",
       " 'KMM1',\n",
       " 'KMS11',\n",
       " 'KMS12BM',\n",
       " 'KMS18',\n",
       " 'KMS20',\n",
       " 'KMS21BM',\n",
       " 'KMS26',\n",
       " 'KMS27',\n",
       " 'KMS28BM',\n",
       " 'KMS34',\n",
       " 'KO52',\n",
       " 'KU812',\n",
       " 'KYO1',\n",
       " 'L1236',\n",
       " 'L363',\n",
       " 'L428',\n",
       " 'L540',\n",
       " 'LAMA84',\n",
       " 'LP1',\n",
       " 'M07E',\n",
       " 'MC116',\n",
       " 'ME1',\n",
       " 'MEC1',\n",
       " 'MHHCALL3',\n",
       " 'MHHCALL4',\n",
       " 'MINO',\n",
       " 'MJ',\n",
       " 'MM1S',\n",
       " 'MOLM13',\n",
       " 'MOLM16',\n",
       " 'MOLM6',\n",
       " 'MOLP2',\n",
       " 'MOLP8',\n",
       " 'MOLT13',\n",
       " 'MOLT16',\n",
       " 'MONOMAC1',\n",
       " 'MONOMAC6',\n",
       " 'MOTN1',\n",
       " 'MV411',\n",
       " 'NALM6',\n",
       " 'NAMALWA',\n",
       " 'NB4',\n",
       " 'NCIH929',\n",
       " 'NCO2',\n",
       " 'NOMO1',\n",
       " 'NUDHL1',\n",
       " 'NUDUL1',\n",
       " 'OCIAML2',\n",
       " 'OCIAML3',\n",
       " 'OCIAML5',\n",
       " 'OCILY10',\n",
       " 'OCILY19',\n",
       " 'OCILY3',\n",
       " 'OCIM1',\n",
       " 'OPM2',\n",
       " 'P12ICHIKAWA',\n",
       " 'P31FUJ',\n",
       " 'P3HR1',\n",
       " 'PEER',\n",
       " 'PF382',\n",
       " 'PFEIFFER',\n",
       " 'PL21',\n",
       " 'RAJI',\n",
       " 'RCHACV',\n",
       " 'REC1',\n",
       " 'REH',\n",
       " 'RI1',\n",
       " 'RL',\n",
       " 'RPMI8226',\n",
       " 'RPMI8402',\n",
       " 'RS411',\n",
       " 'SEM',\n",
       " 'SET2',\n",
       " 'SIGM5',\n",
       " 'SKM1',\n",
       " 'SKMM2',\n",
       " 'SR786',\n",
       " 'ST486',\n",
       " 'SUDHL1',\n",
       " 'SUDHL10',\n",
       " 'SUDHL4',\n",
       " 'SUDHL6',\n",
       " 'SUDHL8',\n",
       " 'SUPB15',\n",
       " 'SUPHD1',\n",
       " 'SUPM2',\n",
       " 'SUPT1',\n",
       " 'SUPT11',\n",
       " 'TALL1',\n",
       " 'TF1',\n",
       " 'THP1',\n",
       " 'TOLEDO',\n",
       " 'U266B1',\n",
       " 'U937',\n",
       " 'WSUDLCL2',\n",
       " '769P',\n",
       " '786O',\n",
       " 'A498',\n",
       " 'ACHN',\n",
       " 'BFTC909',\n",
       " 'CAKI1',\n",
       " 'CAKI2',\n",
       " 'CAL54',\n",
       " 'KMRC1',\n",
       " 'KMRC2',\n",
       " 'KMRC20',\n",
       " 'KMRC3',\n",
       " 'OSRC2',\n",
       " 'RCC10RGB',\n",
       " 'SNU1272',\n",
       " 'TUHR10TKB',\n",
       " 'TUHR14TKB',\n",
       " 'TUHR4TKB',\n",
       " 'VMRCRCW',\n",
       " 'VMRCRCZ',\n",
       " 'C2BBE1',\n",
       " 'CCK81',\n",
       " 'CL11',\n",
       " 'CL34',\n",
       " 'COLO201',\n",
       " 'COLO205',\n",
       " 'COLO320',\n",
       " 'COLO678',\n",
       " 'CW2',\n",
       " 'DLD1',\n",
       " 'GP2D',\n",
       " 'HCC56',\n",
       " 'HCT116',\n",
       " 'HCT15',\n",
       " 'HCT8',\n",
       " 'HT115',\n",
       " 'HT29',\n",
       " 'HT55',\n",
       " 'KM12',\n",
       " 'LOVO',\n",
       " 'LS1034',\n",
       " 'LS123',\n",
       " 'LS180',\n",
       " 'LS411N',\n",
       " 'LS513',\n",
       " 'MDST8',\n",
       " 'NCIH508',\n",
       " 'NCIH716',\n",
       " 'NCIH747',\n",
       " 'OUMS23',\n",
       " 'RCM1',\n",
       " 'RKO',\n",
       " 'SKCO1',\n",
       " 'SNU1033',\n",
       " 'SNU175',\n",
       " 'SNU407',\n",
       " 'SNU503',\n",
       " 'SNU61',\n",
       " 'SNU81',\n",
       " 'SNUC1',\n",
       " 'SNUC2A',\n",
       " 'SNUC4',\n",
       " 'SNUC5',\n",
       " 'SW1116',\n",
       " 'SW1417',\n",
       " 'SW1463',\n",
       " 'SW48',\n",
       " 'SW620',\n",
       " 'SW948',\n",
       " 'T84',\n",
       " 'HEP3B217',\n",
       " 'HEPG2',\n",
       " 'HLF',\n",
       " 'HUH1',\n",
       " 'HUH6',\n",
       " 'JHH1',\n",
       " 'JHH2',\n",
       " 'JHH4',\n",
       " 'JHH5',\n",
       " 'JHH6',\n",
       " 'LI7',\n",
       " 'PLCPRF5',\n",
       " 'SKHEP1',\n",
       " 'SNU182',\n",
       " 'SNU387',\n",
       " 'SNU398',\n",
       " 'SNU423',\n",
       " 'SNU449',\n",
       " 'SNU475',\n",
       " 'SNU761',\n",
       " 'SNU878',\n",
       " 'SNU886',\n",
       " 'A549',\n",
       " 'BEN',\n",
       " 'CAL12T',\n",
       " 'CALU1',\n",
       " 'CALU3',\n",
       " 'CALU6',\n",
       " 'CHAGOK1',\n",
       " 'COLO668',\n",
       " 'CORL105',\n",
       " 'CORL279',\n",
       " 'CORL47',\n",
       " 'CORL51',\n",
       " 'CORL88',\n",
       " 'CORL95',\n",
       " 'DMS114',\n",
       " 'DMS273',\n",
       " 'DMS454',\n",
       " 'DMS53',\n",
       " 'DMS79',\n",
       " 'DV90',\n",
       " 'EBC1',\n",
       " 'EPLC272H',\n",
       " 'HARA',\n",
       " 'HCC1171',\n",
       " 'HCC1195',\n",
       " 'HCC1438',\n",
       " 'HCC15',\n",
       " 'HCC1588',\n",
       " 'HCC1833',\n",
       " 'HCC2108',\n",
       " 'HCC2279',\n",
       " 'HCC2935',\n",
       " 'HCC33',\n",
       " 'HCC366',\n",
       " 'HCC4006',\n",
       " 'HCC44',\n",
       " 'HCC78',\n",
       " 'HCC827',\n",
       " 'HCC95',\n",
       " 'IALM',\n",
       " 'KNS62',\n",
       " 'LC1F',\n",
       " 'LC1SQSF',\n",
       " 'LCLC103H',\n",
       " 'LCLC97TM1',\n",
       " 'LK2',\n",
       " 'LOUNH91',\n",
       " 'LU65',\n",
       " 'LU99',\n",
       " 'LUDLU1',\n",
       " 'LXF289',\n",
       " 'MORCPR',\n",
       " 'NCIH1092',\n",
       " 'NCIH1105',\n",
       " 'NCIH1155',\n",
       " 'NCIH1184',\n",
       " 'NCIH1299',\n",
       " 'NCIH1339',\n",
       " 'NCIH1341',\n",
       " 'NCIH1355',\n",
       " 'NCIH1373',\n",
       " 'NCIH1385',\n",
       " 'NCIH1395',\n",
       " 'NCIH1435',\n",
       " 'NCIH146',\n",
       " 'NCIH1563',\n",
       " 'NCIH1568',\n",
       " 'NCIH1573',\n",
       " 'NCIH1581',\n",
       " 'NCIH1623',\n",
       " 'NCIH1650',\n",
       " 'NCIH1651',\n",
       " 'NCIH1666',\n",
       " 'NCIH1693',\n",
       " 'NCIH1694',\n",
       " 'NCIH1703',\n",
       " 'NCIH1734',\n",
       " 'NCIH1755',\n",
       " 'NCIH1792',\n",
       " 'NCIH1793',\n",
       " 'NCIH1838',\n",
       " 'NCIH1869',\n",
       " 'NCIH1915',\n",
       " 'NCIH1944',\n",
       " 'NCIH196',\n",
       " 'NCIH1975',\n",
       " 'NCIH2009',\n",
       " 'NCIH2023',\n",
       " 'NCIH2029',\n",
       " 'NCIH2030',\n",
       " 'NCIH2081',\n",
       " 'NCIH2085',\n",
       " 'NCIH2087',\n",
       " 'NCIH209',\n",
       " 'NCIH2106',\n",
       " 'NCIH211',\n",
       " 'NCIH2110',\n",
       " 'NCIH2122',\n",
       " 'NCIH2126',\n",
       " 'NCIH2141',\n",
       " 'NCIH2170',\n",
       " 'NCIH2171',\n",
       " 'NCIH2172',\n",
       " 'NCIH2196',\n",
       " 'NCIH2228',\n",
       " 'NCIH226',\n",
       " 'NCIH2286',\n",
       " 'NCIH2291',\n",
       " 'NCIH23',\n",
       " 'NCIH2342',\n",
       " 'NCIH2405',\n",
       " 'NCIH2444',\n",
       " 'NCIH292',\n",
       " 'NCIH322',\n",
       " 'NCIH358',\n",
       " 'NCIH441',\n",
       " 'NCIH446',\n",
       " 'NCIH460',\n",
       " 'NCIH510',\n",
       " 'NCIH520',\n",
       " 'NCIH522',\n",
       " 'NCIH524',\n",
       " 'NCIH526',\n",
       " 'NCIH647',\n",
       " 'NCIH650',\n",
       " 'NCIH661',\n",
       " 'NCIH69',\n",
       " 'NCIH727',\n",
       " 'NCIH810',\n",
       " 'NCIH838',\n",
       " 'NCIH841',\n",
       " 'NCIH889',\n",
       " 'PC14',\n",
       " 'RERFLCAD1',\n",
       " 'RERFLCAD2',\n",
       " 'RERFLCAI',\n",
       " 'RERFLCMS',\n",
       " 'SBC5',\n",
       " 'SCLC21H',\n",
       " 'SHP77',\n",
       " 'SKLU1',\n",
       " 'SKMES1',\n",
       " 'SQ1',\n",
       " 'SW1271',\n",
       " 'SW1573',\n",
       " 'T3M10',\n",
       " 'VMRCLCD',\n",
       " 'COLO680N',\n",
       " 'ECGI10',\n",
       " 'KYSE140',\n",
       " 'KYSE150',\n",
       " 'KYSE180',\n",
       " 'KYSE30',\n",
       " 'KYSE410',\n",
       " 'KYSE450',\n",
       " 'KYSE510',\n",
       " 'KYSE520',\n",
       " 'KYSE70',\n",
       " 'OE19',\n",
       " 'OE33',\n",
       " 'TE1',\n",
       " 'TE10',\n",
       " 'TE11',\n",
       " 'TE14',\n",
       " 'TE15',\n",
       " 'TE4',\n",
       " 'TE5',\n",
       " 'TE6',\n",
       " 'TE8',\n",
       " 'TE9',\n",
       " 'A2780',\n",
       " 'CAOV3',\n",
       " 'COV362',\n",
       " 'COV434',\n",
       " 'COV644',\n",
       " 'EFO21',\n",
       " 'EFO27',\n",
       " 'ES2',\n",
       " 'FUOV1',\n",
       " 'HEYA8',\n",
       " 'IGROV1',\n",
       " 'JHOC5',\n",
       " 'JHOM1',\n",
       " 'JHOS2',\n",
       " 'KURAMOCHI',\n",
       " 'MCAS',\n",
       " 'OAW28',\n",
       " 'OAW42',\n",
       " 'OC314',\n",
       " 'ONCODG1',\n",
       " 'OV56',\n",
       " 'OV7',\n",
       " 'OV90',\n",
       " 'OVCAR4',\n",
       " 'OVCAR8',\n",
       " 'OVISE',\n",
       " 'OVK18',\n",
       " 'OVSAHO',\n",
       " 'OVTOKO',\n",
       " 'RMGI',\n",
       " 'SKOV3',\n",
       " 'SNU119',\n",
       " 'SNU8',\n",
       " 'SNU840',\n",
       " 'TOV112D',\n",
       " 'TOV21G',\n",
       " 'TYKNU',\n",
       " 'ASPC1',\n",
       " 'BXPC3',\n",
       " 'CAPAN1',\n",
       " 'CAPAN2',\n",
       " 'CFPAC1',\n",
       " 'DANG',\n",
       " 'HPAC',\n",
       " 'HPAFII',\n",
       " 'HS766T',\n",
       " 'HUPT3',\n",
       " 'HUPT4',\n",
       " 'KP2',\n",
       " 'KP3',\n",
       " 'KP4',\n",
       " 'L33',\n",
       " 'MIAPACA2',\n",
       " 'PANC0203',\n",
       " 'PANC0327',\n",
       " 'PANC0403',\n",
       " 'PANC0504',\n",
       " 'PANC0813',\n",
       " 'PANC1',\n",
       " 'PANC1005',\n",
       " 'PATU8902',\n",
       " 'PATU8988S',\n",
       " 'PATU8988T',\n",
       " 'PK1',\n",
       " 'PK45H',\n",
       " 'PK59',\n",
       " 'PSN1',\n",
       " 'QGP1',\n",
       " 'SNU213',\n",
       " 'SNU410',\n",
       " 'SU8686',\n",
       " 'SUIT2',\n",
       " 'SW1990',\n",
       " 'TCCPAN2',\n",
       " 'YAPC',\n",
       " 'ACCMESO1',\n",
       " 'ISTMES1',\n",
       " 'JL1',\n",
       " 'MPP89',\n",
       " 'MSTO211H',\n",
       " 'NCIH2052',\n",
       " 'NCIH2452',\n",
       " 'NCIH28',\n",
       " '22RV1',\n",
       " 'DU145',\n",
       " 'LNCAPCLONEFGC',\n",
       " 'NCIH660',\n",
       " 'PC3',\n",
       " 'VCAP',\n",
       " 'A253',\n",
       " 'YD15',\n",
       " 'A101D',\n",
       " 'A2058',\n",
       " 'A375',\n",
       " 'C32',\n",
       " 'CJM',\n",
       " 'COLO741',\n",
       " 'COLO792',\n",
       " 'COLO800',\n",
       " 'COLO829',\n",
       " 'G361',\n",
       " 'HMCB',\n",
       " 'HS294T',\n",
       " 'HS688AT',\n",
       " 'HS839T',\n",
       " 'HS852T',\n",
       " 'HS895T',\n",
       " 'HS934T',\n",
       " 'HS936T',\n",
       " 'HS940T',\n",
       " 'HS944T',\n",
       " 'HT144',\n",
       " 'IGR1',\n",
       " 'IGR37',\n",
       " 'IGR39',\n",
       " 'IPC298',\n",
       " 'K029AX',\n",
       " 'LOXIMVI',\n",
       " 'MALME3M',\n",
       " 'MDAMB435S',\n",
       " 'MELHO',\n",
       " 'MELJUSO',\n",
       " 'MEWO',\n",
       " 'RPMI7951',\n",
       " 'RVH421',\n",
       " 'SH4',\n",
       " 'SKMEL1',\n",
       " 'SKMEL2',\n",
       " 'SKMEL24',\n",
       " 'SKMEL28',\n",
       " 'SKMEL3',\n",
       " 'SKMEL30',\n",
       " 'SKMEL31',\n",
       " 'SKMEL5',\n",
       " 'UACC257',\n",
       " 'UACC62',\n",
       " 'WM115',\n",
       " 'WM1799',\n",
       " 'WM2664',\n",
       " 'WM793',\n",
       " 'WM88',\n",
       " 'WM983B',\n",
       " 'A204',\n",
       " 'G401',\n",
       " 'G402',\n",
       " 'GCT',\n",
       " 'HS729',\n",
       " 'HT1080',\n",
       " 'KYM1',\n",
       " 'MESSA',\n",
       " 'RD',\n",
       " 'RH18',\n",
       " 'RH30',\n",
       " 'RH41',\n",
       " 'RKN',\n",
       " 'SKLMS1',\n",
       " 'SKUT1',\n",
       " 'TE441T',\n",
       " 'TE617T',\n",
       " '2313287',\n",
       " 'AGS',\n",
       " 'ECC10',\n",
       " 'ECC12',\n",
       " 'FU97',\n",
       " 'GSS',\n",
       " 'GSU',\n",
       " 'HGC27',\n",
       " 'HS746T',\n",
       " 'HUG1N',\n",
       " 'IM95',\n",
       " 'KATOIII',\n",
       " 'KE39',\n",
       " 'LMSU',\n",
       " 'MKN1',\n",
       " 'MKN45',\n",
       " 'MKN7',\n",
       " 'MKN74',\n",
       " 'NCCSTCK140',\n",
       " 'NCIN87',\n",
       " 'NUGC3',\n",
       " 'NUGC4',\n",
       " 'OCUM1',\n",
       " 'SH10TC',\n",
       " 'SNU1',\n",
       " 'SNU16',\n",
       " 'SNU216',\n",
       " 'SNU5',\n",
       " 'SNU601',\n",
       " 'SNU620',\n",
       " 'SNU668',\n",
       " 'SNU719',\n",
       " '8305C',\n",
       " '8505C',\n",
       " 'BCPAP',\n",
       " 'BHT101',\n",
       " 'CAL62',\n",
       " 'CGTHW1',\n",
       " 'FTC133',\n",
       " 'FTC238',\n",
       " 'ML1',\n",
       " 'SW579',\n",
       " 'TT',\n",
       " 'TT2609C02',\n",
       " 'BHY',\n",
       " 'BICR16',\n",
       " 'BICR18',\n",
       " 'BICR22',\n",
       " 'BICR31',\n",
       " 'BICR56',\n",
       " 'BICR6',\n",
       " 'CAL27',\n",
       " 'CAL33',\n",
       " 'DETROIT562',\n",
       " 'FADU',\n",
       " 'HSC2',\n",
       " 'HSC3',\n",
       " 'HSC4',\n",
       " 'PECAPJ15',\n",
       " 'PECAPJ34CLONEC12',\n",
       " 'PECAPJ41CLONED2',\n",
       " 'PECAPJ49',\n",
       " 'SCC25',\n",
       " 'SCC4',\n",
       " 'SCC9',\n",
       " 'SNU1041',\n",
       " 'SNU1066',\n",
       " 'SNU1076',\n",
       " 'SNU1214',\n",
       " 'SNU46',\n",
       " 'SNU899',\n",
       " 'YD10B',\n",
       " 'YD38',\n",
       " 'YD8',\n",
       " '253J',\n",
       " '253JBV',\n",
       " '5637',\n",
       " '639V',\n",
       " '647V',\n",
       " 'BC3C',\n",
       " 'BFTC905',\n",
       " 'CAL29',\n",
       " 'HT1197',\n",
       " 'HT1376',\n",
       " 'J82',\n",
       " 'JMSU1',\n",
       " 'KMBC2',\n",
       " 'KU1919',\n",
       " 'RT112',\n",
       " 'RT4',\n",
       " 'SCABER',\n",
       " 'SW1710',\n",
       " 'T24',\n",
       " 'TCCSUP',\n",
       " 'UBLC1',\n",
       " 'UMUC1',\n",
       " 'UMUC3',\n",
       " 'VMCUB1']"
      ]
     },
     "execution_count": 415,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cell3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## It seems like the Pozdeyev data isn't too good regarding gene expression data, therefore I try to download the ctrp data directly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 402,
   "metadata": {},
   "outputs": [],
   "source": [
    "lo2 = pd.read_csv('data/ctrp2.csv', sep=';')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 403,
   "metadata": {},
   "outputs": [],
   "source": [
    "lo = lo2['ccl_name']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 404,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 404,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "miss = []\n",
    "for ccl in lo:\n",
    "    if ccl not in cell2:\n",
    "        miss.append(ccl)\n",
    "len(miss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 411,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'1\\xa0MKN',\n",
       " '45\\xa0MKN',\n",
       " '74\\xa0MKN',\n",
       " '7\\xa0MCF',\n",
       " '7\\xa0MKN',\n",
       " 'H292',\n",
       " 'TTTHYROID'}"
      ]
     },
     "execution_count": 411,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(miss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 409,
   "metadata": {},
   "outputs": [],
   "source": [
    "dif = ['MCF7', 'NCIH292', 'MKN74', 'MKN1', 'MKN45', 'MKN7', 'TT']\n",
    "for i in set(miss):\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 438,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>master_ccl_id</th>\n",
       "      <th>ccl_name</th>\n",
       "      <th>ccle_primary_site</th>\n",
       "      <th>ccle_primary_hist</th>\n",
       "      <th>ccle_hist_subtype_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>697</td>\n",
       "      <td>haematopoietic_and_lymphoid_tissue</td>\n",
       "      <td>lymphoid_neoplasm</td>\n",
       "      <td>acute_lymphoblastic_B_cell_leukaemia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>5637</td>\n",
       "      <td>urinary_tract</td>\n",
       "      <td>carcinoma</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>2313287</td>\n",
       "      <td>stomach</td>\n",
       "      <td>carcinoma</td>\n",
       "      <td>adenocarcinoma</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>22RV1</td>\n",
       "      <td>prostate</td>\n",
       "      <td>carcinoma</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8</td>\n",
       "      <td>253J</td>\n",
       "      <td>urinary_tract</td>\n",
       "      <td>carcinoma</td>\n",
       "      <td>transitional_cell_carcinoma</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>830</th>\n",
       "      <td>155511</td>\n",
       "      <td>NCIH2291</td>\n",
       "      <td>lung</td>\n",
       "      <td>carcinoma</td>\n",
       "      <td>adenocarcinoma</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>831</th>\n",
       "      <td>155513</td>\n",
       "      <td>NCIH2342</td>\n",
       "      <td>lung</td>\n",
       "      <td>carcinoma</td>\n",
       "      <td>adenocarcinoma</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>832</th>\n",
       "      <td>155518</td>\n",
       "      <td>NCIH2405</td>\n",
       "      <td>lung</td>\n",
       "      <td>carcinoma</td>\n",
       "      <td>adenocarcinoma</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>833</th>\n",
       "      <td>155519</td>\n",
       "      <td>NCIH2444</td>\n",
       "      <td>lung</td>\n",
       "      <td>carcinoma</td>\n",
       "      <td>non_small_cell_carcinoma</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>834</th>\n",
       "      <td>155520</td>\n",
       "      <td>NCIH2452</td>\n",
       "      <td>pleura</td>\n",
       "      <td>mesothelioma</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>835 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     master_ccl_id  ccl_name                   ccle_primary_site  \\\n",
       "0                1       697  haematopoietic_and_lymphoid_tissue   \n",
       "1                3      5637                       urinary_tract   \n",
       "2                4   2313287                             stomach   \n",
       "3                7     22RV1                            prostate   \n",
       "4                8      253J                       urinary_tract   \n",
       "..             ...       ...                                 ...   \n",
       "830         155511  NCIH2291                                lung   \n",
       "831         155513  NCIH2342                                lung   \n",
       "832         155518  NCIH2405                                lung   \n",
       "833         155519  NCIH2444                                lung   \n",
       "834         155520  NCIH2452                              pleura   \n",
       "\n",
       "     ccle_primary_hist                   ccle_hist_subtype_1  \n",
       "0    lymphoid_neoplasm  acute_lymphoblastic_B_cell_leukaemia  \n",
       "1            carcinoma                                   NaN  \n",
       "2            carcinoma                        adenocarcinoma  \n",
       "3            carcinoma                                   NaN  \n",
       "4            carcinoma           transitional_cell_carcinoma  \n",
       "..                 ...                                   ...  \n",
       "830          carcinoma                        adenocarcinoma  \n",
       "831          carcinoma                        adenocarcinoma  \n",
       "832          carcinoma                        adenocarcinoma  \n",
       "833          carcinoma              non_small_cell_carcinoma  \n",
       "834       mesothelioma                                   NaN  \n",
       "\n",
       "[835 rows x 5 columns]"
      ]
     },
     "execution_count": 438,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g = pd.read_csv('data/CTRP/ctrp_g.csv', sep='\\t')\n",
    "g"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 433,
   "metadata": {},
   "outputs": [],
   "source": [
    "ge = pd.read_csv('data/CTRP/ctrp_ge.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 447,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "miss = []\n",
    "for i in g['ccl_name'].unique():\n",
    "    if i not in poz:\n",
    "        miss.append(i)\n",
    "\n",
    "print(len(miss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 445,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "887"
      ]
     },
     "execution_count": 445,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(poz)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 446,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "835"
      ]
     },
     "execution_count": 446,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(g['ccl_name'].unique())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}