Skin_problems_module.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "bDI_QZxckSY7"
      },
      "outputs": [],
      "source": [
        "# get the data from kaggle"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Importing required libraries\n",
        "import tensorflow\n",
        "import keras\n",
        "import matplotlib.pyplot as plt\n",
        "from PIL import Image\n",
        "import seaborn as sns\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import os\n",
        "from tensorflow.keras.utils import to_categorical\n",
        "from glob import glob"
      ],
      "metadata": {
        "id": "6m3Xp3asgXo8"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Reading the data from HAM_metadata.csv\n",
        "df = pd.read_csv('/content/sample_data/HAM10000_metadata.csv')"
      ],
      "metadata": {
        "id": "ZW8RM-rOsGgg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "atorjbRhdI6X"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#just a test\n",
        "df.head()"
      ],
      "metadata": {
        "id": "plKG_9vpSOzN",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "3e95afad-898a-4a7e-bca6-8271dda43b4d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     lesion_id      image_id   dx dx_type   age   sex localization\n",
              "0  HAM_0000118  ISIC_0027419  bkl   histo  80.0  male        scalp\n",
              "1  HAM_0000118  ISIC_0025030  bkl   histo  80.0  male        scalp\n",
              "2  HAM_0002730  ISIC_0026769  bkl   histo  80.0  male        scalp\n",
              "3  HAM_0002730  ISIC_0025661  bkl   histo  80.0  male        scalp\n",
              "4  HAM_0001466  ISIC_0031633  bkl   histo  75.0  male          ear"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b883f1ef-4d1c-48ad-ba69-2f7e8156ae99\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>lesion_id</th>\n",
              "      <th>image_id</th>\n",
              "      <th>dx</th>\n",
              "      <th>dx_type</th>\n",
              "      <th>age</th>\n",
              "      <th>sex</th>\n",
              "      <th>localization</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>HAM_0000118</td>\n",
              "      <td>ISIC_0027419</td>\n",
              "      <td>bkl</td>\n",
              "      <td>histo</td>\n",
              "      <td>80.0</td>\n",
              "      <td>male</td>\n",
              "      <td>scalp</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>HAM_0000118</td>\n",
              "      <td>ISIC_0025030</td>\n",
              "      <td>bkl</td>\n",
              "      <td>histo</td>\n",
              "      <td>80.0</td>\n",
              "      <td>male</td>\n",
              "      <td>scalp</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>HAM_0002730</td>\n",
              "      <td>ISIC_0026769</td>\n",
              "      <td>bkl</td>\n",
              "      <td>histo</td>\n",
              "      <td>80.0</td>\n",
              "      <td>male</td>\n",
              "      <td>scalp</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>HAM_0002730</td>\n",
              "      <td>ISIC_0025661</td>\n",
              "      <td>bkl</td>\n",
              "      <td>histo</td>\n",
              "      <td>80.0</td>\n",
              "      <td>male</td>\n",
              "      <td>scalp</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>HAM_0001466</td>\n",
              "      <td>ISIC_0031633</td>\n",
              "      <td>bkl</td>\n",
              "      <td>histo</td>\n",
              "      <td>75.0</td>\n",
              "      <td>male</td>\n",
              "      <td>ear</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b883f1ef-4d1c-48ad-ba69-2f7e8156ae99')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-b883f1ef-4d1c-48ad-ba69-2f7e8156ae99 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-b883f1ef-4d1c-48ad-ba69-2f7e8156ae99');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-02f9304b-92fc-4787-9a70-60e12a8c9682\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-02f9304b-92fc-4787-9a70-60e12a8c9682')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-02f9304b-92fc-4787-9a70-60e12a8c9682 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# test \n",
        "df.dtypes"
      ],
      "metadata": {
        "id": "ybJlA3105YOL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1b7dd376-ac7e-4084-fae9-288ff9f4668c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "lesion_id        object\n",
              "image_id         object\n",
              "dx               object\n",
              "dx_type          object\n",
              "age             float64\n",
              "sex              object\n",
              "localization     object\n",
              "dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# test\n",
        "df.describe()"
      ],
      "metadata": {
        "id": "n9Z4CS3KUVFL",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "outputId": "0fb2fbf9-25c9-434e-d96b-dc2e52305e28"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "               age\n",
              "count  9958.000000\n",
              "mean     51.863828\n",
              "std      16.968614\n",
              "min       0.000000\n",
              "25%      40.000000\n",
              "50%      50.000000\n",
              "75%      65.000000\n",
              "max      85.000000"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-29390e04-ed3c-4a98-833a-b9e2c2d268f3\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>age</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>9958.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>51.863828</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>16.968614</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>40.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>50.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>65.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>85.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-29390e04-ed3c-4a98-833a-b9e2c2d268f3')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-29390e04-ed3c-4a98-833a-b9e2c2d268f3 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-29390e04-ed3c-4a98-833a-b9e2c2d268f3');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-5e5f9dd8-c3a1-4695-aebe-64e4a64d49a2\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-5e5f9dd8-c3a1-4695-aebe-64e4a64d49a2')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-5e5f9dd8-c3a1-4695-aebe-64e4a64d49a2 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# remove null data\n",
        "df.isnull().sum()"
      ],
      "metadata": {
        "id": "DPtkqKdIV0Qv",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4f01e2d8-301a-4aa5-a824-057da158f7ab"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "lesion_id        0\n",
              "image_id         0\n",
              "dx               0\n",
              "dx_type          0\n",
              "age             57\n",
              "sex              0\n",
              "localization     0\n",
              "dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# تعويض القيم المفقودة في العمود 'age' في إطار البيانات بالقيمة الصحيحة تمثل معدل الاعمار\n",
        "df['age'].fillna(int(df['age'].mean()), inplace=True)"
      ],
      "metadata": {
        "id": "sJJlI7ZqXRAV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# للتاكد فقط\n",
        "df.isnull().sum()"
      ],
      "metadata": {
        "id": "JzBDl9JhYTCs",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7da3aac5-a7d7-4949-aa79-1303c6700133"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "lesion_id       0\n",
              "image_id        0\n",
              "dx              0\n",
              "dx_type         0\n",
              "age             0\n",
              "sex             0\n",
              "localization    0\n",
              "dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# إنشاء قاموس للربط بين معرف الصورة ومسار ملف الصورة المناسب لكل صورة في مجلدات مختلفة\n",
        "lesion_type_dict = {\n",
        "    'nv': 'Melanocytic nevi',\n",
        "    'mel': 'Melanoma',\n",
        "    'bkl': 'Benign keratosis-like lesions ',\n",
        "    'bcc': 'Basal cell carcinoma',\n",
        "    'akiec': 'Actinic keratoses',\n",
        "    'vasc': 'Vascular lesions',\n",
        "    'df': 'Dermatofibroma'\n",
        "}\n",
        "base_skin_dir = '/content/sample_data/'\n",
        "\n",
        "# Merge images from both folders into one dictionary\n",
        "\n",
        "imageid_path_dict = {os.path.splitext(os.path.basename(x))[0]: x\n",
        "                     for x in glob(os.path.join(base_skin_dir, '*', '*.jpg'))}"
      ],
      "metadata": {
        "id": "qBarna31Ygeb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "df['path'] = df['image_id'].map(imageid_path_dict.get)\n",
        "df['cell_type'] = df['dx'].map(lesion_type_dict.get)\n",
        "df['cell_type_idx'] = pd.Categorical(df['cell_type']).codes\n",
        "df.head()"
      ],
      "metadata": {
        "id": "AIz2IhPdahYA",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "09b50f91-d926-4f7a-9bbb-3acb8ab0d7fb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     lesion_id      image_id   dx dx_type   age   sex localization  path  \\\n",
              "0  HAM_0000118  ISIC_0027419  bkl   histo  80.0  male        scalp  None   \n",
              "1  HAM_0000118  ISIC_0025030  bkl   histo  80.0  male        scalp  None   \n",
              "2  HAM_0002730  ISIC_0026769  bkl   histo  80.0  male        scalp  None   \n",
              "3  HAM_0002730  ISIC_0025661  bkl   histo  80.0  male        scalp  None   \n",
              "4  HAM_0001466  ISIC_0031633  bkl   histo  75.0  male          ear  None   \n",
              "\n",
              "                        cell_type  cell_type_idx  \n",
              "0  Benign keratosis-like lesions               2  \n",
              "1  Benign keratosis-like lesions               2  \n",
              "2  Benign keratosis-like lesions               2  \n",
              "3  Benign keratosis-like lesions               2  \n",
              "4  Benign keratosis-like lesions               2  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-bf61a6df-d636-4279-b38f-d77ca31513ae\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>lesion_id</th>\n",
              "      <th>image_id</th>\n",
              "      <th>dx</th>\n",
              "      <th>dx_type</th>\n",
              "      <th>age</th>\n",
              "      <th>sex</th>\n",
              "      <th>localization</th>\n",
              "      <th>path</th>\n",
              "      <th>cell_type</th>\n",
              "      <th>cell_type_idx</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>HAM_0000118</td>\n",
              "      <td>ISIC_0027419</td>\n",
              "      <td>bkl</td>\n",
              "      <td>histo</td>\n",
              "      <td>80.0</td>\n",
              "      <td>male</td>\n",
              "      <td>scalp</td>\n",
              "      <td>None</td>\n",
              "      <td>Benign keratosis-like lesions</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>HAM_0000118</td>\n",
              "      <td>ISIC_0025030</td>\n",
              "      <td>bkl</td>\n",
              "      <td>histo</td>\n",
              "      <td>80.0</td>\n",
              "      <td>male</td>\n",
              "      <td>scalp</td>\n",
              "      <td>None</td>\n",
              "      <td>Benign keratosis-like lesions</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>HAM_0002730</td>\n",
              "      <td>ISIC_0026769</td>\n",
              "      <td>bkl</td>\n",
              "      <td>histo</td>\n",
              "      <td>80.0</td>\n",
              "      <td>male</td>\n",
              "      <td>scalp</td>\n",
              "      <td>None</td>\n",
              "      <td>Benign keratosis-like lesions</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>HAM_0002730</td>\n",
              "      <td>ISIC_0025661</td>\n",
              "      <td>bkl</td>\n",
              "      <td>histo</td>\n",
              "      <td>80.0</td>\n",
              "      <td>male</td>\n",
              "      <td>scalp</td>\n",
              "      <td>None</td>\n",
              "      <td>Benign keratosis-like lesions</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>HAM_0001466</td>\n",
              "      <td>ISIC_0031633</td>\n",
              "      <td>bkl</td>\n",
              "      <td>histo</td>\n",
              "      <td>75.0</td>\n",
              "      <td>male</td>\n",
              "      <td>ear</td>\n",
              "      <td>None</td>\n",
              "      <td>Benign keratosis-like lesions</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-bf61a6df-d636-4279-b38f-d77ca31513ae')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-bf61a6df-d636-4279-b38f-d77ca31513ae button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-bf61a6df-d636-4279-b38f-d77ca31513ae');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-4147df54-4a9c-4a9f-86b5-eb0d8d25b979\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-4147df54-4a9c-4a9f-86b5-eb0d8d25b979')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-4147df54-4a9c-4a9f-86b5-eb0d8d25b979 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Image Preprocessing\n",
        "df['image'] = df['path'].map(lambda x: np.asarray(Image.open(x).resize((125,100))))"
      ],
      "metadata": {
        "id": "Dx2A3xLwdLIp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "n_samples = 5\n",
        "fig, m_axs = plt.subplots(7, n_samples, figsize = (4*n_samples, 3*7))\n",
        "for n_axs, (type_name, type_rows) in zip(m_axs,\n",
        "                                         df.sort_values(['cell_type']).groupby('cell_type')):\n",
        "    n_axs[0].set_title(type_name)\n",
        "    for c_ax, (_, c_row) in zip(n_axs, type_rows.sample(n_samples, random_state=2018).iterrows()):\n",
        "        c_ax.imshow(c_row['image'])\n",
        "        c_ax.axis('off')\n",
        "fig.savefig('category_samples.png', dpi=300)"
      ],
      "metadata": {
        "id": "7Z6-a1BldVIc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "BOCYBg4rd4Cg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# See the image size distribution - should just return one row (all images are uniform)\n",
        "df['image'].map(lambda x: x.shape).value_counts()"
      ],
      "metadata": {
        "id": "7cX469pPd2Nw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#<!-- Exploratory Data Analysis  -->"
      ],
      "metadata": {
        "id": "nSEKdGpweMFQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df= df[df['age'] != 0]\n",
        "df= df[df['sex'] != 'unknown']"
      ],
      "metadata": {
        "id": "7sizi865ePg2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# UNIVARIATE ANALYSIS\n",
        "plt.figure(figsize=(20,10))\n",
        "plt.subplots_adjust(left=0.125, bottom=1, right=0.9, top=2, hspace=0.2)\n",
        "plt.subplot(2,4,1)\n",
        "plt.title(\"AGE\",fontsize=15)\n",
        "plt.ylabel(\"Count\")\n",
        "df['age'].value_counts().plot.bar()\n",
        "\n",
        "plt.subplot(2,4,2)\n",
        "plt.title(\"GENDER\",fontsize=15)\n",
        "plt.ylabel(\"Count\")\n",
        "df['sex'].value_counts().plot.bar()\n",
        "\n",
        "plt.subplot(2,4,3)\n",
        "plt.title(\"localization\",fontsize=15)\n",
        "plt.ylabel(\"Count\")\n",
        "plt.xticks(rotation=45)\n",
        "df['localization'].value_counts().plot.bar()\n",
        "\n",
        "plt.subplot(2,4,4)\n",
        "plt.title(\"CELL TYPE\",fontsize=15)\n",
        "plt.ylabel(\"Count\")\n",
        "df['cell_type'].value_counts().plot.bar()"
      ],
      "metadata": {
        "id": "9Iq57PJbfeyi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.figure(figsize=(15,10))\n",
        "plt.subplot(1,2,1)\n",
        "df['dx'].value_counts().plot.pie(autopct=\"%1.1f%%\")\n",
        "plt.subplot(1,2,2)\n",
        "df['dx_type'].value_counts().plot.pie(autopct=\"%1.1f%%\")\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "4c_FD9UaIEjR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.figure(figsize=(25,10))\n",
        "plt.title('LOCALIZATION VS GENDER',fontsize = 15)\n",
        "sns.countplot(y='localization', hue='sex',data=df)"
      ],
      "metadata": {
        "id": "sVrVpI_IILsU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.figure(figsize=(25,10))\n",
        "plt.title('LOCALIZATION VS CELL TYPE',fontsize = 15)\n",
        "sns.countplot(y='localization', hue='cell_type',data=df)"
      ],
      "metadata": {
        "id": "8TSCy4H4IPWH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.figure(figsize=(25,10))\n",
        "plt.subplot(131)\n",
        "plt.title('AGE VS CELL TYPE',fontsize = 15)\n",
        "sns.countplot(y='age', hue='cell_type',data=df)\n",
        "plt.subplot(132)\n",
        "plt.title('GENDER VS CELL TYPE',fontsize = 15)\n",
        "sns.countplot(y='sex', hue='cell_type',data=df)"
      ],
      "metadata": {
        "id": "FT6mn-YVITMm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "M3_g-boyItgV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "import keras\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Dropout\n",
        "import tensorflow as tf\n",
        "from sklearn.preprocessing import StandardScaler"
      ],
      "metadata": {
        "id": "WxUiwyXuIXcT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "A simple neural network is implemented first with the following layers to get patterns in images. The accuracy achieved is approximately 69 percent."
      ],
      "metadata": {
        "id": "TjFQGw7xI8cZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "features=df.drop(columns=['cell_type_idx'],axis=1)\n",
        "target=df['cell_type_idx']"
      ],
      "metadata": {
        "id": "vKgwU-IlI-sD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# just a test\n",
        "features.head()"
      ],
      "metadata": {
        "id": "IhgAyVzIJEud"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x_train_o, x_test_o, y_train_o, y_test_o = train_test_split(features, target, test_size=0.25,random_state=666)\n",
        "tf.unique(x_train_o.cell_type.values)"
      ],
      "metadata": {
        "id": "cCZWMQtUJVzl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x_train = np.asarray(x_train_o['image'].tolist())\n",
        "x_test = np.asarray(x_test_o['image'].tolist())\n",
        "\n",
        "x_train_mean = np.mean(x_train)\n",
        "x_train_std = np.std(x_train)\n",
        "\n",
        "x_test_mean = np.mean(x_test)\n",
        "x_test_std = np.std(x_test)\n",
        "\n",
        "x_train = (x_train - x_train_mean)/x_train_std\n",
        "x_test = (x_test - x_test_mean)/x_test_std"
      ],
      "metadata": {
        "id": "2ioWxXlRJcCU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "y_train = to_categorical(y_train_o, num_classes = 7)\n",
        "y_test = to_categorical(y_test_o, num_classes = 7)\n",
        "y_test"
      ],
      "metadata": {
        "id": "MLnzIfLQKRBy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x_train, x_validate, y_train, y_validate = train_test_split(x_train, y_train, test_size = 0.1, random_state = 999)\n",
        "# Reshape image in 3 dimensions (height = 100, width = 125 , canal = 3)\n",
        "x_train = x_train.reshape(x_train.shape[0], *(100, 125, 3))\n",
        "x_test = x_test.reshape(x_test.shape[0], *(100, 125, 3))\n",
        "x_validate = x_validate.reshape(x_validate.shape[0], *(100, 125, 3))"
      ],
      "metadata": {
        "id": "qse745sjKWts"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x_train = x_train.reshape(6696,125*100*3)\n",
        "x_test = x_test.reshape(2481,125*100*3)\n",
        "print(x_train.shape)\n",
        "print(x_test.shape)"
      ],
      "metadata": {
        "id": "3H5wt3tbKfta"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# define the keras model"
      ],
      "metadata": {
        "id": "PwjF1IjsKi1Z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = Sequential()\n",
        "\n",
        "model.add(Dense(units= 64, kernel_initializer = 'uniform', activation = 'relu', input_dim = 37500))\n",
        "model.add(Dense(units= 64, kernel_initializer = 'uniform', activation = 'relu'))\n",
        "model.add(Dense(units= 64, kernel_initializer = 'uniform', activation = 'relu'))\n",
        "model.add(Dense(units= 64, kernel_initializer = 'uniform', activation = 'relu'))\n",
        "model.add(Dense(units = 7, kernel_initializer = 'uniform', activation = 'softmax'))\n",
        "\n",
        "optimizer = tf.keras.optimizers.Adam(learning_rate = 0.00075,\n",
        "                                    beta_1 = 0.9,\n",
        "                                    beta_2 = 0.999,\n",
        "                                    epsilon = 1e-8)\n",
        "\n",
        "# compile the keras model\n",
        "model.compile(optimizer = optimizer, loss = 'categorical_crossentropy', metrics = ['accuracy'])\n",
        "\n",
        "# fit the keras model on the dataset\n",
        "history = model.fit(x_train, y_train, batch_size = 10, epochs = 50)\n",
        "\n",
        "accuracy = model.evaluate(x_test, y_test, verbose=1)[1]\n",
        "print(\"Test: accuracy = \",accuracy*100,\"%\")"
      ],
      "metadata": {
        "id": "A0tEe_G7Kju5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from keras.utils.vis_utils import plot_model\n",
        "plot_model(model, to_file='model_plot.png', show_shapes=True, show_layer_names=True)"
      ],
      "metadata": {
        "id": "S8Giq48CKqVo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#                                                   CNN"
      ],
      "metadata": {
        "id": "ANW65osdLdeG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "-Applied Data augmentation using ImageDatagenerator before model training\n",
        "Since the data is less, we apply data augmentation using ImageDataGenerator.\n",
        "\n",
        "ImageDataGenerator generates augmentation of images in real-time while the model is still training. One can apply any random transformations on each training image as it is passed to the model."
      ],
      "metadata": {
        "id": "f_MaMoBfLPyP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from tensorflow.keras.layers import Flatten,Dense,Dropout,BatchNormalization,Conv2D, MaxPool2D\n",
        "from keras.optimizers import Adam\n",
        "from keras.callbacks import ReduceLROnPlateau\n",
        "from tensorflow.keras.preprocessing.image import ImageDataGenerator"
      ],
      "metadata": {
        "id": "96akoJ0tLZ_k"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Set the CNN model\n",
        "# my CNN ستكون كالاتي -> [[Conv2D->relu]*2 -> MaxPool2D -> Dropout]*3 -> Flatten -> Dense*2 -> Dropout -> Out"
      ],
      "metadata": {
        "id": "pzgkZu5PMcZU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "input_shape = (100, 125, 3)\n",
        "num_classes = 7\n",
        "# ​\n",
        "model = Sequential()\n",
        "model.add(Conv2D(32, kernel_size=(3, 3),activation='relu',padding = 'Same',input_shape=input_shape))\n",
        "model.add(Conv2D(32,kernel_size=(3, 3), activation='relu',padding = 'Same',))\n",
        "model.add(MaxPool2D(pool_size = (2, 2)))\n",
        "model.add(Dropout(0.16))\n",
        "# ​\n",
        "model.add(Conv2D(32, kernel_size=(3, 3),activation='relu',padding = 'Same'))\n",
        "model.add(Conv2D(32,kernel_size=(3, 3), activation='relu',padding = 'Same',))\n",
        "model.add(MaxPool2D(pool_size = (2, 2)))\n",
        "model.add(Dropout(0.20))\n",
        "# ​\n",
        "model.add(Conv2D(64, (3, 3), activation='relu',padding = 'same'))\n",
        "model.add(Conv2D(64, (3, 3), activation='relu',padding = 'Same'))\n",
        "model.add(MaxPool2D(pool_size=(2, 2)))\n",
        "model.add(Dropout(0.25))\n",
        "#\n",
        "model.add(Flatten())\n",
        "model.add(Dense(256, activation='relu'))\n",
        "model.add(Dense(128, activation='relu'))\n",
        "model.add(Dropout(0.4))\n",
        "model.add(Dense(num_classes, activation='softmax'))\n",
        "model.summary()\n"
      ],
      "metadata": {
        "id": "4pqSSHU8MxdE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the optimizer\n",
        "optimizer = Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)"
      ],
      "metadata": {
        "id": "WBS1VZ5CM07o"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Compile the model\n",
        "model.compile(optimizer = optimizer , loss = \"categorical_crossentropy\", metrics=[\"accuracy\"])"
      ],
      "metadata": {
        "id": "scujV839NCG-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Set a learning rate annealer\n",
        "learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy',\n",
        "                                            patience=4,\n",
        "                                            verbose=1,\n",
        "                                            factor=0.5,\n",
        "                                            min_lr=0.00001)"
      ],
      "metadata": {
        "id": "U73SlSoZNH2k"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x_train, x_validate, y_train, y_validate = train_test_split(x_train, y_train, test_size = 0.1, random_state = 999)\n",
        "# Reshape image in 3 dimensions (height = 100, width = 125 , canal = 3)\n",
        "x_train = x_train.reshape(x_train.shape[0], *(100, 125, 3))\n",
        "x_test = x_test.reshape(x_test.shape[0], *(100, 125, 3))\n",
        "x_validate = x_validate.reshape(x_validate.shape[0], *(100, 125, 3))\n",
        "# With data augmentation to prevent overfitting\n",
        "\n",
        "datagen = ImageDataGenerator(\n",
        "        featurewise_center=False,  # set input mean to 0 over the dataset\n",
        "        samplewise_center=False,  # set each sample mean to 0\n",
        "        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n",
        "        samplewise_std_normalization=False,  # divide each input by its std\n",
        "        zca_whitening=False,  # apply ZCA whitening\n",
        "        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n",
        "        zoom_range = 0.1, # Randomly zoom image\n",
        "        width_shift_range=0.12,  # randomly shift images horizontally (fraction of total width)\n",
        "        height_shift_range=0.12,  # randomly shift images vertically (fraction of total height)\n",
        "        horizontal_flip=True,  # randomly flip images\n",
        "        vertical_flip=True)  # randomly flip images\n",
        "\n",
        "datagen.fit(x_train)"
      ],
      "metadata": {
        "id": "TnZl06gSNKpN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Fit the model\n",
        "epochs = 60\n",
        "batch_size = 16\n",
        "history = model.fit_generator(datagen.flow(x_train,y_train, batch_size=batch_size),\n",
        "                              epochs = epochs, validation_data = (x_validate,y_validate),\n",
        "                              verbose = 1, steps_per_epoch=x_train.shape[0] // batch_size\n",
        "                              , callbacks=[learning_rate_reduction])\n",
        "\n",
        "from tensorflow.keras.metrics import Recall\n",
        "from sklearn.metrics import classification_report,confusion_matrix"
      ],
      "metadata": {
        "id": "shNPZVZ3NRuI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from keras.utils.vis_utils import plot_model\n",
        "plot_model(model, to_file='model_plot.png', show_shapes=True, show_layer_names=True)"
      ],
      "metadata": {
        "id": "7JTjEaomNW5T"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# The CNN model has been visualised in the model attached above."
      ],
      "metadata": {
        "id": "d2F3MxToNgtX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "loss, accuracy = model.evaluate(x_test, y_test, verbose=1)\n",
        "loss_v, accuracy_v = model.evaluate(x_validate, y_validate, verbose=1)\n",
        "print(\"Validation: accuracy = %f  ;  loss_v = %f\" % (accuracy_v, loss_v))\n",
        "print(\"Test: accuracy = %f  ;  loss = %f\" % (accuracy, loss))\n",
        "model.save(\"model.h5\")"
      ],
      "metadata": {
        "id": "ABqhdiHoNcFF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import itertools"
      ],
      "metadata": {
        "id": "sv_f7Lz3Nija"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Function to plot confusion matrix\n",
        "def plot_confusion_matrix(cm, classes,\n",
        "                          normalize=False,\n",
        "                          title='Confusion matrix',\n",
        "                          cmap=plt.cm.Blues):\n",
        "    \"\"\"\n",
        "    This function prints and plots the confusion matrix.\n",
        "    Normalization can be applied by setting `normalize=True`.\n",
        "    \"\"\"\n",
        "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
        "    plt.title(title)\n",
        "    plt.colorbar()\n",
        "    tick_marks = np.arange(len(classes))\n",
        "    plt.xticks(tick_marks, classes, rotation=45)\n",
        "    plt.yticks(tick_marks, classes)\n",
        "\n",
        "    if normalize:\n",
        "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
        "\n",
        "    thresh = cm.max() / 2.\n",
        "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
        "        plt.text(j, i, cm[i, j],\n",
        "                 horizontalalignment=\"center\",\n",
        "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
        "\n",
        "    plt.tight_layout()\n",
        "    plt.ylabel('True label')\n",
        "    plt.xlabel('Predicted label')\n",
        "\n",
        "# Predict the values from the validation dataset\n",
        "Y_pred = model.predict(x_validate)\n",
        "# Convert predictions classes to one hot vectors\n",
        "Y_pred_classes = np.argmax(Y_pred,axis = 1)\n",
        "# Convert validation observations to one hot vectors\n",
        "Y_true = np.argmax(y_validate,axis = 1)\n",
        "# compute the confusion matrix\n",
        "confusion_mtx = confusion_matrix(Y_true, Y_pred_classes)\n",
        "\n",
        "\n",
        "\n",
        "# plot the confusion matrix\n",
        "plot_confusion_matrix(confusion_mtx, classes = range(7))"
      ],
      "metadata": {
        "id": "gty1lg8mSEKi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Predict the values from the validation dataset\n",
        "Y_pred = model.predict(x_test)\n",
        "# Convert predictions classes to one hot vectors\n",
        "Y_pred_classes = np.argmax(Y_pred,axis = 1)\n",
        "# Convert validation observations to one hot vectors\n",
        "Y_true = np.argmax(y_test,axis = 1)\n",
        "# compute the confusion matrix\n",
        "confusion_mtx = confusion_matrix(Y_true, Y_pred_classes)\n",
        "\n",
        "\n",
        "\n",
        "# plot the confusion matrix\n",
        "plot_confusion_matrix(confusion_mtx, classes = range(7))"
      ],
      "metadata": {
        "id": "NN5EG8GuSKcB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "label_frac_error = 1 - np.diag(confusion_mtx) / np.sum(confusion_mtx, axis=1)\n",
        "plt.bar(np.arange(7),label_frac_error)\n",
        "plt.xlabel('True Label')\n",
        "plt.ylabel('Fraction classified incorrectly')"
      ],
      "metadata": {
        "id": "9avPZHUgSVQk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# رسم خسارة التدريب والتحقق من صحة الدقة لنموذج شبكة عصبية عبر العصور"
      ],
      "metadata": {
        "id": "w3kqcL1MXjup"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def plot_model_history(model_history):\n",
        "    fig, axs = plt.subplots(1,2,figsize=(15,5))\n",
        "    # summarize history for accuracy\n",
        "    axs[0].plot(range(1,len(model_history.history['accuracy'])+1),model_history.history['accuracy'])\n",
        "    axs[0].plot(range(1,len(model_history.history['val_accuracy'])+1),model_history.history['val_accuracy'])\n",
        "    axs[0].set_title('Model Accuracy')\n",
        "    axs[0].set_ylabel('Accuracy')\n",
        "    axs[0].set_xlabel('Epoch')\n",
        "    axs[0].set_xticks(np.arange(1,len(model_history.history['accuracy'])+1),len(model_history.history['accuracy'])/10)\n",
        "    axs[0].legend(['train', 'val'], loc='best')\n",
        "    # summarize history for loss\n",
        "    axs[1].plot(range(1,len(model_history.history['loss'])+1),model_history.history['loss'])\n",
        "    axs[1].plot(range(1,len(model_history.history['val_loss'])+1),model_history.history['val_loss'])\n",
        "    axs[1].set_title('Model Loss')\n",
        "    axs[1].set_ylabel('Loss')\n",
        "    axs[1].set_xlabel('Epoch')\n",
        "    axs[1].set_xticks(np.arange(1,len(model_history.history['loss'])+1),len(model_history.history['loss'])/10)\n",
        "    axs[1].legend(['train', 'val'], loc='best')\n",
        "    plt.show()\n",
        "plot_model_history(history)"
      ],
      "metadata": {
        "id": "8HcStADcXtNp"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}