on jupyter notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "370a0b07",
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "from scipy.stats import norm\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.metrics import f1_score, recall_score, accuracy_score, precision_score, confusion_matrix\n",
    "import torch\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from imblearn.under_sampling import RandomUnderSampler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "8eb54194",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = pd.read_csv(\"data/magic04.data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "76e4ea81",
   "metadata": {},
   "outputs": [],
   "source": [
    "def ready_data(dataset, label_encode=True, resample=True, split=True):\n",
    "    inputs = np.array(dataset.values[:, 0:-1], dtype=np.float32)\n",
    "    temp_target = dataset.values[:, -1]\n",
    "    # label encode the target\n",
    "    if label_encode:\n",
    "        temp_target = LabelEncoder().fit_transform(temp_target)\n",
    "\n",
    "    targets = np.array(temp_target, dtype=np.float16)\n",
    "    targets = targets.reshape((targets.shape[0], 1))\n",
    "    # under sampling\n",
    "    if resample:\n",
    "        rus = RandomUnderSampler(random_state=0, replacement=True)\n",
    "        inputs, targets = rus.fit_resample(inputs, targets)\n",
    "\n",
    "    # splitting\n",
    "    # random_state=0 means that every time we have the same set of selected\n",
    "    return train_test_split(inputs, targets, test_size=0.3, random_state=0) if split else (inputs, targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "87d0cc49",
   "metadata": {},
   "outputs": [],
   "source": [
    "def main(dataset, method=\"NB\"):\n",
    "    global result\n",
    "    if method == \"NB\":\n",
    "        # no need to resample any thing\n",
    "        X_train, X_test, y_train, y_test = ready_data(dataset, resample=False)\n",
    "        # X_train, X_test, y_train, y_test = train_test_split(inputs, targets, test_size=0.3, random_state=0)\n",
    "        result = naive_bayes_gaussian(X_train, y_train, X_test, y_test)\n",
    "    elif method == \"DT\":\n",
    "        pass\n",
    "    elif method == \"RF\":\n",
    "        pass\n",
    "    elif method == \"AB\":\n",
    "        pass\n",
    "    elif method == \"KNN\":\n",
    "        pass\n",
    "\n",
    "    for title in result:\n",
    "        print(f\"{title} --> {result.get(title)}\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "35559782",
   "metadata": {},
   "outputs": [],
   "source": [
    "class NaiveBayesGaussian:\n",
    "    def __init__(self):\n",
    "        self.ones_num = 0\n",
    "        self.zero_num = 0\n",
    "        self.ones_mean = None\n",
    "        self.zero_mean = None\n",
    "        self.ones_std = None\n",
    "        self.zero_std = None\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        X = torch.tensor(X)\n",
    "        y = torch.tensor(y)\n",
    "        self.ones_num = torch.count_nonzero(y).item()\n",
    "        self.zero_num = y.size()[0] - self.ones_num\n",
    "        X_0 = torch.zeros((self.zero_num, X.shape[1]))\n",
    "        X_1 = torch.zeros((self.ones_num, X.shape[1]))\n",
    "\n",
    "        i = 0\n",
    "        j = 0\n",
    "        for col, cls in enumerate(y):\n",
    "            if cls == 1:\n",
    "                X_1[i, :] = X[col, :]\n",
    "                i += 1\n",
    "            else:\n",
    "                X_0[j, :] = X[col, :]\n",
    "                j += 1\n",
    "\n",
    "        self.zero_mean = torch.mean(X_0, dim=0)\n",
    "        self.ones_mean = torch.mean(X_1, dim=0)\n",
    "        self.zero_std = torch.std(X_0, dim=0)\n",
    "        self.ones_std = torch.std(X_1, dim=0)\n",
    "\n",
    "    def predict(self, X):\n",
    "        predictions = torch.ones((X.shape[0], 1))\n",
    "        for i, instance in enumerate(X):\n",
    "            if self.predict_instance(instance) == 0:\n",
    "                predictions[i, 0] = 0\n",
    "        return predictions.numpy()\n",
    "\n",
    "    def predict_instance(self, x):\n",
    "        score_0 = math.log(self.zero_num / (self.zero_num + self.ones_num))\n",
    "\n",
    "        for (point, mean, std) in zip(x, self.zero_mean, self.zero_std):\n",
    "            score_0 += norm.logpdf(point, loc=mean, scale=std)\n",
    "\n",
    "        score_1 = math.log(self.ones_num / (self.zero_num + self.ones_num))\n",
    "        for (point, mean, std) in zip(x, self.ones_mean, self.ones_std):\n",
    "            score_1 += norm.logpdf(point, loc=mean, scale=std)\n",
    "\n",
    "        if score_1 > score_0:\n",
    "            return 1.0\n",
    "        else:\n",
    "            return 0.0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "3b1cc020",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_result(true, predictions):\n",
    "    (tn, fp), (fn, tp) = confusion_matrix(true, predictions)\n",
    "    result = {\n",
    "        \"f\": f1_score(true, predictions, pos_label=1.0),\n",
    "        \"recall\": recall_score(true, predictions, pos_label=1.0),\n",
    "        \"accuracy\": accuracy_score(true, predictions),\n",
    "        \"precision\": precision_score(true, predictions, pos_label=1.0),\n",
    "        \"tn\": tn,\n",
    "        \"fp\": fp,\n",
    "        \"fn\": fn,\n",
    "        \"tp\": tp\n",
    "    }\n",
    "    return result\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "30967584",
   "metadata": {},
   "outputs": [],
   "source": [
    "def naive_bayes_gaussian(X, y, X_test, y_test):\n",
    "    model = NaiveBayesGaussian()\n",
    "    model.fit(X, y)\n",
    "\n",
    "    predictions = model.predict(X_test)\n",
    "    return get_result(y_test, predictions)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "13eda17b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f --> 0.5092180546726001\n",
      "recall --> 0.3922624877571009\n",
      "accuracy --> 0.7294076410795653\n",
      "precision --> 0.7255434782608695\n",
      "tn --> 3361\n",
      "fp --> 303\n",
      "fn --> 1241\n",
      "tp --> 801\n"
     ]
    }
   ],
   "source": [
    "main(dataset, \"NB\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "e068ade4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def weighted_impurity_temp(true_correct, true_wrong, false_correct, false_wrong):\n",
    "    gini_true_leaf = 1 - (true_correct / (true_wrong + true_correct)) ** 2 - (\n",
    "            true_wrong / (true_wrong + true_correct)) ** 2\n",
    "    gini_false_leaf = 1 - (false_correct / (false_wrong + false_correct)) ** 2 - (\n",
    "            false_wrong / (false_wrong + false_correct)) ** 2\n",
    "\n",
    "    true_weight = (true_correct + true_wrong) / (true_correct + true_wrong + false_correct + false_wrong)\n",
    "    false_weight = (false_correct + false_wrong) / (true_correct + true_wrong + false_correct + false_wrong)\n",
    "\n",
    "    return gini_true_leaf * true_weight + gini_false_leaf * false_weight\n",
    "\n",
    "\n",
    "def is_numeric(value):\n",
    "    \"\"\"Test if a value is numeric.\"\"\"\n",
    "    return isinstance(value, int) or isinstance(value, float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "02bb6193",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = np.array([\"ahmed\", \"kamal\", \"hepalla\", 3,4.5,\"howa\",\"ahmed\",\"ahmed\",\"kamal\",3])\n",
    "training_data = np.array([\n",
    "    [0, 3, 0],\n",
    "    [1, 3, 0],\n",
    "    [2, 1, 1],\n",
    "    [2, 1, 1],\n",
    "    [1, 3, 2.],\n",
    "])\n",
    "m_1 = [\n",
    "    (np.array([105,39]),0),\n",
    "    (np.array([34,125]),1)\n",
    "]\n",
    "m_2 = [\n",
    "    (np.array([37,127]),0),\n",
    "    (np.array([100,33]),1)\n",
    "]\n",
    "m_3 = [\n",
    "    (np.array([92,31]),0),\n",
    "    (np.array([45,129]),1)\n",
    "]\n",
    "state_quest = np.array([\n",
    "    [220,1],\n",
    "    [180,1],\n",
    "    [225,1],\n",
    "    [190,0],\n",
    "    [155,0]\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b722341d",
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs ,targets = ready_data(dataset,split=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "5052a0e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = ready_data(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "id": "4dd2cc73",
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs , targets = training_data[:,0:-1] , training_data[:,-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "cb0f8b23",
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs , targets = state_quest[:,0:-1] , state_quest[:,-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "95b9c784",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = torch.tensor(X_train[:,0])\n",
    "y = torch.tensor(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "id": "8ee30fb0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def weighted_impurity(counts):\n",
    "    total_weighted_gini = 0\n",
    "    sums = np.zeros(len(counts))\n",
    "    for i, (leaf, _) in enumerate(counts):\n",
    "        sums[i] = np.sum(leaf)\n",
    "    for i, (leaf, _) in enumerate(counts):\n",
    "        gini = 1\n",
    "        for j in leaf:\n",
    "            gini -= (j / sums[i]) ** 2\n",
    "        weight = sums[i] / np.sum(sums)\n",
    "        total_weighted_gini += gini * weight\n",
    "    return total_weighted_gini"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "289e4d03",
   "metadata": {},
   "outputs": [],
   "source": [
    "def count(x, y):\n",
    "    # if is_numeric(x):\n",
    "    merged_data = (torch.vstack((x, y))).t()\n",
    "    sorted_data = merged_data[merged_data[:, 0].argsort()].detach().clone()\n",
    "\n",
    "    classes = np.array(list(sorted(set(y.numpy()))))\n",
    "    leaf = np.zeros(classes.shape)\n",
    "    possible_gini_counts = []\n",
    "\n",
    "    partition = len(sorted_data) / 100\n",
    "    for part in np.arange(partition, 99 * partition, partition):\n",
    "        part = int(part)\n",
    "        avg = (sorted_data[part, 0] + sorted_data[part + int(partition), 0]) / 2\n",
    "        count_stump = [(leaf.copy(), answer) for answer in [True, False]]\n",
    "\n",
    "        for instance in sorted_data:\n",
    "            cls_index = np.where(classes == instance[1].item())[0][0]\n",
    "            if instance[0].item() < avg:\n",
    "                # true:: left branch\n",
    "                count_stump[0][0][cls_index] += 1\n",
    "            else:\n",
    "                # false:: right branch\n",
    "                count_stump[1][0][cls_index] += 1\n",
    "\n",
    "        possible_gini_counts.append([weighted_impurity(count_stump), avg, count_stump])\n",
    "        # print(f\"\\t\\tstump building in progress: {(part / (100 * partition)):.2%}\")\n",
    "\n",
    "    gini_avg_cutoff, separation_point, counts = sorted(possible_gini_counts, key=lambda record: record[0])[0]\n",
    "    # print(f\"gini: {gini_avg_cutoff}, separation point: {separation_point} \\n--> {counts}\")\n",
    "    return (gini_avg_cutoff, float(separation_point.item())), (counts, classes)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "id": "b064a550",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class Stump:\n",
    "    def __init__(self, X, y, weights, root_feature=0):\n",
    "        # self._true_correct = 0\n",
    "        # self._true_wrong = 0\n",
    "        # self._false_correct = 0\n",
    "        # self._false_wrong = 0\n",
    "        # self.x =\n",
    "        # self.y =\n",
    "        # self.total_gini = weighted_impurity(self.counts)\n",
    "        # build the stump\n",
    "        print(f\"\\tstart creating trial stump {root_feature}#\")\n",
    "        self._root_feature = root_feature\n",
    "        (self._total_gini, self._separation_point), (self._counts, self._classes) = count(X[:, root_feature], y)\n",
    "        self._stump_weight = self.calc_stump_weight(X[:, root_feature], y, weights)\n",
    "        self._new_weights = self.get_new_weights(X[:, root_feature], y, weights)\n",
    "        print(f\"\\tfinish creating trial stump {root_feature}#\")\n",
    "\n",
    "\n",
    "    @property\n",
    "    def new_weights(self):\n",
    "        return self._new_weights\n",
    "\n",
    "    @property\n",
    "    def stump_weight(self):\n",
    "        return self._stump_weight\n",
    "\n",
    "    @property\n",
    "    def root_feature(self):\n",
    "        return self._root_feature\n",
    "\n",
    "    @property\n",
    "    def total_gini(self):\n",
    "        return self._total_gini\n",
    "\n",
    "    @property\n",
    "    def counts(self):\n",
    "        return self._counts\n",
    "\n",
    "    @property\n",
    "    def classes(self):\n",
    "        return self._classes\n",
    "\n",
    "    @property\n",
    "    def separation_point(self):\n",
    "        return self._separation_point\n",
    "    # works\n",
    "    def _predict(self, x):\n",
    "        index = 1\n",
    "        if x < self.separation_point:\n",
    "            index = 0\n",
    "        side = self.classes[int(self.counts[index][0].argsort()[-1])]\n",
    "        return side\n",
    "\n",
    "    # predict an instance using the stump, works\n",
    "    def predict(self, x):\n",
    "        x_instance = x[self.root_feature].item()\n",
    "        return self._predict(x_instance)\n",
    "    # works\n",
    "    def calc_stump_weight(self, x, y, weights, error_term=1e-15):\n",
    "        total_error = sum([((self._predict(x[i].item()) != y[i].item()) and weights[i].item()) for i in range(len(y))])\n",
    "        weight = 0.5 * math.log((1 - total_error + error_term) / (total_error + error_term))\n",
    "        return weight\n",
    "\n",
    "    def get_new_weights(self, x, y, weights):\n",
    "        new_weights = torch.clone(weights)\n",
    "        exponents = torch.tensor([(-1. if (self._predict(x[i].item()) == y[i].item()) else 1.) for i in range(len(weights))]) * self.stump_weight\n",
    "        new_weights = new_weights * (math.e ** exponents)\n",
    "        return new_weights / new_weights.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "id": "3f9280b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the best stump using the available dat\n",
    "def get_best_stump(X, y, data_weights, features_not_included):\n",
    "    print(f\"create the {len(features_not_included)}# stump\")\n",
    "    stumps = {}\n",
    "    for root_feature in (set(range(X.shape[1])) - set(features_not_included)):\n",
    "        stump = Stump(X, y, data_weights, root_feature=root_feature)\n",
    "        stumps[stump.total_gini] = stump\n",
    "    chosen = stumps.get(sorted(stumps)[0])\n",
    "    print(f\"chosen stump with weight: {chosen.stump_weight}, and its gini: {chosen.total_gini}\" )\n",
    "    return chosen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "id": "632b31d4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tstart creating trial stump 0#\n",
      "sum_weight:  1.0\n",
      "total_error: 0.38513296796008945\n",
      "stump_weight: 0.2339086951896266\n",
      "\tfinish creating trial stump 0#\n"
     ]
    }
   ],
   "source": [
    "X = torch.tensor(X_train)\n",
    "y = torch.tensor(y_train)\n",
    "weight = torch.ones(X.shape[0]) / float(X.shape[0])\n",
    "stump = Stump(X, y,weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "id": "c40c0bd1",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# print(weight)\n",
    "# print(stump.new_weights) #didn't work\n",
    "# stump.stump_weight\n",
    "# stump.separation_point\n",
    "# stump.classes\n",
    "# stump.counts\n",
    "# stump.total_gini\n",
    "# stump.root_feature\n",
    "# new_stump = Stump(X,y,weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19f4c234",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "id": "a534bd83",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "create the 0# stump\n",
      "\tstart creating trial stump 0#\n",
      "sum_weight:  1.0\n",
      "total_error: 0.38513296796008945\n",
      "stump_weight: 0.2339086951896266\n",
      "\tfinish creating trial stump 0#\n",
      "\tstart creating trial stump 1#\n",
      "sum_weight:  1.0\n",
      "total_error: 0.398376586381346\n",
      "stump_weight: 0.2061169679659755\n",
      "\tfinish creating trial stump 1#\n",
      "\tstart creating trial stump 2#\n",
      "sum_weight:  1.0\n",
      "total_error: 0.451991880312562\n",
      "stump_weight: 0.09631294401275386\n",
      "\tfinish creating trial stump 2#\n",
      "\tstart creating trial stump 3#\n",
      "sum_weight:  1.0\n",
      "total_error: 0.4680123864673078\n",
      "stump_weight: 0.06406272192970792\n",
      "\tfinish creating trial stump 3#\n",
      "\tstart creating trial stump 4#\n",
      "sum_weight:  1.0\n",
      "total_error: 0.4687600100878626\n",
      "stump_weight: 0.06256147281602917\n",
      "\tfinish creating trial stump 4#\n",
      "\tstart creating trial stump 5#\n",
      "sum_weight:  1.0\n",
      "total_error: 0.43842785176821053\n",
      "stump_weight: 0.12377249688421173\n",
      "\tfinish creating trial stump 5#\n",
      "\tstart creating trial stump 6#\n",
      "sum_weight:  1.0\n",
      "total_error: 0.4029691314790398\n",
      "stump_weight: 0.1965544279671782\n",
      "\tfinish creating trial stump 6#\n",
      "\tstart creating trial stump 7#\n",
      "sum_weight:  1.0\n",
      "total_error: 0.44601089134812355\n",
      "stump_weight: 0.10840082764796484\n",
      "\tfinish creating trial stump 7#\n",
      "\tstart creating trial stump 8#\n",
      "sum_weight:  1.0\n",
      "total_error: 0.27063975064083934\n",
      "stump_weight: 0.49568958803032437\n",
      "\tfinish creating trial stump 8#\n",
      "\tstart creating trial stump 9#\n",
      "sum_weight:  1.0\n",
      "total_error: 0.47175050457008183\n",
      "stump_weight: 0.05655922375190313\n",
      "\tfinish creating trial stump 9#\n",
      "list of weights to choose form\n",
      "\t0.44599197727158624\n",
      "\t0.45227161986304415\n",
      "\t0.48958738321120115\n",
      "\t0.4920622240700126\n",
      "\t0.49259025366612896\n",
      "\t0.47531123511534373\n",
      "\t0.4592402315810326\n",
      "\t0.47885699064469484\n",
      "\t0.3935627981141744\n",
      "\t0.4949403203200257\n",
      "chosen stump with weight: 0.49568958803032437, and its gini: 0.3935627981141744\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<__main__.Stump at 0x21c4f191460>"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_best_stump(X,y,weight,[])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "id": "890ccea8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'torch.Tensor'>\n",
      "<class 'torch.Tensor'>\n",
      "<class 'torch.Tensor'>\n",
      "<class 'torch.Tensor'>\n",
      "<class 'torch.Tensor'>\n",
      "<class 'torch.Tensor'>\n",
      "<class 'torch.Tensor'>\n",
      "<class 'torch.Tensor'>\n",
      "<class 'torch.Tensor'>\n",
      "<class 'torch.Tensor'>\n"
     ]
    }
   ],
   "source": [
    "for key in stumps:\n",
    "    print(type(stumps.get(key).new_weights.sum()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37c295d1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}