synthetic_demo_verify_s1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3405bd25",
   "metadata": {
    "id": "3405bd25"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.nn.parameter import Parameter\n",
    "import math\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from math import log, e\n",
    "import torch.optim as optim\n",
    "import pickle\n",
    "import random\n",
    "\n",
    "torch.manual_seed(11)\n",
    "np.random.seed(11)\n",
    "random.seed(11)\n",
    "\n",
    "plt.rcParams.update({'font.size': 13})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "be1a23c5",
   "metadata": {
    "id": "be1a23c5"
   },
   "outputs": [],
   "source": [
    "class ConvNet(nn.Module):\n",
    "    def __init__(self, input_dim, out_channel, patch_num, small=True, activation='linear'):\n",
    "        super(ConvNet, self).__init__()\n",
    "        self.conv1 = nn.Conv1d(1, out_channel*2, int(input_dim/patch_num), int(input_dim/patch_num))        \n",
    "        # small initialization\n",
    "        if small:\n",
    "            self.conv1.weight = torch.nn.Parameter(self.conv1.weight*0.001) \n",
    "            self.conv1.bias = torch.nn.Parameter(self.conv1.bias*0.001) \n",
    "        self.out_channel = out_channel\n",
    "        self.activation = activation\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        if self.activation == 'cubic':\n",
    "            x = x**3\n",
    "        elif self.activation == 'relu':\n",
    "            x = F.relu(x)\n",
    "        elif self.activation == 'celu':\n",
    "            x = F.celu(x)\n",
    "        elif self.activation == 'gelu':\n",
    "            x = F.gelu(x)\n",
    "        elif self.activation == 'tanh':\n",
    "            x = torch.tanh(x)\n",
    "        x = torch.sum(x,2)\n",
    "        output = torch.stack([torch.sum(x[:,:self.out_channel],1), torch.sum(x[:,self.out_channel:],1)]).transpose(1,0)\n",
    "        return output"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "929bedec",
   "metadata": {
    "id": "929bedec"
   },
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3bc373c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_single(model, criterion, data, labels, optimizers, epochs):\n",
    "    \n",
    "    min_loss = float('inf')\n",
    "    \n",
    "    for epoch in range(epochs):  \n",
    "        for optimizer in optimizers:\n",
    "            optimizer.zero_grad()\n",
    "        outputs = model(data) \n",
    "        loss = criterion(outputs, labels) \n",
    "        \n",
    "        if loss.item() <= min_loss:\n",
    "            min_loss = loss.item()\n",
    "        elif epoch > 500 and loss > min_loss+0.02:\n",
    "            break\n",
    "        \n",
    "        loss.backward() \n",
    "                \n",
    "        for optimizer in optimizers:\n",
    "            optimizer.step()\n",
    "        \n",
    "        if epoch%100 == 0:   \n",
    "            print('Epoch %d --- loss: %.3f' %\n",
    "                    (epoch + 1, loss.item()))\n",
    "    print('Finished Training')\n",
    "\n",
    "    \n",
    "def test_single(model, criterion, data, labels):\n",
    "    correct = 0\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        outputs = model(data) # ,_\n",
    "        predicted = torch.max(outputs.data, 1).indices\n",
    "        correct += (predicted == labels).sum().item()\n",
    "\n",
    "    print('Accuracy of the network on the %d test images: %.4f %%' % (data.shape[0],\n",
    "        100 * correct / data.shape[0]))\n",
    "    \n",
    "    return 100 * correct / data.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "61aaef9c",
   "metadata": {
    "id": "61aaef9c"
   },
   "source": [
    "### Data Generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6cfa9699",
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_NUM = 16000\n",
    "CLUSTER_NUM = 4\n",
    "EXPERT_NUM = 8\n",
    "PATCH_NUM = 4\n",
    "PATCH_LEN = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "49b853f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_data = torch.load('synthetic_data_s1/train_data.pt')\n",
    "training_labels = torch.load('synthetic_data_s1/train_labels.pt')\n",
    "\n",
    "test_data = torch.load('synthetic_data_s1/test_data.pt')\n",
    "test_labels = torch.load('synthetic_data_s1/test_labels.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a94c844",
   "metadata": {},
   "source": [
    "### Experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fcb25871",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 --- loss: 7.830\n",
      "Epoch 101 --- loss: 0.528\n",
      "Epoch 201 --- loss: 0.528\n",
      "Epoch 301 --- loss: 0.528\n",
      "Epoch 401 --- loss: 0.537\n",
      "Epoch 501 --- loss: 0.532\n",
      "Epoch 601 --- loss: 0.535\n",
      "Epoch 701 --- loss: 0.531\n",
      "Epoch 801 --- loss: 0.530\n",
      "Finished Training\n",
      "Accuracy of the network on the 16000 test images: 66.2313 %\n",
      "Accuracy of the network on the 16000 test images: 65.8688 %\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "65.86875"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_epochs = 801\n",
    "\n",
    "linear_single = ConvNet(200, 64, PATCH_NUM, small=False).cuda() \n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "optimizer =  torch.optim.Adam(linear_single.parameters(), lr=0.001, weight_decay=5e-4) \n",
    "train_single(linear_single, criterion, training_data, training_labels, [optimizer], num_epochs)\n",
    "\n",
    "test_single(linear_single, criterion, training_data, training_labels)\n",
    "test_single(linear_single, criterion, test_data, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "045815de",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 --- loss: 95.751\n",
      "Epoch 101 --- loss: 2.389\n",
      "Epoch 201 --- loss: 0.747\n",
      "Epoch 301 --- loss: 0.398\n",
      "Epoch 401 --- loss: 0.317\n",
      "Epoch 501 --- loss: 0.420\n",
      "Finished Training\n",
      "Accuracy of the network on the 16000 test images: 83.7062 %\n",
      "Accuracy of the network on the 16000 test images: 75.5250 %\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "75.525"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_epochs = 801\n",
    "\n",
    "nonlinear_single = ConvNet(200, 64, PATCH_NUM, small=False, activation='cubic').cuda()\n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "optimizer =  torch.optim.Adam(nonlinear_single.parameters(), lr=0.01, weight_decay=5e-4) \n",
    "train_single(nonlinear_single, criterion, training_data, training_labels, \n",
    "                                                           [optimizer], num_epochs)\n",
    "\n",
    "test_single(nonlinear_single, criterion, training_data, training_labels)\n",
    "test_single(nonlinear_single, criterion, test_data, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fded964f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 --- loss: 8.672\n",
      "Epoch 101 --- loss: 0.593\n",
      "Epoch 201 --- loss: 1.270\n",
      "Epoch 301 --- loss: 0.207\n",
      "Epoch 401 --- loss: 0.202\n",
      "Epoch 501 --- loss: 0.452\n",
      "Finished Training\n",
      "Accuracy of the network on the 16000 test images: 76.4625 %\n",
      "Accuracy of the network on the 16000 test images: 72.2812 %\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "72.28125"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_epochs = 801\n",
    "\n",
    "nonlinear_single = ConvNet(200, 64, PATCH_NUM, small=False, activation='relu').cuda()\n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "optimizer =  torch.optim.Adam(nonlinear_single.parameters(), lr=0.01, weight_decay=5e-4) \n",
    "train_single(nonlinear_single, criterion, training_data, training_labels, \n",
    "                                                           [optimizer], num_epochs)\n",
    "\n",
    "test_single(nonlinear_single, criterion, training_data, training_labels)\n",
    "test_single(nonlinear_single, criterion, test_data, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ecfd5088",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 --- loss: 5.317\n",
      "Epoch 101 --- loss: 0.556\n",
      "Epoch 201 --- loss: 0.317\n",
      "Epoch 301 --- loss: 0.317\n",
      "Epoch 401 --- loss: 0.277\n",
      "Epoch 501 --- loss: 0.309\n",
      "Finished Training\n",
      "Accuracy of the network on the 16000 test images: 85.6375 %\n",
      "Accuracy of the network on the 16000 test images: 81.7500 %\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "81.75"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_epochs = 801\n",
    "\n",
    "nonlinear_single = ConvNet(200, 64, PATCH_NUM, small=False, activation='celu').cuda()\n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "optimizer =  torch.optim.Adam(nonlinear_single.parameters(), lr=0.01, weight_decay=5e-4) \n",
    "train_single(nonlinear_single, criterion, training_data, training_labels, \n",
    "                                                           [optimizer], num_epochs)\n",
    "\n",
    "test_single(nonlinear_single, criterion, training_data, training_labels)\n",
    "test_single(nonlinear_single, criterion, test_data, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "daebc39c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 --- loss: 3.824\n",
      "Epoch 101 --- loss: 0.782\n",
      "Epoch 201 --- loss: 0.377\n",
      "Epoch 301 --- loss: 0.251\n",
      "Epoch 401 --- loss: 0.600\n",
      "Epoch 501 --- loss: 0.300\n",
      "Finished Training\n",
      "Accuracy of the network on the 16000 test images: 82.7188 %\n",
      "Accuracy of the network on the 16000 test images: 79.0375 %\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "79.0375"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_epochs = 801\n",
    "\n",
    "nonlinear_single = ConvNet(200, 64, PATCH_NUM, small=False, activation='gelu').cuda()\n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "optimizer =  torch.optim.Adam(nonlinear_single.parameters(), lr=0.01, weight_decay=5e-4) \n",
    "train_single(nonlinear_single, criterion, training_data, training_labels, \n",
    "                                                           [optimizer], num_epochs)\n",
    "\n",
    "test_single(nonlinear_single, criterion, training_data, training_labels)\n",
    "test_single(nonlinear_single, criterion, test_data, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ea8d0b25",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 --- loss: 9.262\n",
      "Epoch 101 --- loss: 0.322\n",
      "Epoch 201 --- loss: 0.199\n",
      "Epoch 301 --- loss: 0.142\n",
      "Epoch 401 --- loss: 0.106\n",
      "Epoch 501 --- loss: 0.084\n",
      "Finished Training\n",
      "Accuracy of the network on the 16000 test images: 95.9938 %\n",
      "Accuracy of the network on the 16000 test images: 81.7188 %\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "81.71875"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_epochs = 801\n",
    "\n",
    "nonlinear_single = ConvNet(200, 64, PATCH_NUM, small=False, activation='tanh').cuda()\n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "optimizer =  torch.optim.Adam(nonlinear_single.parameters(), lr=0.01, weight_decay=5e-4) \n",
    "train_single(nonlinear_single, criterion, training_data, training_labels, \n",
    "                                                           [optimizer], num_epochs)\n",
    "\n",
    "test_single(nonlinear_single, criterion, training_data, training_labels)\n",
    "test_single(nonlinear_single, criterion, test_data, test_labels)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Copy of tensorpowernew.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}