EVA5Session4_v3_3.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "EVA4Session2_v3_3.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/thamizhannal/eva5/blob/master/EVA5Session4_v3_3.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0m2JWFliFfKT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from __future__ import print_function\n",
        "import torch\n",
        "import torch.nn as nn \n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from torchvision import datasets, transforms"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FqrLhiEi5iPT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        },
        "outputId": "20b7e411-4259-4af4-d695-9128a79c9182"
      },
      "source": [
        "!pip install torchsummary\n",
        "from torchsummary import summary\n"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: torchsummary in /usr/local/lib/python3.6/dist-packages (1.5.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h_Cx9q2QFgM7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Net(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(Net, self).__init__()\n",
        "        \n",
        "        self.conv1 = nn.Conv2d(in_channels = 1, out_channels=8, kernel_size=3, padding=0) \n",
        "        self.bn1 = nn.BatchNorm2d(8)\n",
        "        \n",
        "        self.conv2 = nn.Conv2d(in_channels = 8, out_channels=16, kernel_size=3, padding=0) #24\n",
        "        self.bn2 = nn.BatchNorm2d(16)\n",
        "        #self.dp1 = nn.Dropout2d(0.1)\n",
        "\n",
        "        self.conv3 = nn.Conv2d(in_channels = 16, out_channels=20, kernel_size=3, padding=0) # 22\n",
        "        self.bn3 = nn.BatchNorm2d(20)\n",
        "        #self.dp2 = nn.Dropout2d(0.1)\n",
        "\n",
        "\n",
        "        self.pool1 = nn.MaxPool2d(kernel_size=2) # 11\n",
        "        # Apply 1x1 & reduce channel size\n",
        "\n",
        "        self.conv4 = nn.Conv2d(in_channels = 20, out_channels=8, kernel_size=1, padding=0)\n",
        "        self.bn4 = nn.BatchNorm2d(8)\n",
        "        \n",
        "        self.conv5 = nn.Conv2d(in_channels = 8, out_channels=16, kernel_size=3, padding=0)\n",
        "        self.bn5 = nn.BatchNorm2d(16)\n",
        "        #self.dp3 = nn.Dropout2d(0.1)\n",
        "\n",
        "        self.conv6 = nn.Conv2d(in_channels = 16, out_channels=20, kernel_size=3, padding=0) # nn.Conv2d(in_channels=24, out_channels=16, kernel_size=1, padding=1)\n",
        "        self.bn6 = nn.BatchNorm2d(20)\n",
        "        #self.dp4 = nn.Dropout2d(0.1)\n",
        "\n",
        "        self.pool2 = nn.AvgPool2d(kernel_size=2)\n",
        "        \n",
        "        # Apply 1x1 & reduce channel size\n",
        "        \n",
        "        #self.conv8 = nn.Conv2d(24, 24, 1, padding=0)\n",
        "        #self.bn8 = nn.BatchNorm2d(24)\n",
        "        \n",
        "        #self.conv6 = nn.Conv2d(16, 32, 3)\n",
        "        #self.bn6 = nn.BatchNorm2d(32)\n",
        "\n",
        "        self.conv7 = nn.Conv2d(in_channels=20, out_channels=10, kernel_size=3)\n",
        "        \n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.pool1(F.relu(self.bn3(self.conv3(F.relu(self.bn2(self.conv2(F.relu(self.bn1(self.conv1(x))))))))))\n",
        "        #x = self.bn7(self.conv1_1x1(x))\n",
        "        #x = self.pool2(F.relu(self.bn6(self.conv6(F.relu(self.bn5(self.conv5(F.relu(self.bn4(self.conv4(x))))))))))\n",
        "        #x = self.pool2(F.relu(self.bn6(self.conv6(F.relu(self.bn5(self.conv5(F.relu(self.bn4(self.conv4(x))))))))))\n",
        "        x = self.pool2(F.relu(self.bn6(self.conv6(F.relu(F.relu(self.bn5(self.conv5(F.relu(self.bn4(self.conv4(x)))))))))))\n",
        "        # F.relu(self.bn6(self.conv6(F.relu(\n",
        "        #x = F.relu(self.bn8(self.conv8(x)))\n",
        "        #x = F.relu(self.bn6(self.conv6(F.relu(self.bn5(self.conv5(x))))))\n",
        "        #x = F.relu(self.bn6(self.conv6(F.relu(self.bn5(self.conv5(x))))))\n",
        "        #x = F.relu(self.conv7(x))\n",
        "        x = self.conv7(x)\n",
        "        x = x.view(-1, 10)\n",
        "        return F.log_softmax(x)"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "xdydjYTZFyi3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 522
        },
        "outputId": "d5b39507-2cd3-417a-a745-14dcc6880a69"
      },
      "source": [
        "use_cuda = torch.cuda.is_available()\n",
        "device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
        "model = Net().to(device)\n",
        "summary(model, input_size=(1, 28, 28))"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----------------------------------------------------------------\n",
            "        Layer (type)               Output Shape         Param #\n",
            "================================================================\n",
            "            Conv2d-1            [-1, 8, 26, 26]              80\n",
            "       BatchNorm2d-2            [-1, 8, 26, 26]              16\n",
            "            Conv2d-3           [-1, 16, 24, 24]           1,168\n",
            "       BatchNorm2d-4           [-1, 16, 24, 24]              32\n",
            "            Conv2d-5           [-1, 20, 22, 22]           2,900\n",
            "       BatchNorm2d-6           [-1, 20, 22, 22]              40\n",
            "         MaxPool2d-7           [-1, 20, 11, 11]               0\n",
            "            Conv2d-8            [-1, 8, 11, 11]             168\n",
            "       BatchNorm2d-9            [-1, 8, 11, 11]              16\n",
            "           Conv2d-10             [-1, 16, 9, 9]           1,168\n",
            "      BatchNorm2d-11             [-1, 16, 9, 9]              32\n",
            "           Conv2d-12             [-1, 20, 7, 7]           2,900\n",
            "      BatchNorm2d-13             [-1, 20, 7, 7]              40\n",
            "        AvgPool2d-14             [-1, 20, 3, 3]               0\n",
            "           Conv2d-15             [-1, 10, 1, 1]           1,810\n",
            "================================================================\n",
            "Total params: 10,370\n",
            "Trainable params: 10,370\n",
            "Non-trainable params: 0\n",
            "----------------------------------------------------------------\n",
            "Input size (MB): 0.00\n",
            "Forward/backward pass size (MB): 0.44\n",
            "Params size (MB): 0.04\n",
            "Estimated Total Size (MB): 0.48\n",
            "----------------------------------------------------------------\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:57: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DqTWLaM5GHgH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "torch.manual_seed(1)\n",
        "batch_size = 32\n",
        "\n",
        "kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}\n",
        "train_loader = torch.utils.data.DataLoader(\n",
        "    datasets.MNIST('../data', train=True, download=True,\n",
        "                    transform=transforms.Compose([\n",
        "                        transforms.ToTensor(),\n",
        "                        transforms.Normalize((0.1307,), (0.3081,))\n",
        "                    ])),\n",
        "    batch_size=batch_size, shuffle=True, **kwargs)\n",
        "test_loader = torch.utils.data.DataLoader(\n",
        "    datasets.MNIST('../data', train=False, transform=transforms.Compose([\n",
        "                        transforms.ToTensor(),\n",
        "                        transforms.Normalize((0.1307,), (0.3081,))\n",
        "                    ])),\n",
        "    batch_size=batch_size, shuffle=True, **kwargs)\n"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8fDefDhaFlwH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tqdm import tqdm\n",
        "\n",
        "def train(model, device, train_loader, optimizer, epoch):\n",
        "    train_loss = 0\n",
        "    train_correct = 0\n",
        "    model.train()\n",
        "    pbar = tqdm(train_loader)\n",
        "    for batch_idx, (data, target) in enumerate(pbar):\n",
        "        data, target = data.to(device), target.to(device)\n",
        "        optimizer.zero_grad()\n",
        "        output = model(data)\n",
        "        loss = F.nll_loss(output, target)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        pbar.set_description(desc= f'loss={loss.item()} batch_id={batch_idx}')\n",
        "        \n",
        "        #train_loss += F.nll_loss(output, target, reduction='sum').item()  # sum up batch loss\n",
        "        #train_pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability\n",
        "        #train_correct += train_pred.eq(target.view_as(train_pred)).sum().item()\n",
        "\n",
        "    #train_losss /= len(train_loader.dataset)\n",
        "    #print('\\nTrain Set: Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)'.format(\n",
        "    #    train_loss, train_correct, len(train_loader.dataset),\n",
        "    #    100. * train_correct / len(train_loader.dataset)))\n",
        "       \n",
        "def test(model, device, test_loader):\n",
        "    model.eval()\n",
        "    test_loss = 0\n",
        "    correct = 0\n",
        "    with torch.no_grad():\n",
        "        for data, target in test_loader:\n",
        "            data, target = data.to(device), target.to(device)\n",
        "            output = model(data)\n",
        "            test_loss += F.nll_loss(output, target, reduction='sum').item()  # sum up batch loss\n",
        "            pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability\n",
        "            correct += pred.eq(target.view_as(pred)).sum().item()\n",
        "\n",
        "    test_loss /= len(test_loader.dataset)\n",
        "\n",
        "    print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)'.format(\n",
        "        test_loss, correct, len(test_loader.dataset),\n",
        "        100. * correct / len(test_loader.dataset)))"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MMWbLWO6FuHb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "6738dc76-855e-46e7-dc37-5dda4d38f4d3"
      },
      "source": [
        "\n",
        "model = Net().to(device)\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)\n",
        "\n",
        "for epoch in range(1, 20):\n",
        "    print(\"epoch:{}\".format(epoch))\n",
        "    train(model, device, train_loader, optimizer, epoch)\n",
        "    test(model, device, test_loader)"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r  0%|          | 0/1875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "epoch:1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:57: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n",
            "loss=0.02032921463251114 batch_id=1874: 100%|██████████| 1875/1875 [00:22<00:00, 82.28it/s]\n",
            "  0%|          | 0/1875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0404, Accuracy: 9865/10000 (98.65%)\n",
            "epoch:2\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.10088248550891876 batch_id=1874: 100%|██████████| 1875/1875 [00:22<00:00, 83.47it/s]\n",
            "  0%|          | 0/1875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0308, Accuracy: 9900/10000 (99.00%)\n",
            "epoch:3\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.0294856708496809 batch_id=1874: 100%|██████████| 1875/1875 [00:22<00:00, 83.03it/s]\n",
            "  0%|          | 0/1875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0283, Accuracy: 9909/10000 (99.09%)\n",
            "epoch:4\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.008544371463358402 batch_id=1874: 100%|██████████| 1875/1875 [00:22<00:00, 82.13it/s]\n",
            "  0%|          | 0/1875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0242, Accuracy: 9913/10000 (99.13%)\n",
            "epoch:5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.03578805923461914 batch_id=1874: 100%|██████████| 1875/1875 [00:22<00:00, 82.63it/s]\n",
            "  0%|          | 0/1875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0243, Accuracy: 9924/10000 (99.24%)\n",
            "epoch:6\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.0016344727482646704 batch_id=1874: 100%|██████████| 1875/1875 [00:22<00:00, 82.74it/s]\n",
            "  0%|          | 0/1875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0306, Accuracy: 9909/10000 (99.09%)\n",
            "epoch:7\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.0012250362196937203 batch_id=1874: 100%|██████████| 1875/1875 [00:22<00:00, 81.66it/s]\n",
            "  0%|          | 0/1875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0234, Accuracy: 9916/10000 (99.16%)\n",
            "epoch:8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.002004024339839816 batch_id=1874: 100%|██████████| 1875/1875 [00:22<00:00, 83.42it/s]\n",
            "  0%|          | 0/1875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0220, Accuracy: 9927/10000 (99.27%)\n",
            "epoch:9\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.000856564671266824 batch_id=1874: 100%|██████████| 1875/1875 [00:24<00:00, 77.11it/s]\n",
            "  0%|          | 0/1875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0219, Accuracy: 9929/10000 (99.29%)\n",
            "epoch:10\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.00022139737848192453 batch_id=1874: 100%|██████████| 1875/1875 [00:25<00:00, 73.59it/s]\n",
            "  0%|          | 0/1875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0201, Accuracy: 9935/10000 (99.35%)\n",
            "epoch:11\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.00024380745890084654 batch_id=1874: 100%|██████████| 1875/1875 [00:25<00:00, 73.74it/s]\n",
            "  0%|          | 0/1875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0216, Accuracy: 9932/10000 (99.32%)\n",
            "epoch:12\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.07573018968105316 batch_id=1874: 100%|██████████| 1875/1875 [00:24<00:00, 77.14it/s]\n",
            "  0%|          | 0/1875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0274, Accuracy: 9915/10000 (99.15%)\n",
            "epoch:13\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.0007794589619152248 batch_id=1874: 100%|██████████| 1875/1875 [00:25<00:00, 74.61it/s]\n",
            "  0%|          | 0/1875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0260, Accuracy: 9915/10000 (99.15%)\n",
            "epoch:14\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.0060095032677054405 batch_id=1874: 100%|██████████| 1875/1875 [00:26<00:00, 69.93it/s]\n",
            "  0%|          | 0/1875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0184, Accuracy: 9946/10000 (99.46%)\n",
            "epoch:15\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.016123449429869652 batch_id=1874: 100%|██████████| 1875/1875 [00:26<00:00, 72.07it/s]\n",
            "  0%|          | 0/1875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0208, Accuracy: 9933/10000 (99.33%)\n",
            "epoch:16\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.0016812128014862537 batch_id=1874: 100%|██████████| 1875/1875 [00:25<00:00, 72.83it/s]\n",
            "  0%|          | 0/1875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0225, Accuracy: 9926/10000 (99.26%)\n",
            "epoch:17\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.002580386120826006 batch_id=1874: 100%|██████████| 1875/1875 [00:25<00:00, 74.19it/s]\n",
            "  0%|          | 0/1875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0235, Accuracy: 9932/10000 (99.32%)\n",
            "epoch:18\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=0.0020940324757248163 batch_id=1874: 100%|██████████| 1875/1875 [00:25<00:00, 74.51it/s]\n",
            "  0%|          | 0/1875 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0198, Accuracy: 9937/10000 (99.37%)\n",
            "epoch:19\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "loss=6.286335701588541e-05 batch_id=1874: 100%|██████████| 1875/1875 [00:24<00:00, 77.54it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 0.0208, Accuracy: 9934/10000 (99.34%)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "So5uk4EkHW6R",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}