{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from torch import nn, optim\n",
    "from torch.functional import F\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import transforms, datasets\n",
    "\n",
    "%config IPCompleter.greedy = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# download FashionMNIST dataset\n",
    "\n",
    "datasets.FashionMNIST.resources = [\n",
    "    (\"http://kleshwong.com/download/train-images-idx3-ubyte.gz\",\n",
    "     \"8d4fb7e6c68d591d4c3dfef9ec88bf0d\"),\n",
    "    (\"http://kleshwong.com/download/train-labels-idx1-ubyte.gz\",\n",
    "     \"25c81989df183df01b3e8a0aad5dffbe\"),\n",
    "    (\"http://kleshwong.com/download/t10k-images-idx3-ubyte.gz\",\n",
    "     \"bef4ecab320f06d8554ea6380940ec79\"),\n",
    "    (\"http://kleshwong.com/download/t10k-labels-idx1-ubyte.gz\",\n",
    "     \"bb300cfdad3c16e7a12a480ee83cd310\")\n",
    "]\n",
    "\n",
    "train_set = datasets.FashionMNIST(\n",
    "    root=\"./data\",\n",
    "    download=True,\n",
    "    train=True,\n",
    "    transform=transforms.Compose([\n",
    "        transforms.ToTensor()\n",
    "    ])\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f8d966bf820>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 482360 characters]\n",
      "text/plain": [
       "<Figure size 1080x1080 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# observing dataset\n",
    "\n",
    "images, labels = next(iter(DataLoader(train_set, batch_size=100)))\n",
    "\n",
    "grid = torchvision.utils.make_grid(images, nrow=10)\n",
    "\n",
    "plt.figure(figsize=(15, 15))\n",
    "plt.imshow(np.transpose(grid, (1, 2, 0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "networ Network(\n",
      "  (conv1): Conv2d(1, 6, kernel_size=(3, 3), stride=(1, 1))\n",
      "  (conv2): Conv2d(6, 12, kernel_size=(3, 3), stride=(1, 1))\n",
      "  (fc1): Linear(in_features=432, out_features=120, bias=True)\n",
      "  (fc2): Linear(in_features=120, out_features=60, bias=True)\n",
      "  (out): Linear(in_features=60, out_features=10, bias=True)\n",
      ")\n",
      "conv1.weight \t\t torch.Size([6, 1, 3, 3])\n",
      "conv1.bias \t\t torch.Size([6])\n",
      "conv2.weight \t\t torch.Size([12, 6, 3, 3])\n",
      "conv2.bias \t\t torch.Size([12])\n",
      "fc1.weight \t\t torch.Size([120, 432])\n",
      "fc1.bias \t\t torch.Size([120])\n",
      "fc2.weight \t\t torch.Size([60, 120])\n",
      "fc2.bias \t\t torch.Size([60])\n",
      "out.weight \t\t torch.Size([10, 60])\n",
      "out.bias \t\t torch.Size([10])\n"
     ]
    }
   ],
   "source": [
    "# define network\n",
    "class Network(nn.Module):\n",
    "    \n",
    "    def __init__(self, kernel_size=5):\n",
    "        super().__init__()\n",
    "        self.pool_stride = (2, 2)\n",
    "        w = h = 28\n",
    "        o = lambda n, f, p, s: (n - f + 2*p) // s + 1\n",
    "        self.conv1 = nn.Conv2d(in_channels=1, out_channels=6, kernel_size=kernel_size)\n",
    "        w = o(w, kernel_size, 0, self.pool_stride[0])\n",
    "        h = o(h, kernel_size, 0, self.pool_stride[1])\n",
    "        \n",
    "        self.conv2 = nn.Conv2d(in_channels=6, out_channels=12, kernel_size=kernel_size)\n",
    "        w = o(w, kernel_size, 0, self.pool_stride[0])\n",
    "        h = o(h, kernel_size, 0, self.pool_stride[1])\n",
    "        \n",
    "        self.fc1 = nn.Linear(in_features=12*w*h, out_features=120)\n",
    "        self.fc2 = nn.Linear(in_features=120, out_features=60)\n",
    "        self.out = nn.Linear(in_features=60, out_features=10)\n",
    "        \n",
    "        self.w = w\n",
    "        self.h = h\n",
    "        \n",
    "    def forward(self, x):\n",
    "        # conv layer #1\n",
    "        x = self.conv1(x)\n",
    "        x = F.relu(x)\n",
    "        x = F.max_pool2d(x, kernel_size=2, stride=2)\n",
    "        \n",
    "        # conv layer #2\n",
    "        x = self.conv2(x)\n",
    "        x = F.relu(x)\n",
    "        x = F.max_pool2d(x, kernel_size=2, stride=2)\n",
    "        \n",
    "        # linear layer #1\n",
    "        x = x.reshape(-1, 12*self.w*self.h)\n",
    "        x = self.fc1(x)\n",
    "        x = F.relu(x)\n",
    "        \n",
    "        # linear layer #2\n",
    "        x = self.fc2(x)\n",
    "        x = F.relu(x)\n",
    "        \n",
    "        # output layer\n",
    "        x = self.out(x)\n",
    "        \n",
    "        return x\n",
    "    \n",
    "network = Network(kernel_size=3)\n",
    "print('networ', network)\n",
    "for name, param in network.named_parameters():\n",
    "    print(name, '\\t\\t', param.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# epoch\n",
    "\n",
    "def epoch(epoch_id, network, optimizer, loader, run):\n",
    "    total_loss = 0\n",
    "    total_correct = 0\n",
    "    for images, labels in loader:\n",
    "        images = images.cuda()\n",
    "        labels = labels.cuda()\n",
    "        preds = network(images)\n",
    "        loss = F.cross_entropy(preds, labels)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            total_loss += loss.item()\n",
    "            total_correct += preds.argmax(dim=1).eq(labels).sum().item()\n",
    "    return total_loss, total_correct\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run(batch_size=64, lr=0.005)\t epoch 0\t loss: 553.2858839333057\t accuracy: 0.7750333333333334\n",
      "Run(batch_size=64, lr=0.005)\t epoch 1\t loss: 358.71706852316856\t accuracy: 0.8585\n",
      "Run(batch_size=64, lr=0.01)\t epoch 0\t loss: 532.5651339441538\t accuracy: 0.7883166666666667\n",
      "Run(batch_size=64, lr=0.01)\t epoch 1\t loss: 375.26991252601147\t accuracy: 0.8519333333333333\n",
      "Run(batch_size=128, lr=0.005)\t epoch 0\t loss: 308.8334187567234\t accuracy: 0.7493333333333333\n",
      "Run(batch_size=128, lr=0.005)\t epoch 1\t loss: 191.86540931463242\t accuracy: 0.84855\n",
      "Run(batch_size=128, lr=0.01)\t epoch 0\t loss: 281.14268481731415\t accuracy: 0.7702833333333333\n",
      "Run(batch_size=128, lr=0.01)\t epoch 1\t loss: 185.1960816681385\t accuracy: 0.8537\n"
     ]
    }
   ],
   "source": [
    "# finding hyper parameters\n",
    "\n",
    "from collections import namedtuple, OrderedDict\n",
    "from itertools import product\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "\n",
    "hyper_params = OrderedDict(\n",
    "    batch_size=[64, 128],\n",
    "    lr=[0.001, 0.01],\n",
    ")\n",
    "\n",
    "Run = namedtuple('Run', hyper_params.keys())\n",
    "runs = [Run(*values) for values in product(*hyper_params.values())]\n",
    "\n",
    "\n",
    "# loop through all combinations\n",
    "for run in runs:\n",
    "    network = Network().cuda()\n",
    "    optimizer = optim.Adam(network.parameters(), lr=run.lr)\n",
    "    loader = DataLoader(train_set, batch_size=run.batch_size, shuffle=True, num_workers=1)\n",
    "    for epoch_id in range(2):\n",
    "        total_loss, total_correct = epoch(epoch_id, network, optimizer, loader, run)\n",
    "        print(f'{run}\\t epoch {epoch_id}\\t loss: {total_loss}\\t accuracy: {total_correct/len(train_set)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'pandas'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-38d4b0363d82>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'pandas'"
     ]
    }
   ],
   "source": [
    "import pandas"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
