ConvNets.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ConvNets.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Machine-Learning-Tokyo/DL-workshop-series/blob/master/ConvNets.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "0AHJ-a_r5unA",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Code for plotting the keras model graph\n",
        "Uncomment -> run -> comment out -> restart runtime"
      ]
    },
    {
      "metadata": {
        "id": "I5WEOZdbL_rL",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# %%capture\n",
        "# !pip install pydot\n",
        "# !apt-get install graphviz\n",
        "# !sed -i 's/def _check_pydot()/def _check__pydot()/g' /usr/local/lib/python3.6/dist-packages/keras/utils/vis_utils.py\n",
        "# !sed -i 's/_check_pydot()/#_check_pydot()/g' /usr/local/lib/python3.6/dist-packages/keras/utils/vis_utils.py"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "-sBgr9Xr8gA1",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### [Cheatsheet](https://drive.google.com/open?id=1iWhJiMT9pgWqYA_3-iRyvQ1DwlhV3hGdR-pinZiiHfk)"
      ]
    },
    {
      "metadata": {
        "id": "D9GySzxhOLJ0",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### **Disclaimer**:\n",
        "\n",
        "These functions were developed for teaching reasons.\n",
        "\n",
        "The main idea is to keep the code as simple and clean as possible for someone to get the structure of each model easily.\n",
        "\n",
        "For more configurable implementations of these models please refer to other available github repositories."
      ]
    },
    {
      "metadata": {
        "id": "ppyNTxqV6XXv",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Imports"
      ]
    },
    {
      "metadata": {
        "id": "buDTHaDL7bqK",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "%%capture\n",
        "import keras\n",
        "import keras.backend as K\n",
        "from keras.models import Model\n",
        "from keras.layers import Input, Dense, Conv2D, DepthwiseConv2D, SeparableConv2D\n",
        "from keras.layers import Flatten, MaxPool2D, AvgPool2D, GlobalAvgPool2D, UpSampling2D\n",
        "from keras.layers import BatchNormalization, concatenate, add, Dropout, ReLU, Lambda, Activation, LeakyReLU\n",
        "\n",
        "from IPython.display import SVG\n",
        "from keras.utils.vis_utils import model_to_dot\n",
        "\n",
        "from time import time\n",
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "mbbOdmRM_Adq",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## AlexNet"
      ]
    },
    {
      "metadata": {
        "id": "XpOhEDia_334",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def alexnet(input_shape, n_classes):\n",
        "  input = Input(input_shape)\n",
        "  \n",
        "  # actually batch normalization didn't exist back then\n",
        "  # they used LRN (Local Response Normalization) for regularization\n",
        "  x = Conv2D(96, 11, strides=4, padding='same', activation='relu')(input)\n",
        "  x = BatchNormalization()(x)\n",
        "  x = MaxPool2D(3, strides=2)(x)\n",
        "  \n",
        "  x = Conv2D(256, 5, padding='same', activation='relu')(x)\n",
        "  x = BatchNormalization()(x)\n",
        "  x = MaxPool2D(3, strides=2)(x)\n",
        "  \n",
        "  x = Conv2D(384, 3, strides=1, padding='same', activation='relu')(x)\n",
        "  \n",
        "  x = Conv2D(384, 3, strides=1, padding='same', activation='relu')(x)\n",
        "  \n",
        "  x = Conv2D(256, 3, strides=1, padding='same', activation='relu')(x)\n",
        "  x = BatchNormalization()(x)\n",
        "  x = MaxPool2D(3, strides=2)(x)\n",
        "  \n",
        "  x = Flatten()(x)\n",
        "  x = Dense(4096, activation='relu')(x)\n",
        "  x = Dense(4096, activation='relu')(x)\n",
        "  \n",
        "  output = Dense(n_classes, activation='softmax')(x)\n",
        "  \n",
        "  model = Model(input, output)\n",
        "  return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "8KzDeXAB_Co0",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## VGG"
      ]
    },
    {
      "metadata": {
        "id": "6lwZD4p_7l02",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def vgg(input_shape, n_classes):\n",
        "  filters = 64, 128, 256, 512, 512\n",
        "  repetitions = 2, 2, 3, 3, 3\n",
        "  \n",
        "  def vgg_block(x, f, r):\n",
        "    for _ in range(r):\n",
        "      x = Conv2D(f, 3, padding='same', activation='relu')(x)\n",
        "    x = MaxPool2D(2, strides=2)(x)\n",
        "    return x\n",
        "  \n",
        "  \n",
        "  input = Input(input_shape)\n",
        "  \n",
        "  x = input\n",
        "  for f, r in zip(filters, repetitions):\n",
        "    x = vgg_block(x, f, r)\n",
        "  \n",
        "  x = Flatten()(x)\n",
        "  x = Dense(4096, activation='relu')(x)\n",
        "  x = Dense(4096, activation='relu')(x)\n",
        "  output = Dense(n_classes, activation='softmax')(x)\n",
        "  \n",
        "  model = Model(input, output)\n",
        "  return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "QmxPPvv6_Ekt",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Inception"
      ]
    },
    {
      "metadata": {
        "id": "Lo98U9x_GUR5",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def googlenet(input_shape, n_classes):\n",
        "  \n",
        "  def incpeption_block(x, f):\n",
        "    t1 = Conv2D(f[0], 1, activation='relu')(x)\n",
        "    \n",
        "    t2 = Conv2D(f[1], 1, activation='relu')(x)\n",
        "    t2 = Conv2D(f[2], 3, padding='same', activation='relu')(t2)\n",
        "    \n",
        "    t3 = Conv2D(f[3], 1, activation='relu')(x)\n",
        "    t3 = Conv2D(f[4], 5, padding='same', activation='relu')(t3)\n",
        "    \n",
        "    t4 = MaxPool2D(3, 1, padding='same')(x)\n",
        "    t4 = Conv2D(f[5], 1, activation='relu')(t4)\n",
        "    \n",
        "    output = concatenate([t1, t2, t3, t4])\n",
        "    return output\n",
        "  \n",
        "  \n",
        "  input = Input(input_shape)\n",
        "  \n",
        "  x = Conv2D(64, 7, strides=2, padding='same', activation='relu')(input)\n",
        "  x = MaxPool2D(3, strides=2, padding='same')(x)\n",
        "  \n",
        "  x = Conv2D(64, 1, activation='relu')(x)\n",
        "  x = Conv2D(192, 3, padding='same', activation='relu')(x)\n",
        "  x = MaxPool2D(3, strides=2)(x)\n",
        "  \n",
        "  x = inception_block(x, [64, 96, 128, 16, 32, 32])\n",
        "  x = inception_block(x, [128, 128, 192, 32, 96, 64])\n",
        "  x = MaxPool2D(3, strides=2, padding='same')(x)\n",
        "  \n",
        "  x = inception_block(x, [192, 96, 208, 16, 48, 64])\n",
        "  x = inception_block(x, [160, 112, 224, 24, 64, 64])\n",
        "  x = inception_block(x, [128, 128, 256, 24, 64, 64])\n",
        "  x = inception_block(x, [112, 144, 288, 32, 64, 64])\n",
        "  x = inception_block(x, [256, 160, 32, 32, 128, 128])\n",
        "  x = MaxPool2D(3, strides=2, padding='same')(x)\n",
        "\n",
        "  x = inception_block(x, [256, 160, 320, 32, 128, 128])\n",
        "  x = inception_block(x, [384, 192, 384, 48, 128, 128])\n",
        "  \n",
        "  x = AvgPool2D(7, strides=1)(x)\n",
        "  x = Dropout(0.4)(x)\n",
        "  \n",
        "  x = Flatten()(x)\n",
        "  output = Dense(n_classes, activation='softmax')(x)\n",
        "  \n",
        "  model = Model(input, output)\n",
        "  return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Rx6nnBXy_GpA",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## MobileNet"
      ]
    },
    {
      "metadata": {
        "id": "7cX_4MhW_lyl",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def mobilenet(input_shape, n_classes):\n",
        "  \n",
        "  def conv_bn_rl(x, f, k=1, s=1, p='same'):\n",
        "    x = Conv2D(f, k, strides=s, padding=p)(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    x = ReLU()(x)\n",
        "    return x\n",
        "  \n",
        "  \n",
        "  def mobilenet_block(x, f, s=1):\n",
        "    x = DepthwiseConv2D(3, s, padding='same')(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    x = ReLU()(x)\n",
        "    \n",
        "    x = conv_bn_rl(x, f)\n",
        "    return x\n",
        "  \n",
        "  \n",
        "  input = Input(input_shape)\n",
        "  \n",
        "  x = conv_bn_rl(input, 32, 3, 2)\n",
        "  x = mobilenet_block(x, 64)\n",
        "  \n",
        "  x = mobilenet_block(x, 128, 2)\n",
        "  x = mobilenet_block(x, 128)\n",
        "  \n",
        "  x = mobilenet_block(x, 256, 2)\n",
        "  x = mobilenet_block(x, 256)\n",
        "  \n",
        "  x = mobilenet_block(x, 512, 2)\n",
        "  for i in range(5):\n",
        "    x = mobilenet_block(x, 512)\n",
        "  \n",
        "  x = mobilenet_block(x, 1024, 2)\n",
        "  x = mobilenet_block(x, 1024)\n",
        "  \n",
        "  x = AvgPool2D(7)(x)\n",
        "  \n",
        "  x = Flatten()(x)\n",
        "  output = Dense(n_classes, activation='softmax')(x)\n",
        "  \n",
        "  model = Model(input, output)\n",
        "  return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "zBTuyT22_l-3",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## ShuffleNet"
      ]
    },
    {
      "metadata": {
        "id": "RfosZPYf_oFe",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def shufflenet(input_shape, n_classes, g=8):\n",
        "  channels = 384, 769, 1536\n",
        "  repetitions = 3, 7, 3\n",
        "  \n",
        "  def ch_shuffle(x, g):\n",
        "#     1 2 3 4 5 6 7 8 9 -reshape-> 1 2 3 -permute dims-> 1 4 7 -reshape-> 1 4 7 2 5 8 3 6 9\n",
        "#                                  4 5 6                 2 5 8\n",
        "#                                  7 8 9                 3 6 9\n",
        "    \n",
        "    _, w, h, ch = K.int_shape(x)\n",
        "    ch_g = ch // g\n",
        "    \n",
        "    def shuffle_op(x):\n",
        "      x = K.reshape(x, [-1, w, h, ch_g, g])\n",
        "      x = K.permute_dimensions(x, [0, 1, 2, 4, 3])\n",
        "      x = K.reshape(x, [-1, w, h, ch])\n",
        "      return x\n",
        "    \n",
        "    x = Lambda(shuffle_op)(x)\n",
        "    return x\n",
        "    \n",
        "  \n",
        "  def gconv(tensor, ch, g):\n",
        "    _, _, _, in_ch = K.int_shape(tensor)\n",
        "    ch_g = in_ch // g\n",
        "    out_ch = ch // g\n",
        "    group = []\n",
        "    \n",
        "    for i in range(g):\n",
        "#       x = tensor[:, :, :, i*ch_g:(i+1)*ch_g]\n",
        "      x = Lambda(lambda x: x[:, :, :, i*ch_g: (i+1)*ch_g])(tensor)\n",
        "      x = Conv2D(out_ch, 1)(x)\n",
        "      group.append(x)\n",
        "    \n",
        "    x = concatenate(group)\n",
        "    return x\n",
        "  \n",
        "  \n",
        "  def shufflenet_block(tensor, ch, s, g):\n",
        "    x = gconv(tensor, ch // 4, g)\n",
        "    x = BatchNormalization()(x)\n",
        "    x = ReLU()(x)\n",
        "    x = ch_shuffle(x, g)\n",
        "    x = DepthwiseConv2D(3, strides=s, padding='same')(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    x = gconv(x, ch if s==1 else ch-K.int_shape(tensor)[-1], g)\n",
        "    x = BatchNormalization()(x)\n",
        "    \n",
        "    if s == 1:\n",
        "      x = add([tensor, x])\n",
        "    else:\n",
        "      avg = AvgPool2D(3, strides=2, padding='same')(tensor)\n",
        "      x = concatenate([avg, x])\n",
        "    \n",
        "    output = ReLU()(x)\n",
        "    return output\n",
        "  \n",
        "  \n",
        "  def stage(x, ch, r, g):\n",
        "    x = shufflenet_block(x, ch, 2, g)\n",
        "    \n",
        "    for i in range(r):\n",
        "      x = shufflenet_block(x, ch, 1, g)\n",
        "      \n",
        "    return x\n",
        "  \n",
        "  input = Input(input_shape)\n",
        "  \n",
        "  x = Conv2D(24, 3, strides=2, padding='same')(input)\n",
        "  x = BatchNormalization()(x)\n",
        "  x = ReLU()(x)\n",
        "  x = MaxPool2D(3, strides=2, padding='same')(x)\n",
        "  \n",
        "  for ch, r in zip(channels, repetitions):\n",
        "    x = stage(x, ch, r, g)\n",
        "    \n",
        "  x = GlobalAvgPool2D()(x)\n",
        "  output = Dense(n_classes, activation='softmax')(x)\n",
        "  \n",
        "  model = Model(input, output)\n",
        "  return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "kfJ8zng4_oWe",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## ResNet"
      ]
    },
    {
      "metadata": {
        "id": "-LbF4hlj_t7m",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def resnet(input_shape, n_classes):\n",
        "  \n",
        "  def conv_bn_rl(x, f, k=1, s=1, p='same'):\n",
        "    x = Conv2D(f, k, strides=s, padding=p)(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    x = ReLU()(x)\n",
        "    return x\n",
        "  \n",
        "  \n",
        "  def conv_block(tensor, f1, f2, s):\n",
        "    x = conv_bn_rl(tensor, f1)\n",
        "    x = conv_bn_rl(x, f1, 3, s=s)\n",
        "    x = Conv2D(f2, 1)(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    \n",
        "    shortcut = Conv2D(f2, 1, strides=s, padding='same')(tensor)\n",
        "    shortcut = BatchNormalization()(shortcut)\n",
        "    \n",
        "    x = add([shortcut, x])\n",
        "    output = ReLU()(x)\n",
        "    \n",
        "    return output\n",
        "  \n",
        "  \n",
        "  def identity_block(tensor, f1, f2):\n",
        "    x = conv_bn_rl(tensor, f1)\n",
        "    x = conv_bn_rl(x, f1, 3)\n",
        "    x = Conv2D(f2, 1)(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    \n",
        "    x = add([tensor, x])\n",
        "    output = ReLU()(x)\n",
        "    \n",
        "    return output\n",
        "  \n",
        "  \n",
        "  def resnet_block(x, f1, f2, r, s=2):\n",
        "    x = conv_block(x, f1, f2, s)\n",
        "    \n",
        "    for _ in range(r-1):\n",
        "      x = identity_block(x, f1, f2)\n",
        "    \n",
        "    return x\n",
        "  \n",
        "  \n",
        "  input = Input(input_shape)\n",
        "  \n",
        "  x = conv_bn_rl(input, 64, 7, 2)\n",
        "  x = MaxPool2D(3, strides=2, padding='same')(x)\n",
        "  \n",
        "  x = resnet_block(x, 64, 256, 3, 1)\n",
        "  x = resnet_block(x, 128, 512, 4)\n",
        "  x = resnet_block(x, 256, 1024, 6)\n",
        "  x = resnet_block(x, 512, 2048, 3)\n",
        "  \n",
        "  x = AvgPool2D(7, strides=1)(x)\n",
        "  x = Flatten()(x)\n",
        "  \n",
        "  output = Dense(n_classes, activation='softmax')(x)\n",
        "  model = Model(input, output)\n",
        "  \n",
        "  return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "FLz0AGu4_wEi",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## DenseNet"
      ]
    },
    {
      "metadata": {
        "id": "s1yjw8vW-5ge",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def densenet(input_shape, n_classes, k=32):\n",
        "  repetitions = 6, 12, 48, 32\n",
        "  \n",
        "  def bn_rl_conv(x, f, k=1, s=1, p='same'):\n",
        "    x = BatchNormalization()(x)\n",
        "    x = ReLU()(x)\n",
        "    x = Conv2D(f, k, strides=s, padding=p)(x)\n",
        "    return x\n",
        "  \n",
        "  \n",
        "  def dense_block(tensor, k):\n",
        "    x = bn_rl_conv(tensor, 4*k)\n",
        "    x = bn_rl_conv(x, k, 3)\n",
        "    output = concatenate([tensor, x])\n",
        "    return output\n",
        "  \n",
        "  \n",
        "  def transition(x):\n",
        "    x = bn_rl_conv(x, K.int_shape(x)[-1] // 2)\n",
        "    x = AvgPool2D(2, strides=2, padding='same')(x)\n",
        "    return x\n",
        "  \n",
        "  \n",
        "  def dense_blocks(x, r, k):\n",
        "    for _ in range(r):\n",
        "      x = dense_block(x, k)\n",
        "    return x\n",
        "  \n",
        "  \n",
        "  input = Input(input_shape)\n",
        "  \n",
        "  x = Conv2D(64, 7, strides=2, padding='same')(input)\n",
        "  x = MaxPool2D(3, strides=2, padding='same')(x)\n",
        "  \n",
        "  for r in repetitions:\n",
        "    d = dense_blocks(x, r, k)\n",
        "    x = transition(d)\n",
        "  \n",
        "  x = AvgPool2D(7, strides=1)(d)\n",
        "  x = Flatten()(x)\n",
        "  \n",
        "  output = Dense(n_classes, activation='softmax')(x)\n",
        "  model = Model(input, output)\n",
        "  \n",
        "  return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "czKlYBNv6O4E",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Xception"
      ]
    },
    {
      "metadata": {
        "id": "AH6T1YN24tcm",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def xception(input_shape, n_classes):\n",
        "  \n",
        "  def conv_bn(x, f, k=1, s=1, p='same'):\n",
        "    x = Conv2D(f, k, strides=s, padding=p)(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    return x\n",
        "  \n",
        "  \n",
        "  def rl_sep(x, f, k=3, s=1, p='same'):\n",
        "    x = ReLU()(x)\n",
        "    x = SeparableConv2D(f, k, strides=s, padding=p)(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    return x\n",
        "  \n",
        "  \n",
        "  def sep_rl(x, f, k=3, s=1, p='same'):\n",
        "    x = SeparableConv2D(f, k, strides=s, padding=p)(x)\n",
        "    x = BatchNormalization()(x)\n",
        "    x = ReLU()(x)\n",
        "    return x\n",
        "  \n",
        "  \n",
        "  def entry_flow(x, filters):\n",
        "    res = x\n",
        "    for f in filters:\n",
        "      res = conv_bn(res, f, s=2)\n",
        "      x = rl_sep(x, f)\n",
        "      x = rl_sep(x, f)\n",
        "      x = MaxPool2D(3, strides=2, padding='same')(x)\n",
        "      x = add([res, x])\n",
        "    return x\n",
        "      \n",
        "      \n",
        "  def midde_flow(tensor, f):\n",
        "    x = rl_sep(tensor, f)\n",
        "    x = rl_sep(x, f)\n",
        "    x = rl_sep(x, f)\n",
        "    x = add([tensor, x])\n",
        "    return x\n",
        "  \n",
        "  \n",
        "  def exit_flow(x, f):\n",
        "    res = conv_bn(x, f[1], s=2)\n",
        "    \n",
        "    x = rl_sep(x, f[0])\n",
        "    x = rl_sep(x, f[1])\n",
        "    x = MaxPool2D(3, strides=2, padding='same')(x)\n",
        "    x = add([res, x])\n",
        "    \n",
        "    x = sep_rl(x, f[2])\n",
        "    x = sep_rl(x, f[3])\n",
        "    return x\n",
        "  \n",
        "  \n",
        "  input = Input(input_shape)\n",
        "  \n",
        "  x = conv_bn(input, 32, 3, 2)\n",
        "  x = ReLU()(x)\n",
        "  x = conv_bn(input, 64, 3)\n",
        "  x = ReLU()(x)\n",
        "  \n",
        "  x = entry_flow(x, [128, 256, 728])\n",
        "  for _ in range(8):\n",
        "    x = midde_flow(x, 728)\n",
        "  x = exit_flow(x, [728, 1024, 1536, 2048])\n",
        "  \n",
        "  x = GlobalAvgPool2D()(x)\n",
        "  output = Dense(n_classes, activation='softmax')(x)\n",
        "  model = Model(input, output)\n",
        "  \n",
        "  return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "7bwiihQfg5PM",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Unet"
      ]
    },
    {
      "metadata": {
        "id": "a_L-XieEg6yT",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def unet(input_shape=(572, 572, 1), f=64, steps=4, n_classes=2):\n",
        "  \n",
        "  def downstream(x, f):\n",
        "    x = Conv2D(f, 3, activation='relu')(x)\n",
        "    d = Conv2D(f, 3, activation='relu')(x)\n",
        "    x = MaxPool2D(2, strides=2, padding='same')(d)\n",
        "    return d, x\n",
        "  \n",
        "  \n",
        "  def crop_merge(x, d):\n",
        "    _, xw, xh, _ = K.int_shape(x)\n",
        "    _, dw, dh, _ = K.int_shape(d)\n",
        "    mw, mh = (dw-xw)//2, (dh-xh)//2\n",
        "    \n",
        "    d = Lambda(lambda x: x[:, mw: dw-mw, mh: dh-mh, :])(d)\n",
        "    x = concatenate([d, x])\n",
        "    return x\n",
        "    \n",
        "  \n",
        "  def upstream(x, f, d):\n",
        "    x = UpSampling2D()(x)\n",
        "    x = Conv2D(f, 2, padding='same')(x)\n",
        "    x = crop_merge(x, d)\n",
        "    x = Conv2D(f, 3, activation='relu')(x)\n",
        "    x = Conv2D(f, 3, activation='relu')(x)\n",
        "    return x\n",
        "    \n",
        "  \n",
        "  input = Input(input_shape)\n",
        "  x = input\n",
        "  \n",
        "  downsampled = []\n",
        "  for i in range(steps+1):\n",
        "    d, x = downstream(x, f*2**i)\n",
        "    downsampled.append(d)\n",
        "  x = downsampled.pop()\n",
        "  \n",
        "  for i in range(steps-1, -1, -1):\n",
        "    x = upstream(x, f*2**i, downsampled[i])\n",
        "  \n",
        "  output = Conv2D(n_classes, 1)(x)\n",
        "  model = Model(input, output)\n",
        "  \n",
        "  return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "x_mKW2Yh_dM6",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## SqueezeNet"
      ]
    },
    {
      "metadata": {
        "id": "enUGc9iu_ggU",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def squeezenet(input_shape, n_classes):\n",
        "  \n",
        "  def fire_module(x, f1, f2, f3):\n",
        "    x = Conv2D(f1, 1, activation='relu')(x)\n",
        "    ones = Conv2D(f2, 1, activation='relu')(x)\n",
        "    threes = Conv2D(f3, 3, padding='same', activation='relu')(x)\n",
        "    output = concatenate([ones, threes])\n",
        "    return output\n",
        "  \n",
        "  \n",
        "  input = Input(input_shape)\n",
        "  \n",
        "  x = Conv2D(96, 7, strides=2, padding='same', activation='relu')(input)\n",
        "  x = MaxPool2D(3, strides=2)(x)\n",
        "  \n",
        "  x = fire_module(x, 16, 64, 64)\n",
        "  x = fire_module(x, 16, 64, 64)\n",
        "  x = fire_module(x, 32, 128, 128)\n",
        "  x = MaxPool2D(3, strides=2)(x)\n",
        "  \n",
        "  x = fire_module(x, 32, 128, 128)\n",
        "  x = fire_module(x, 48, 192, 192)\n",
        "  x = fire_module(x, 48, 192, 192)\n",
        "  x = fire_module(x, 64, 256, 256)\n",
        "  x = MaxPool2D(3, strides=2)(x)\n",
        "  \n",
        "  x = fire_module(x, 64, 256, 256)\n",
        "  x = Conv2D(n_classes, 1)(x)\n",
        "  x = GlobalAvgPool2D()(x)\n",
        "  \n",
        "  output = Activation('softmax')(x)\n",
        "  model = Model(input, output)\n",
        "  \n",
        "  return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "PCqote7DVRsv",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## YOLO"
      ]
    },
    {
      "metadata": {
        "id": "HeM_X8b6VW3p",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def yolo(input_shape=(448, 448, 3), n_outputs=30):\n",
        "  activation = LeakyReLU(0.1)\n",
        "  \n",
        "  def conv_1_3(x, f1, f2, r=1):\n",
        "    for _ in range(r):\n",
        "      x = Conv2D(f1, 1, padding='same', activation=activation)(x)\n",
        "      x = Conv2D(f2, 3, padding='same', activation=activation)(x)\n",
        "    return x\n",
        "    \n",
        "  \n",
        "  input = Input(input_shape)\n",
        "  \n",
        "  x = Conv2D(64, 7, strides=2, padding='same', activation=activation)(input)\n",
        "  x = MaxPool2D(2, strides=2, padding='same')(x)\n",
        "  \n",
        "  x = Conv2D(192, 3, padding='same', activation=activation)(x)\n",
        "  x = MaxPool2D(2, strides=2, padding='same')(x)\n",
        "  \n",
        "  x = conv_1_3(x, 128, 256)\n",
        "  x = conv_1_3(x, 256, 512)\n",
        "  x = MaxPool2D(2, strides=2, padding='same')(x)\n",
        "  \n",
        "  x = conv_1_3(x, 256, 512, 4)\n",
        "  x = conv_1_3(x, 512, 1024)\n",
        "  x = MaxPool2D(2, strides=2, padding='same')(x)\n",
        "  \n",
        "  x = conv_1_3(x, 512, 1024, 2)\n",
        "  x = Conv2D(1024, 3, padding='same', activation=activation)(x)\n",
        "  x = Conv2D(1024, 3, strides=2, padding='same', activation=activation)(x)\n",
        "  \n",
        "  x = Conv2D(1024, 3, padding='same', activation=activation)(x)\n",
        "  x = Conv2D(1024, 3, padding='same', activation=activation)(x)\n",
        "  \n",
        "  x = Dense(4096, activation=activation)(x)\n",
        "  output = Dense(n_outputs)(x)\n",
        "  \n",
        "  model = Model(input, output)\n",
        "  return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "UY9TNeWjUnye",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## RefineNet"
      ]
    },
    {
      "metadata": {
        "id": "AJ6vj_oFUoNc",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def refinenet(backbone_model, n_classes=21, tensor_names=('add_3', 'add_7', 'add_13', 'add_16')):\n",
        "  '''\n",
        "  backbone_model: (pretrained, non trainable) model used to extract the features (refinenet's inputs) \n",
        "  tensor_names: list of names of the backbone_model's tensors to be used as inputs (default: names from resnet)\n",
        "  '''\n",
        "  \n",
        "  filters = 512, 256, 256, 256\n",
        "  input = backbone_model.input\n",
        "  \n",
        "  def rcu(tensor, f, conv_shortcut=False):\n",
        "    x = ReLU()(tensor)\n",
        "    x = Conv2D(f, 3, padding='same')(x)\n",
        "    x = ReLU()(tensor)\n",
        "    x = Conv2D(f, 3, padding='same')(x)\n",
        "    \n",
        "    if conv_shortcut:\n",
        "      tensor = Conv2D(f, 3, padding='same')(tensor)\n",
        "      \n",
        "    output = add([tensor, x])\n",
        "    return output\n",
        "  \n",
        "  \n",
        "  def fusion_block(x1, x2, f):\n",
        "    x1 = Conv2D(f, 3, padding='same')(x1)\n",
        "    \n",
        "    x2 = Conv2D(f, 3, padding='same')(x2)\n",
        "    x2 = UpSampling2D()(x2)\n",
        "    \n",
        "    output = add([x1, x2])\n",
        "    return output\n",
        "  \n",
        "  \n",
        "  \n",
        "  def chained_block(tensor, steps=2):\n",
        "    f = K.int_shape(tensor)[-1]\n",
        "    tensor = ReLU()(tensor)\n",
        "    \n",
        "    x = tensor\n",
        "    for _ in range(steps):\n",
        "      x = MaxPool2D(5, strides=1, padding='same')(x)\n",
        "      x = Conv2D(f, 3, padding='same')(x)\n",
        "      tensor = add([tensor, x])\n",
        "    \n",
        "    return tensor\n",
        "  \n",
        "  \n",
        "  def refine_block(x1, x2=None, f=256):\n",
        "    x = rcu(x1, f, True)\n",
        "    x = rcu(x, f)\n",
        "    \n",
        "    if x2 is not None:\n",
        "      x2 = rcu(x2, f, True)\n",
        "      x2 = rcu(x2, f)\n",
        "      x = fusion_block(x, x2, f)\n",
        "    \n",
        "    x = chained_block(x)\n",
        "    \n",
        "    output = rcu(x, f)\n",
        "    return output\n",
        "  \n",
        "  \n",
        "  inputs = [layer.output for layer in backbone_model.layers if layer.name in tensor_names][::-1]\n",
        "  x = None\n",
        "  \n",
        "  for t, f in zip(inputs, filters):\n",
        "    x = refine_block(t, x, f)\n",
        "  \n",
        "  x = rcu(x, filters[-1])\n",
        "  x = rcu(x, filters[-1])\n",
        "  \n",
        "  output = Dense(n_classes, activation='softmax')(x)\n",
        "  model = Model(input, output)\n",
        "  \n",
        "  return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "M4_wmfYx6Jkg",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Print and plot the model"
      ]
    },
    {
      "metadata": {
        "id": "_hT79naf9xN6",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "input_shape = 224, 224, 3\n",
        "n_classes = 1000\n",
        "\n",
        "K.clear_session()\n",
        "model = alexnet(input_shape, n_classes)\n",
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "7N_s-8hlMXE_",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "SVG(model_to_dot(model).create(prog='dot', format='svg'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "WNC4vNba_hYw",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Calculate inference time"
      ]
    },
    {
      "metadata": {
        "id": "qVQLIdGx_kdm",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "repetitions = 10\n",
        "input = np.random.randn(1, *input_shape)\n",
        "\n",
        "output = model.predict(input)\n",
        "start = time()\n",
        "for _ in range(repetitions):\n",
        "  output = model.predict(input)\n",
        "  \n",
        "print((time() - start) / repetitions)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}