conv_vs_depthwisesepconv.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.14.0\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import time\n",
    "\n",
    "# python -c \"from tensorflow.python.client import device_lib; print(device_lib.list_local_devices())\"\n",
    "\n",
    "import tensorflow as tf\n",
    "tf.logging.set_verbosity(tf.logging.ERROR)\n",
    "from tensorflow.python.client import device_lib\n",
    "# print(device_lib.list_local_devices())\n",
    "print(tf.__version__)\n",
    "assert tf.test.is_gpu_available(), \"¯\\_(ツ)_/¯\"\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_op(inp, func=lambda x: tf.identity(x)):\n",
    "  inp = func(inp)\n",
    "  y   = tf.nn.l2_loss(inp)\n",
    "  return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.Session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _vars_init():\n",
    "  uninitialized_vars = []\n",
    "  for var in tf.all_variables():\n",
    "    try:\n",
    "      sess.run(var)\n",
    "    except tf.errors.FailedPreconditionError:\n",
    "      uninitialized_vars.append(var)\n",
    "  init_new_vars_op = tf.initialize_variables(uninitialized_vars)\n",
    "  sess.run(init_new_vars_op)\n",
    "\n",
    "def get_ops(filters=64, kernel=[3, 3], shape=[128, 256, 256, 3], format=\"channels_last\"):\n",
    "  with tf.device(\"gpu\"):\n",
    "    inp = tf.random.normal(shape)\n",
    "    id_op     = get_op(inp)\n",
    "    dwconv_op = get_op(inp, func=lambda x: tf.layers.separable_conv2d(inp, filters, kernel, padding=\"SAME\", data_format=format))\n",
    "    conv_op   = get_op(inp, func=lambda x: tf.layers.conv2d(inp, filters, kernel, data_format=format))\n",
    "    _vars_init()\n",
    "    return id_op, dwconv_op, conv_op\n",
    "  print('¯\\_(ツ)_/¯')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_avg(op, n=10, name=None):\n",
    "  times = []\n",
    "  res   = []\n",
    "  for i in range(n):\n",
    "    start = time.time()\n",
    "    res += sess.run(op)\n",
    "    times.append(time.time() - start)  \n",
    "#   if name: print('%10s: %f (+/-%f) [%f, %f]' % (name, np.mean(times), np.std(times), min(times), max(times)))\n",
    "  return np.mean(times)\n",
    "\n",
    "\n",
    "op_cache = {}\n",
    "def run_single_comparison(**kwargs):\n",
    "  # caching\n",
    "  key = str(kwargs)\n",
    "  if key not in op_cache:\n",
    "    op_cache[key] = get_ops(**kwargs)\n",
    "  id_op, conv_op, dwconv_op = op_cache[key]\n",
    "  \n",
    "  id_t    = run_avg(id_op, name='id')\n",
    "  conv_t  = run_avg(conv_op, name='conv')    - id_t\n",
    "  dconv_t = run_avg(dwconv_op, name='dconv') - id_t\n",
    "  print('%.4f -> %.4f (%.2f%%)' % (conv_t, dconv_t, ((conv_t/dconv_t-1)*100)), kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stability test for given N:\n",
      "0.1116 -> 0.0556 (100.82%) {'shape': [128, 256, 256, 3]}\n",
      "0.0465 -> 0.0445 (4.55%) {'shape': [128, 256, 256, 3]}\n",
      "0.0464 -> 0.0444 (4.48%) {'shape': [128, 256, 256, 3]}\n",
      "0.1774 -> 0.2146 (-17.30%) {'shape': [128, 256, 256, 128]}\n",
      "0.1701 -> 0.1491 (14.03%) {'shape': [128, 256, 256, 128]}\n",
      "0.1710 -> 0.1498 (14.18%) {'shape': [128, 256, 256, 128]}\n"
     ]
    }
   ],
   "source": [
    "print(\"Stability test for given N:\")\n",
    "run_single_comparison(shape=[128, 256, 256, 3])\n",
    "run_single_comparison(shape=[128, 256, 256, 3])\n",
    "run_single_comparison(shape=[128, 256, 256, 3])\n",
    "\n",
    "run_single_comparison(shape=[128, 256, 256, 128])\n",
    "run_single_comparison(shape=[128, 256, 256, 128])\n",
    "run_single_comparison(shape=[128, 256, 256, 128])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First run is not like the others"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Channel_first vs channel_last forward path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1702 -> 0.1494 (13.92%) {'shape': [128, 256, 256, 128], 'format': 'channels_last'}\n",
      "0.1345 -> 0.1126 (19.52%) {'shape': [128, 128, 256, 256], 'format': 'channels_first'}\n",
      "0.1711 -> 0.1501 (14.00%) {'shape': [128, 256, 256, 128], 'format': 'channels_last'}\n",
      "0.1348 -> 0.1127 (19.59%) {'shape': [128, 128, 256, 256], 'format': 'channels_first'}\n"
     ]
    }
   ],
   "source": [
    "# what about bigger volume\n",
    "run_single_comparison(shape=[128, 256, 256, 128], format=\"channels_last\")\n",
    "run_single_comparison(shape=[128, 128, 256, 256], format=\"channels_first\")\n",
    "run_single_comparison(shape=[128, 256, 256, 128], format=\"channels_last\")\n",
    "run_single_comparison(shape=[128, 128, 256, 256], format=\"channels_first\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0304 -> 0.0208 (46.23%) {'shape': [128, 64, 64, 512], 'format': 'channels_last'}\n",
      "0.0302 -> 0.0208 (44.82%) {'shape': [128, 64, 64, 512], 'format': 'channels_last'}\n",
      "0.0302 -> 0.0210 (43.75%) {'shape': [128, 64, 64, 512], 'format': 'channels_last'}\n",
      "0.0231 -> 0.0150 (53.92%) {'shape': [128, 512, 64, 64], 'format': 'channels_first'}\n",
      "0.0238 -> 0.0146 (62.80%) {'shape': [128, 512, 64, 64], 'format': 'channels_first'}\n",
      "0.0237 -> 0.0147 (61.37%) {'shape': [128, 512, 64, 64], 'format': 'channels_first'}\n"
     ]
    }
   ],
   "source": [
    "run_single_comparison(shape=[128, 64, 64, 512], format=\"channels_last\")\n",
    "run_single_comparison(shape=[128, 64, 64, 512], format=\"channels_last\")\n",
    "run_single_comparison(shape=[128, 64, 64, 512], format=\"channels_last\")\n",
    "run_single_comparison(shape=[128, 512, 64, 64], format=\"channels_first\")\n",
    "run_single_comparison(shape=[128, 512, 64, 64], format=\"channels_first\")\n",
    "run_single_comparison(shape=[128, 512, 64, 64], format=\"channels_first\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0287 -> 0.0197 (45.60%) {'shape': [128, 64, 64, 512], 'format': 'channels_last'}\n",
      "0.0296 -> 0.0203 (45.36%) {'shape': [128, 64, 64, 512], 'format': 'channels_last'}\n",
      "0.0296 -> 0.0211 (40.34%) {'shape': [128, 64, 64, 512], 'format': 'channels_last'}\n",
      "0.0237 -> 0.0145 (62.88%) {'shape': [128, 512, 64, 64], 'format': 'channels_first'}\n",
      "0.0240 -> 0.0148 (62.40%) {'shape': [128, 512, 64, 64], 'format': 'channels_first'}\n",
      "0.0240 -> 0.0145 (65.60%) {'shape': [128, 512, 64, 64], 'format': 'channels_first'}\n"
     ]
    }
   ],
   "source": [
    "s, c = 64, 512\n",
    "run_single_comparison(shape=[128, s, s, c], format=\"channels_last\")\n",
    "run_single_comparison(shape=[128, s, s, c], format=\"channels_last\")\n",
    "run_single_comparison(shape=[128, s, s, c], format=\"channels_last\")\n",
    "run_single_comparison(shape=[128, c, s, s], format=\"channels_first\")\n",
    "run_single_comparison(shape=[128, c, s, s], format=\"channels_first\")\n",
    "run_single_comparison(shape=[128, c, s, s], format=\"channels_first\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0327 -> 0.0596 (-45.18%) {'shape': [128, 32, 32, 4096], 'format': 'channels_last'}\n",
      "0.0251 -> 0.0348 (-27.98%) {'shape': [128, 32, 32, 4096], 'format': 'channels_last'}\n",
      "0.0252 -> 0.0346 (-27.24%) {'shape': [128, 32, 32, 4096], 'format': 'channels_last'}\n",
      "0.0132 -> 0.0228 (-42.21%) {'shape': [128, 4096, 32, 32], 'format': 'channels_first'}\n",
      "0.0130 -> 0.0225 (-42.06%) {'shape': [128, 4096, 32, 32], 'format': 'channels_first'}\n",
      "0.0132 -> 0.0228 (-41.99%) {'shape': [128, 4096, 32, 32], 'format': 'channels_first'}\n"
     ]
    }
   ],
   "source": [
    "s, c = 32, 4096\n",
    "run_single_comparison(shape=[128, s, s, c], format=\"channels_last\")\n",
    "run_single_comparison(shape=[128, s, s, c], format=\"channels_last\")\n",
    "run_single_comparison(shape=[128, s, s, c], format=\"channels_last\")\n",
    "run_single_comparison(shape=[128, c, s, s], format=\"channels_first\")\n",
    "run_single_comparison(shape=[128, c, s, s], format=\"channels_first\")\n",
    "run_single_comparison(shape=[128, c, s, s], format=\"channels_first\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}