Object Counting with MXNet and Sagemaker.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "import json\n",
    "import base64\n",
    "import subprocess\n",
    "from pickle import dump\n",
    "from shutil import rmtree, copy2\n",
    "\n",
    "import cv2\n",
    "import boto3\n",
    "import mxnet as mx\n",
    "import numpy as np\n",
    "from pandas import read_csv\n",
    "import sagemaker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sagemaker_session = sagemaker.Session()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Synthetic Data Generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_size = 12000\n",
    "img_size = 128\n",
    "shape_size = 8\n",
    "max_rows = 14\n",
    "max_cols = 14\n",
    "target_shape = \"circle\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "shapes = [\"circle\", \"triangle\", \"square\"]\n",
    "img_width = img_height = img_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.89 s, sys: 2.12 s, total: 5.01 s\n",
      "Wall time: 5.13 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "os.makedirs(\"data/raw\")\n",
    "    \n",
    "random.seed(42)\n",
    "\n",
    "with open(\"data/dataset.csv\", \"w+\") as dataset:\n",
    "    dataset.write(\"img_path,target\\n\")\n",
    "    for i in range(1, dataset_size + 1):\n",
    "        target_number_of_shapes = 0\n",
    "\n",
    "        filename = str(i).zfill(6) + \".shape\"\n",
    "\n",
    "        file_content = \"img_dim:{},shp_dim:{}>>>\".format(img_size, shape_size)\n",
    "        for x in range(0, random.randint(1, max_rows)):\n",
    "            for y in range(0, random.randint(1, max_cols)):\n",
    "                shape_idx = random.randint(0, len(shapes) - 1)\n",
    "                shape = shapes[shape_idx]\n",
    "                if shape == target_shape:\n",
    "                    target_number_of_shapes += 1\n",
    "                file_content = file_content + shape + \",\"\n",
    "            file_content = file_content[:-1]\n",
    "            file_content += \"|\"\n",
    "        file_content = file_content[:-1]\n",
    "        file_content += \"<<<\"\n",
    "\n",
    "        with open(\"./data/raw/{}\".format(filename), \"w\") as shape_file:\n",
    "            shape_file.write(file_content)\n",
    "\n",
    "        dataset.write(\"./raw/{}.png,{}\\n\".format(filename, str(target_number_of_shapes)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.12 ms, sys: 10.5 ms, total: 13.6 ms\n",
      "Wall time: 49.8 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "subprocess.call(\"\"\"\n",
    "    java -cp bin/shaper-all.jar com.cosminsanda.shaper.compiler.Shaper2Image --source-dir data/raw\"\"\", shell=True);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = read_csv(\"data/dataset.csv\")\n",
    "\n",
    "train = df.sample(frac=.8333, random_state=42)\n",
    "validation = df.loc[~df.index.isin(train.index), :].sample(frac=.5, random_state=42)\n",
    "test = df.loc[np.logical_not(np.logical_xor(~df.index.isin(train.index), ~df.index.isin(validation.index))), :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transform(row):\n",
    "    img = cv2.imread(\"./data/{}\".format(row[\"img_path\"]))\n",
    "    img = mx.nd.array(img)\n",
    "    img = img.astype(np.float32)\n",
    "    img = mx.nd.transpose(img, (2, 0, 1))\n",
    "    img = img / 255\n",
    "    label = np.float32(row[\"target\"])\n",
    "    return img, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 10.3 s, sys: 2.66 s, total: 13 s\n",
      "Wall time: 11.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_nd = [transform(row) for _, row in train.iterrows()]\n",
    "validation_nd = [transform(row) for _, row in validation.iterrows()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_to_disk(data, type):\n",
    "    os.makedirs(\"data/pickles/{}\".format(type))\n",
    "    with open(\"data/pickles/{}/data.p\".format(type), \"wb\") as out:\n",
    "        dump(data, out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.74 s, sys: 7.38 s, total: 11.1 s\n",
      "Wall time: 12.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "save_to_disk(train_nd, \"train\")\n",
    "save_to_disk(validation_nd, \"validation\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 23.9 s, sys: 19.1 s, total: 43 s\n",
      "Wall time: 5min 17s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "inputs = sagemaker_session.upload_data(path=\"data/pickles\", bucket=\"redacted\", key_prefix=\"sagemaker/demo\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "rmtree(\"./test\", True)\n",
    "os.makedirs(\"./test\")\n",
    "for _, row in test.iterrows():\n",
    "    os.makedirs(\"test/{}\".format(row[\"target\"]), exist_ok=True)\n",
    "    copy2(\"./data/{}\".format(row[\"img_path\"]), \"./test/{}\".format(row[\"target\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "rmtree(\"data\", True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "estimator = sagemaker.mxnet.MXNet(\"object-counting-sagemaker-script.py\", \n",
    "          role=sagemaker.get_execution_role(), \n",
    "          train_instance_count=1, \n",
    "          train_instance_type=\"ml.p2.xlarge\",\n",
    "          hyperparameters={\"epochs\": 5},\n",
    "          py_version=\"py3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:sagemaker:Created S3 bucket: redacted\n",
      "INFO:sagemaker:Creating training-job with name: sagemaker-mxnet-py3-gpu-2018-03-19-19-42-08-133\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "..................................................................................\n",
      "\u001b[31mexecuting startup script (first run)\u001b[0m\n",
      "\u001b[31m2018-03-19 19:49:04,334 INFO - root - running container entrypoint\u001b[0m\n",
      "\u001b[31m2018-03-19 19:49:04,335 INFO - root - starting train task\u001b[0m\n",
      "\u001b[31m2018-03-19 19:49:05,936 INFO - mxnet_container.train - MXNetTrainingEnvironment: {'output_dir': '/opt/ml/output', 'input_dir': '/opt/ml/input', 'user_script_name': 'object-counting-sagemaker-script.py', 'hosts': ['algo-1'], 'enable_cloudwatch_metrics': False, 'available_gpus': 1, 'resource_config': {'hosts': ['algo-1'], 'current_host': 'algo-1'}, '_scheduler_host': 'algo-1', 'input_config_dir': '/opt/ml/input/config', 'model_dir': '/opt/ml/model', 'base_dir': '/opt/ml', 'available_cpus': 4, '_scheduler_ip': '10.32.0.4', 'channels': {'training': {'RecordWrapperType': 'None', 'S3DistributionType': 'FullyReplicated', 'TrainingInputMode': 'File'}}, 'hyperparameters': {'sagemaker_container_log_level': 20, 'epochs': 5, 'sagemaker_region': 'us-east-1', 'sagemaker_job_name': 'sagemaker-mxnet-py3-gpu-2018-03-19-19-42-08-133', 'sagemaker_enable_cloudwatch_metrics': False, 'sagemaker_program': 'object-counting-sagemaker-script.py', 'sagemaker_submit_directory': 's3://redacted/sagemaker-mxnet-py3-gpu-2018-03-19-19-42-08-133/source/sourcedir.tar.gz'}, 'current_host': 'algo-1', '_ps_port': 8000, 'code_dir': '/opt/ml/code', 'user_requirements_file': None, 'output_data_dir': '/opt/ml/output/data/', 'sagemaker_region': 'us-east-1', 'channel_dirs': {'training': '/opt/ml/input/data/training'}, 'user_script_archive': 's3://redacted/sagemaker-mxnet-py3-gpu-2018-03-19-19-42-08-133/source/sourcedir.tar.gz', 'container_log_level': 20, '_ps_verbose': 0}\u001b[0m\n",
      "\u001b[31mDownloading s3://redacted/sagemaker-mxnet-py3-gpu-2018-03-19-19-42-08-133/source/sourcedir.tar.gz to /tmp/script.tar.gz\u001b[0m\n",
      "\u001b[31m2018-03-19 19:49:06,034 INFO - botocore.vendored.requests.packages.urllib3.connectionpool - Starting new HTTP connection (1): 169.254.170.2\u001b[0m\n",
      "\u001b[31m2018-03-19 19:49:06,121 INFO - botocore.vendored.requests.packages.urllib3.connectionpool - Starting new HTTPS connection (1): s3.amazonaws.com\u001b[0m\n",
      "\u001b[31m2018-03-19 19:49:06,201 INFO - mxnet_container.train - Starting distributed training task\u001b[0m\n",
      "\u001b[31mCollecting opencv-python\n",
      "  Downloading opencv_python-3.4.0.12-cp35-cp35m-manylinux1_x86_64.whl (24.9MB)\u001b[0m\n",
      "\u001b[31mRequirement already satisfied: numpy>=1.11.1 in /usr/local/lib/python3.5/dist-packages (from opencv-python)\u001b[0m\n",
      "\u001b[31mInstalling collected packages: opencv-python\u001b[0m\n",
      "\u001b[31mSuccessfully installed opencv-python-3.4.0.12\u001b[0m\n",
      "\u001b[31mYou are using pip version 9.0.1, however version 9.0.2 is available.\u001b[0m\n",
      "\u001b[31mYou should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n",
      "\u001b[31m/opt/amazon/lib/python3.4/site-packages/mxnet_container/train.py:163: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  train_args = inspect.getargspec(user_module.train)\u001b[0m\n",
      "\u001b[31mLoading data from /opt/ml/input/data/training\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/cosmin/playground/object-counting-with-mxnet-and-sagemaker/env/lib/python3.6/site-packages/sagemaker/session.py:616: DeprecationWarning: generator 'multi_stream_iter' raised StopIteration\n",
      "  for idx, event in sagemaker.logs.multi_stream_iter(client, log_group, stream_names, positions):\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31m[12:49:17] src/operator/././cudnn_algoreg-inl.h:107: Running performance tests to find the best convolution algorithm, this can take a while... (setting env variable MXNET_CUDNN_AUTOTUNE_DEFAULT to 0 to disable)\u001b[0m\n",
      "\u001b[31mEpoch 0: loss: 56.1768 Test accuracy: 0.20 Validation accuracy: 0.21\u001b[0m\n",
      "\u001b[31mEpoch 1: loss: 0.7820 Test accuracy: 0.61 Validation accuracy: 0.59\u001b[0m\n",
      "\u001b[31mEpoch 2: loss: 0.1410 Test accuracy: 0.90 Validation accuracy: 0.87\u001b[0m\n",
      "\u001b[31mEpoch 3: loss: 0.1065 Test accuracy: 0.89 Validation accuracy: 0.90\u001b[0m\n",
      "\u001b[31mEpoch 4: loss: 0.0920 Test accuracy: 0.88 Validation accuracy: 0.86\u001b[0m\n",
      "===== Job Complete =====\n",
      "Billable seconds: 324\n"
     ]
    }
   ],
   "source": [
    "estimator.fit(inputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Deployment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:sagemaker:Creating model with name: sagemaker-mxnet-py3-gpu-2018-03-19-19-42-08-133\n",
      "INFO:sagemaker:Creating endpoint with name sagemaker-mxnet-py3-gpu-2018-03-19-19-42-08-133\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------------------------------------------!"
     ]
    }
   ],
   "source": [
    "predictor = estimator.deploy(1, \"ml.m4.xlarge\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "sagemaker_runtime_client = boto3.client(\"sagemaker-runtime\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Images with 50 circles:\n",
      "        Total: 2\n",
      "        Detected: 0\n",
      "        Accuracy: 0.00\n",
      "    \n",
      "Images with 32 circles:\n",
      "        Total: 23\n",
      "        Detected: 13\n",
      "        Accuracy: 0.57\n",
      "    \n",
      "Images with 35 circles:\n",
      "        Total: 12\n",
      "        Detected: 8\n",
      "        Accuracy: 0.67\n",
      "    \n",
      "Images with 34 circles:\n",
      "        Total: 15\n",
      "        Detected: 12\n",
      "        Accuracy: 0.80\n",
      "    \n",
      "Images with 33 circles:\n",
      "        Total: 17\n",
      "        Detected: 12\n",
      "        Accuracy: 0.71\n",
      "    \n",
      "Images with 20 circles:\n",
      "        Total: 36\n",
      "        Detected: 31\n",
      "        Accuracy: 0.86\n",
      "    \n",
      "Images with 18 circles:\n",
      "        Total: 25\n",
      "        Detected: 22\n",
      "        Accuracy: 0.88\n",
      "    \n",
      "Images with 27 circles:\n",
      "        Total: 25\n",
      "        Detected: 20\n",
      "        Accuracy: 0.80\n",
      "    \n",
      "Images with 9 circles:\n",
      "        Total: 30\n",
      "        Detected: 29\n",
      "        Accuracy: 0.97\n",
      "    \n",
      "Images with 0 circles:\n",
      "        Total: 9\n",
      "        Detected: 9\n",
      "        Accuracy: 1.00\n",
      "    \n",
      "Images with 11 circles:\n",
      "        Total: 30\n",
      "        Detected: 30\n",
      "        Accuracy: 1.00\n",
      "    \n",
      "Images with 7 circles:\n",
      "        Total: 32\n",
      "        Detected: 32\n",
      "        Accuracy: 1.00\n",
      "    \n",
      "Images with 29 circles:\n",
      "        Total: 13\n",
      "        Detected: 11\n",
      "        Accuracy: 0.85\n",
      "    \n",
      "Images with 16 circles:\n",
      "        Total: 29\n",
      "        Detected: 28\n",
      "        Accuracy: 0.97\n",
      "    \n",
      "Images with 42 circles:\n",
      "        Total: 3\n",
      "        Detected: 1\n",
      "        Accuracy: 0.33\n",
      "    \n",
      "Images with 45 circles:\n",
      "        Total: 3\n",
      "        Detected: 0\n",
      "        Accuracy: 0.00\n",
      "    \n",
      "Images with 6 circles:\n",
      "        Total: 34\n",
      "        Detected: 34\n",
      "        Accuracy: 1.00\n",
      "    \n",
      "Images with 28 circles:\n",
      "        Total: 24\n",
      "        Detected: 21\n",
      "        Accuracy: 0.88\n",
      "    \n",
      "Images with 17 circles:\n",
      "        Total: 30\n",
      "        Detected: 28\n",
      "        Accuracy: 0.93\n",
      "    \n",
      "Images with 1 circles:\n",
      "        Total: 22\n",
      "        Detected: 22\n",
      "        Accuracy: 1.00\n",
      "    \n",
      "Images with 10 circles:\n",
      "        Total: 32\n",
      "        Detected: 31\n",
      "        Accuracy: 0.97\n",
      "    \n",
      "Images with 19 circles:\n",
      "        Total: 22\n",
      "        Detected: 21\n",
      "        Accuracy: 0.95\n",
      "    \n",
      "Images with 26 circles:\n",
      "        Total: 27\n",
      "        Detected: 18\n",
      "        Accuracy: 0.67\n",
      "    \n",
      "Images with 8 circles:\n",
      "        Total: 34\n",
      "        Detected: 34\n",
      "        Accuracy: 1.00\n",
      "    \n",
      "Images with 21 circles:\n",
      "        Total: 21\n",
      "        Detected: 14\n",
      "        Accuracy: 0.67\n",
      "    \n",
      "Images with 44 circles:\n",
      "        Total: 4\n",
      "        Detected: 2\n",
      "        Accuracy: 0.50\n",
      "    \n",
      "Images with 43 circles:\n",
      "        Total: 4\n",
      "        Detected: 2\n",
      "        Accuracy: 0.50\n",
      "    \n",
      "Images with 38 circles:\n",
      "        Total: 11\n",
      "        Detected: 8\n",
      "        Accuracy: 0.73\n",
      "    \n",
      "Images with 36 circles:\n",
      "        Total: 15\n",
      "        Detected: 9\n",
      "        Accuracy: 0.60\n",
      "    \n",
      "Images with 31 circles:\n",
      "        Total: 22\n",
      "        Detected: 17\n",
      "        Accuracy: 0.77\n",
      "    \n",
      "Images with 30 circles:\n",
      "        Total: 23\n",
      "        Detected: 18\n",
      "        Accuracy: 0.78\n",
      "    \n",
      "Images with 37 circles:\n",
      "        Total: 11\n",
      "        Detected: 4\n",
      "        Accuracy: 0.36\n",
      "    \n",
      "Images with 39 circles:\n",
      "        Total: 11\n",
      "        Detected: 7\n",
      "        Accuracy: 0.64\n",
      "    \n",
      "Images with 46 circles:\n",
      "        Total: 2\n",
      "        Detected: 0\n",
      "        Accuracy: 0.00\n",
      "    \n",
      "Images with 41 circles:\n",
      "        Total: 3\n",
      "        Detected: 2\n",
      "        Accuracy: 0.67\n",
      "    \n",
      "Images with 24 circles:\n",
      "        Total: 26\n",
      "        Detected: 22\n",
      "        Accuracy: 0.85\n",
      "    \n",
      "Images with 23 circles:\n",
      "        Total: 34\n",
      "        Detected: 27\n",
      "        Accuracy: 0.79\n",
      "    \n",
      "Images with 4 circles:\n",
      "        Total: 24\n",
      "        Detected: 24\n",
      "        Accuracy: 1.00\n",
      "    \n",
      "Images with 15 circles:\n",
      "        Total: 25\n",
      "        Detected: 24\n",
      "        Accuracy: 0.96\n",
      "    \n",
      "Images with 3 circles:\n",
      "        Total: 33\n",
      "        Detected: 33\n",
      "        Accuracy: 1.00\n",
      "    \n",
      "Images with 12 circles:\n",
      "        Total: 20\n",
      "        Detected: 20\n",
      "        Accuracy: 1.00\n",
      "    \n",
      "Images with 49 circles:\n",
      "        Total: 2\n",
      "        Detected: 1\n",
      "        Accuracy: 0.50\n",
      "    \n",
      "Images with 40 circles:\n",
      "        Total: 7\n",
      "        Detected: 3\n",
      "        Accuracy: 0.43\n",
      "    \n",
      "Images with 47 circles:\n",
      "        Total: 2\n",
      "        Detected: 1\n",
      "        Accuracy: 0.50\n",
      "    \n",
      "Images with 2 circles:\n",
      "        Total: 23\n",
      "        Detected: 23\n",
      "        Accuracy: 1.00\n",
      "    \n",
      "Images with 13 circles:\n",
      "        Total: 37\n",
      "        Detected: 37\n",
      "        Accuracy: 1.00\n",
      "    \n",
      "Images with 5 circles:\n",
      "        Total: 35\n",
      "        Detected: 35\n",
      "        Accuracy: 1.00\n",
      "    \n",
      "Images with 14 circles:\n",
      "        Total: 28\n",
      "        Detected: 27\n",
      "        Accuracy: 0.96\n",
      "    \n",
      "Images with 22 circles:\n",
      "        Total: 26\n",
      "        Detected: 23\n",
      "        Accuracy: 0.88\n",
      "    \n",
      "Images with 25 circles:\n",
      "        Total: 22\n",
      "        Detected: 19\n",
      "        Accuracy: 0.86\n",
      "    \n"
     ]
    }
   ],
   "source": [
    "for directory in os.listdir(\"./test\"):\n",
    "    batch = []\n",
    "    for file in os.listdir(\"./test/{}\".format(directory)):\n",
    "        with open(\"./test/{}/{}\".format(directory, file), \"rb\") as image_file:\n",
    "            batch.append(base64.b64encode(image_file.read()).decode(\"utf-8\"))\n",
    "    binary_json = json.dumps(batch).encode(\"utf-8\")\n",
    "    response = sagemaker_runtime_client.invoke_endpoint(\n",
    "        EndpointName=predictor.endpoint,\n",
    "        Body=binary_json,\n",
    "        ContentType=\"application/json\",\n",
    "        Accept=\"application/json\"\n",
    "    )[\"Body\"].read()\n",
    "    individual_predictions = json.loads(response, encoding=\"utf-8\")\n",
    "    total = 0\n",
    "    detected = 0\n",
    "    for prediction in individual_predictions:\n",
    "        total += 1 \n",
    "        if int(prediction) == int(directory):\n",
    "            detected += 1\n",
    "    print(\"\"\"Images with {} circles:\n",
    "        Total: {}\n",
    "        Detected: {}\n",
    "        Accuracy: {:0.2f}\n",
    "    \"\"\".format(directory, str(total), str(detected), detected/total))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}