examples/NeuralNetwork_Simple.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# This is simple NN example\n",
    "- This example will require Keras setup to run.\n",
    "- Keras=1.2\n",
    "- Tensorflow: 1.14\n",
    "- Python 3.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Some Imports\n",
    "from sklearn import datasets\n",
    "from mango import Tuner, scheduler"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Search Space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Defining only the number of layers in the param_space.\n",
    "\n",
    "During Neural Network training: hidden dimensions will be decided.\n",
    "\"\"\"\n",
    "\n",
    "# n_neighbors can vary between 1 and 50, with different choices of algorithm\n",
    "param_space = dict(num_of_layers=range(1, 7))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "iris = datasets.load_iris()\n",
    "\n",
    "# Take only the first two features.\n",
    "X = iris.data\n",
    "y = iris.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(150, 4) (150,)\n"
     ]
    }
   ],
   "source": [
    "print(X.shape, y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert y to one-hot encoded\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(150, 3)\n"
     ]
    }
   ],
   "source": [
    "from keras.utils import np_utils\n",
    "\n",
    "y = np_utils.to_categorical(y, 3)\n",
    "\n",
    "print(y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NN model using Keras\n",
    "\n",
    "- Hidden dimensions are function of number of layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "\n",
    "def get_model(num_of_layers):\n",
    "    global X, y\n",
    "    model = Sequential()\n",
    "    model.add(Dense(4, input_dim=X.shape[1], activation='relu'))\n",
    "    \n",
    "    \n",
    "    # Hidden dimension of layers = 4*num_of_layers\n",
    "    for i in range(num_of_layers):\n",
    "         model.add(Dense(4*num_of_layers, activation='relu'))\n",
    "        \n",
    "    model.add(Dense(y.shape[1], activation='softmax'))\n",
    "    model.compile(loss='categorical_crossentropy', \n",
    "                      optimizer='adam', \n",
    "                      metrics=['accuracy'])\n",
    "    \n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Uncomment to see the model\n",
    "\n",
    "# model = get_model(5)\n",
    "# model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Scheduler.\n",
    "#We return the train accuracy after 10 epochs\n",
    "\n",
    "@scheduler.serial\n",
    "def objective(**params):\n",
    "    global X, y\n",
    "    model = get_model(**params)\n",
    "    \n",
    "    #train the clf.\n",
    "    model.fit(X, y,\n",
    "          batch_size=32,\n",
    "          epochs=100,\n",
    "          verbose=0)\n",
    "    \n",
    "    score = model.evaluate(X,y, verbose=0)\n",
    "      \n",
    "    return score[1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/sandeep/anaconda3/envs/mango/lib/python3.7/site-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a9bfa781e51b41f8b1d575b24506a4af",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "best parameters: {'num_of_layers': 5}\n",
      "best accuracy: 0.97333336\n"
     ]
    }
   ],
   "source": [
    "conf_dict = dict(num_iteration=5)\n",
    "\n",
    "tuner = Tuner(param_space, objective, conf_dict)\n",
    "results = tuner.maximize()\n",
    "print('best parameters:', results['best_params'])\n",
    "print('best accuracy:', results['best_objective'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'random_params': array([{'num_of_layers': 6}], dtype=object),\n",
       " 'random_params_objective': array([0.93333334], dtype=float32),\n",
       " 'params_tried': array([{'num_of_layers': 6}, {'num_of_layers': 1}, {'num_of_layers': 4},\n",
       "        {'num_of_layers': 5}, {'num_of_layers': 3}, {'num_of_layers': 2}],\n",
       "       dtype=object),\n",
       " 'objective_values': array([0.93333334, 0.33333334, 0.9533333 , 0.97333336, 0.9533333 ,\n",
       "        0.9533333 ], dtype=float32),\n",
       " 'surrogate_values': array([0.93333334, 0.77340587, 0.72558583, 0.98862434, 0.8273985 ,\n",
       "        0.7403795 ]),\n",
       " 'best_objective': 0.97333336,\n",
       " 'best_params': {'num_of_layers': 5}}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}