ray.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'utils.ml_utils' from '/home/darylchang/ml-experiments/utils/ml_utils.py'>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_datasets as tfds\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers, initializers\n",
    "\n",
    "import numpy as np\n",
    "import os\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import importlib\n",
    "import utils.ml_utils as ml_utils\n",
    "importlib.reload(ml_utils)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "ml_utils.init_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Constants\n",
    "IMG_SIZE = 128 # All images in the dataset will be resized to this size\n",
    "BATCH_SIZE = 32 # Batch size\n",
    "\n",
    "# Read in training and validation data\n",
    "train, validation = ml_utils.load_batched_and_resized_dataset(\n",
    "    dataset_name='cats_vs_dogs',\n",
    "    batch_size=BATCH_SIZE,\n",
    "    img_size=IMG_SIZE\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow.keras as keras\n",
    "from ray.tune import track\n",
    "from ray import tune\n",
    "import ray\n",
    "from ray.tune.utils import pin_in_object_store, get_pinned_object\n",
    "\n",
    "ray.shutdown()\n",
    "ray.init(memory=8*10**9, object_store_memory=6*10**9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TuneReporterCallback(keras.callbacks.Callback):\n",
    "    \"\"\"Tune Callback for Keras.\n",
    "    \n",
    "    The callback is invoked every epoch.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, logs={}):\n",
    "        self.iteration = 0\n",
    "        super(TuneReporterCallback, self).__init__()\n",
    "\n",
    "    def on_epoch_end(self, batch, logs={}):\n",
    "        self.iteration += 1\n",
    "        track.log(keras_info=logs, mean_accuracy=logs.get(\"accuracy\"), mean_loss=logs.get(\"loss\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x = np.concatenate([x for x, y in tfds.as_numpy(train)], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_y = np.concatenate([y for x, y in tfds.as_numpy(train)], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_x = np.concatenate([x for x, y in tfds.as_numpy(validation)], axis=0)\n",
    "validation_y = np.concatenate([y for x, y in tfds.as_numpy(validation)], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x_id = pin_in_object_store(train_x)\n",
    "train_y_id = pin_in_object_store(train_y)\n",
    "validation_x_id = pin_in_object_store(validation_x)\n",
    "validation_y_id = pin_in_object_store(validation_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(config):\n",
    "    tr = TuneReporterCallback()\n",
    "    train_x, train_y = get_pinned_object(train_x_id), get_pinned_object(train_y_id)\n",
    "    validation_x, validation_y = get_pinned_object(validation_x_id), get_pinned_object(validation_y_id)\n",
    "    model = ml_utils.build_model(optimizer=keras.optimizers.SGD(lr=config['lr']))\n",
    "    model.fit(\n",
    "        train_x,\n",
    "        train_y,\n",
    "        validation_data=(validation_x, validation_y),\n",
    "        epochs=1,\n",
    "        verbose=1,\n",
    "        callbacks=[tr],\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'tune' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-86779c876683>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m analysis = tune.run(\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0mtrain_model\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mconfig\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m\"lr\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtune\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloguniform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0.001\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mnum_samples\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'tune' is not defined"
     ]
    }
   ],
   "source": [
    "analysis = tune.run(\n",
    "    train_model, \n",
    "    verbose=1,\n",
    "    config={\"lr\": tune.loguniform(0.001, 0.1)},\n",
    "    num_samples=3,\n",
    "    resources_per_trial={\"gpu\": 1}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}