Linear_Model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Housing Price Prediction Using Kubeflow and Feast\n",
    "\n",
    "* Predict housing prices using Feast and Kubeflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Setup the notebook\n",
    "- Install `feast` with pip.\n",
    "- Activate user service account with credentials JSON.\n",
    "- Hacks to retrieve essential information for deployments and serving.\n",
    "\n",
    "**NOTE**: This code block might hangs for a long time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import demo_util\n",
    "demo_util.notebook_setup()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import importlib\n",
    "importlib.reload(demo_util)\n",
    "working_dir = \"/home/jovyan/LinearModel\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROJECT = aliz-development\n",
      "APP_NAME = kubeflow-asia\n",
      "ZONE = asia-southeast1-a\n"
     ]
    }
   ],
   "source": [
    "PROJECT, ZONE, APP_NAME = demo_util.get_project_config()\n",
    "print('PROJECT =', PROJECT)\n",
    "print('APP_NAME =', APP_NAME)\n",
    "print('ZONE =', ZONE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import fairing\n",
    "import sys\n",
    "import importlib\n",
    "import deploy_with_fairing\n",
    "import uuid\n",
    "import os\n",
    "import json\n",
    "import requests\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from retrying import retry\n",
    "from feast.sdk.resources.entity import Entity\n",
    "from feast.sdk.resources.storage import Storage\n",
    "from feast.sdk.resources.feature import Feature, Datastore, ValueType\n",
    "from feast.sdk.resources.feature_set import FeatureSet, FileType\n",
    "import feast.specs.FeatureSpec_pb2 as feature_pb\n",
    "\n",
    "from feast.sdk.importer import Importer\n",
    "from feast.sdk.client import Client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Connect to the Feast deployment\n",
    "FEAST_CORE_URL = '10.148.0.99:6565'\n",
    "FEAST_SERVING_URL = '10.148.0.100:6566'\n",
    "STAGING_LOCATION = 'gs://kubecon-19-gojek/staging'\n",
    "fs = Client(core_url=FEAST_CORE_URL,serving_url=FEAST_SERVING_URL, verbose=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load precomputed feature data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>avg_area_income</th>\n",
       "      <th>avg_area_house_age</th>\n",
       "      <th>avg_area_number_of_rooms</th>\n",
       "      <th>avg_area_number_of_bedrooms</th>\n",
       "      <th>area_population</th>\n",
       "      <th>price</th>\n",
       "      <th>area_code</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>79545.458574</td>\n",
       "      <td>5.682861</td>\n",
       "      <td>7.009188</td>\n",
       "      <td>4.09</td>\n",
       "      <td>23086.800503</td>\n",
       "      <td>1.059034e+06</td>\n",
       "      <td>NE 37010-5101</td>\n",
       "      <td>2018-01-01T00:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>79248.642455</td>\n",
       "      <td>6.002900</td>\n",
       "      <td>6.730821</td>\n",
       "      <td>3.09</td>\n",
       "      <td>40173.072174</td>\n",
       "      <td>1.505891e+06</td>\n",
       "      <td>CA 48958</td>\n",
       "      <td>2018-01-01T00:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>61287.067179</td>\n",
       "      <td>5.865890</td>\n",
       "      <td>8.512727</td>\n",
       "      <td>5.13</td>\n",
       "      <td>36882.159400</td>\n",
       "      <td>1.058988e+06</td>\n",
       "      <td>WI 06482-3489</td>\n",
       "      <td>2018-01-01T00:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>63345.240046</td>\n",
       "      <td>7.188236</td>\n",
       "      <td>5.586729</td>\n",
       "      <td>3.26</td>\n",
       "      <td>34310.242831</td>\n",
       "      <td>1.260617e+06</td>\n",
       "      <td>FPO AP 44820</td>\n",
       "      <td>2018-01-01T00:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>59982.197226</td>\n",
       "      <td>5.040555</td>\n",
       "      <td>7.839388</td>\n",
       "      <td>4.23</td>\n",
       "      <td>26354.109472</td>\n",
       "      <td>6.309435e+05</td>\n",
       "      <td>FPO AE 09386</td>\n",
       "      <td>2018-01-01T00:00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   avg_area_income  avg_area_house_age  avg_area_number_of_rooms  \\\n",
       "0     79545.458574            5.682861                  7.009188   \n",
       "1     79248.642455            6.002900                  6.730821   \n",
       "2     61287.067179            5.865890                  8.512727   \n",
       "3     63345.240046            7.188236                  5.586729   \n",
       "4     59982.197226            5.040555                  7.839388   \n",
       "\n",
       "   avg_area_number_of_bedrooms  area_population         price      area_code  \\\n",
       "0                         4.09     23086.800503  1.059034e+06  NE 37010-5101   \n",
       "1                         3.09     40173.072174  1.505891e+06       CA 48958   \n",
       "2                         5.13     36882.159400  1.058988e+06  WI 06482-3489   \n",
       "3                         3.26     34310.242831  1.260617e+06   FPO AP 44820   \n",
       "4                         4.23     26354.109472  6.309435e+05   FPO AE 09386   \n",
       "\n",
       "             timestamp  \n",
       "0  2018-01-01T00:00:00  \n",
       "1  2018-01-01T00:00:00  \n",
       "2  2018-01-01T00:00:00  \n",
       "3  2018-01-01T00:00:00  \n",
       "4  2018-01-01T00:00:00  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('usa_housing.csv', index_col=False)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Register entity and features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully applied entity with name: usa_housing\n",
      "---\n",
      "name: usa_housing\n",
      "description: entity level description\n",
      "\n",
      "Successfully applied feature with id: usa_housing.avg_area_income\n",
      "---\n",
      "id: usa_housing.avg_area_income\n",
      "name: avg_area_income\n",
      "owner: user@website.com\n",
      "description: feature level description\n",
      "valueType: DOUBLE\n",
      "entity: usa_housing\n",
      "dataStores:\n",
      "  serving:\n",
      "    id: SERVING\n",
      "  warehouse:\n",
      "    id: WAREHOUSE\n",
      "\n",
      "Successfully applied feature with id: usa_housing.avg_area_house_age\n",
      "---\n",
      "id: usa_housing.avg_area_house_age\n",
      "name: avg_area_house_age\n",
      "owner: user@website.com\n",
      "description: feature level description\n",
      "valueType: DOUBLE\n",
      "entity: usa_housing\n",
      "dataStores:\n",
      "  serving:\n",
      "    id: SERVING\n",
      "  warehouse:\n",
      "    id: WAREHOUSE\n",
      "\n",
      "Successfully applied feature with id: usa_housing.avg_area_number_of_rooms\n",
      "---\n",
      "id: usa_housing.avg_area_number_of_rooms\n",
      "name: avg_area_number_of_rooms\n",
      "owner: user@website.com\n",
      "description: feature level description\n",
      "valueType: DOUBLE\n",
      "entity: usa_housing\n",
      "dataStores:\n",
      "  serving:\n",
      "    id: SERVING\n",
      "  warehouse:\n",
      "    id: WAREHOUSE\n",
      "\n",
      "Successfully applied feature with id: usa_housing.avg_area_number_of_bedrooms\n",
      "---\n",
      "id: usa_housing.avg_area_number_of_bedrooms\n",
      "name: avg_area_number_of_bedrooms\n",
      "owner: user@website.com\n",
      "description: feature level description\n",
      "valueType: DOUBLE\n",
      "entity: usa_housing\n",
      "dataStores:\n",
      "  serving:\n",
      "    id: SERVING\n",
      "  warehouse:\n",
      "    id: WAREHOUSE\n",
      "\n",
      "Successfully applied feature with id: usa_housing.area_population\n",
      "---\n",
      "id: usa_housing.area_population\n",
      "name: area_population\n",
      "owner: user@website.com\n",
      "description: feature level description\n",
      "valueType: DOUBLE\n",
      "entity: usa_housing\n",
      "dataStores:\n",
      "  serving:\n",
      "    id: SERVING\n",
      "  warehouse:\n",
      "    id: WAREHOUSE\n",
      "\n",
      "Successfully applied feature with id: usa_housing.price\n",
      "---\n",
      "id: usa_housing.price\n",
      "name: price\n",
      "owner: user@website.com\n",
      "description: feature level description\n",
      "valueType: DOUBLE\n",
      "entity: usa_housing\n",
      "dataStores:\n",
      "  serving:\n",
      "    id: SERVING\n",
      "  warehouse:\n",
      "    id: WAREHOUSE\n",
      "\n",
      "Staging file to remote path gs://kubecon-19-gojek/staging/tmp_usa_housing_1558318616283.csv\n",
      "Submitting job with spec:\n",
      " type: file.csv\n",
      "sourceOptions:\n",
      "  path: gs://kubecon-19-gojek/staging/tmp_usa_housing_1558318616283.csv\n",
      "entities:\n",
      "- usa_housing\n",
      "schema:\n",
      "  entityIdColumn: area_code\n",
      "  fields:\n",
      "  - featureId: usa_housing.avg_area_income\n",
      "    name: avg_area_income\n",
      "  - featureId: usa_housing.avg_area_house_age\n",
      "    name: avg_area_house_age\n",
      "  - featureId: usa_housing.avg_area_number_of_rooms\n",
      "    name: avg_area_number_of_rooms\n",
      "  - featureId: usa_housing.avg_area_number_of_bedrooms\n",
      "    name: avg_area_number_of_bedrooms\n",
      "  - featureId: usa_housing.area_population\n",
      "    name: area_population\n",
      "  - featureId: usa_housing.price\n",
      "    name: price\n",
      "  - name: area_code\n",
      "  - name: timestamp\n",
      "  timestampColumn: timestamp\n",
      "\n",
      "Submitted job with id: feastimport1558318618682\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'feastimport1558318618682'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create importer\n",
    "importer = Importer.from_df(df, \n",
    "                           entity='usa_housing', \n",
    "                           owner='user@website.com',  \n",
    "                           staging_location=STAGING_LOCATION,\n",
    "                           id_column='area_code', \n",
    "                           timestamp_column='timestamp',\n",
    "                           serving_store=Datastore(id='SERVING'),\n",
    "                           warehouse_store=Datastore(id='WAREHOUSE'))\n",
    "\n",
    "# Update feature and entity metadata. Ideally you want to update these manually\n",
    "# so that they contain adequate information for the next user\n",
    "importer.entity.description = 'entity level description' \n",
    "for feature_id in importer.features:\n",
    "    importer.features[feature_id].description = 'feature level description'\n",
    "    \n",
    "# Ingest the feature data into the store\n",
    "fs.run(importer, apply_features=True, apply_entity=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define a Feature Set for this project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "ENTITY_ID = 'usa_housing'\n",
    "TRAINING_FEATURES_SET = [\n",
    "    'usa_housing.avg_area_income',\n",
    "    'usa_housing.avg_area_house_age',\n",
    "    'usa_housing.avg_area_number_of_rooms',\n",
    "    'usa_housing.avg_area_number_of_bedrooms',\n",
    "    'usa_housing.area_population',\n",
    "    'usa_housing.price'\n",
    "]\n",
    "\n",
    "feature_set = FeatureSet(entity=ENTITY_ID, \n",
    "                         features=TRAINING_FEATURES_SET)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retrieve a Training Set from Feast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "creating training dataset for features: ['usa_housing.avg_area_income', 'usa_housing.avg_area_house_age', 'usa_housing.avg_area_number_of_rooms', 'usa_housing.avg_area_number_of_bedrooms', 'usa_housing.area_population', 'usa_housing.price']\n",
      "created dataset usa_housing_1558318631722_20180101_20180131: aliz-development.fs_usa_housing.1558318631722_20180101_20180131\n"
     ]
    }
   ],
   "source": [
    "# Retrieve feature data for training from Feast\n",
    "dataset = fs.create_dataset(feature_set, \"2018-01-01\", \"2018-01-31\")\n",
    "training_df = fs.download_dataset_to_df(dataset, STAGING_LOCATION)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train Linear Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2.158e+01 1.656e+05 1.207e+05 1.651e+03 1.520e+01] -2637299.033328295\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "# Train model\n",
    "np.set_printoptions(precision=3)\n",
    "train_data = training_df[[x.split('.')[1] for x in TRAINING_FEATURES_SET]].to_numpy()\n",
    "train_data[:, len(train_data[0]) - 1] = 1\n",
    "Y = training_df['price'].to_numpy()\n",
    "\n",
    "x = np.linalg.lstsq(train_data, Y, rcond=0)[0]\n",
    "m, b = x[:len(train_data[0])-1], x[len(train_data[0])-1]\n",
    "\n",
    "print(m, b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Local Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "845388.7662965846\n"
     ]
    }
   ],
   "source": [
    "SERVING_FEATURES_SET = [\n",
    "    'usa_housing.avg_area_income',\n",
    "    'usa_housing.avg_area_house_age',\n",
    "    'usa_housing.avg_area_number_of_rooms',\n",
    "    'usa_housing.avg_area_number_of_bedrooms',\n",
    "    'usa_housing.area_population'\n",
    "]\n",
    "serving_fs = FeatureSet(entity=ENTITY_ID, features=SERVING_FEATURES_SET)\n",
    "\n",
    "def local_predict(id):\n",
    "    # retrieve features from Feast serving\n",
    "    features = fs.get_serving_data(serving_fs, entity_keys=[id])\n",
    "    x = features.to_numpy()[0][1:]\n",
    "    return sum(m * x) + b\n",
    "\n",
    "p = local_predict('FPO AE 09386')\n",
    "print(p)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "writing to /home/jovyan/simple_model.dat\n"
     ]
    }
   ],
   "source": [
    "MODEL_FILE = 'simple_model.dat'\n",
    "\n",
    "model = {\n",
    "    'm': m.tolist(),\n",
    "    'b': b,\n",
    "    'FEAST_CORE_URL': FEAST_CORE_URL,\n",
    "    'FEAST_SERVING_URL': FEAST_SERVING_URL,\n",
    "    'ENTITY_ID': ENTITY_ID,\n",
    "    'FEATURES_SET': SERVING_FEATURES_SET,\n",
    "}\n",
    "\n",
    "# TODO(gabrielwen): Use PWD instead.\n",
    "model_path = os.path.join(os.getenv('HOME', '/home'), MODEL_FILE)\n",
    "print('writing to', model_path)\n",
    "\n",
    "with open(model_path, 'w+') as f:\n",
    "    json.dump(model, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deploy with Kubeflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fetching cluster endpoint and auth data.\n",
      "kubeconfig entry generated for kubeflow-asia.\n"
     ]
    }
   ],
   "source": [
    "# Need to set up KUBECONFIG. Kubernetes API client depends on it.\n",
    "!gcloud container clusters get-credentials \"$APP_NAME\" --zone \"$ZONE\" --project \"$PROJECT\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "docker registry: gcr.io/aliz-development/fairing-job\n",
      "base image: gcr.io/kubeflow-images-public/fairing-base:v20190516\n"
     ]
    }
   ],
   "source": [
    "DOCKER_REGISTRY = 'gcr.io/{}/fairing-job'.format(PROJECT)\n",
    "BASE_IMAGE = 'gcr.io/kubeflow-images-public/fairing-base:v20190516'\n",
    "SERVING_LABEL = 'kubeflow-fairing-demo'\n",
    "\n",
    "print('docker registry:', DOCKER_REGISTRY)\n",
    "print('base image:', BASE_IMAGE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deploying service with selector kubeflow-fairing-demo-b12c\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Using preprocessor: <fairing.preprocessors.base.BasePreProcessor object at 0x7f50fa510b38>\n",
      "INFO:root:Using builder: <fairing.builders.append.append.AppendBuilder object at 0x7f513165b6d8>\n",
      "WARNING:fairing.builders.append.append:Building image...\n",
      "INFO:root:Creating docker context: /tmp/fairing.context.tar.gz\n",
      "INFO:root:Adding files to context: {'transport_pool_.py', 'main.py', 'notebook_util.py', 'HousingServe.py', 'docker.py', 'cluster.py', 'metadata_.py', 'test_gcp_creds.py', 'trained_ames_model.dat', 'docker_digest_.py', 'test_function_preprocessor.py', 'test_full_notebook.py', 'nested_.py', 'gcs_context.py', 'deploy_with_fairing.py', 'context_source.py', 'ipython_kernel_config.py', 'test_gcp_high_level_apis.py', 'docker_creds_.py', 'docker_image_.py', 'test_full_notebook_preprocessor.py', 'test_base_preprocessor.py', 'conftest.py', 'test_kubeflow_training.py', 'serving.py', 'config.py', 'docker_name_.py', 'test_gcpserving.py', 'dockerfile.py', 'demo_util.py', 'test_gcp.py', '__init__.py', 'oci_compat_.py', 'docker_image_list_.py', 'builder.py', 'test_running_in_notebooks.py', 'docker_session_.py', 'gcpserving.py', 'full_notebook.py', 'config_test.py', 'append.py', 'append_.py', 'monitor_.py', 'gcp.py', 'tfjob.py', 'docker_http_.py', 'v1_compat_.py', 'base.py', 'function_shim.py', 'base_builder.py', 'job.py', 'test_dockerfile.py', 'LabelPrediction.py', 'v2_compat_.py', 'save_.py', 'utils.py', 'hello-world-gpu-pytorch.py', 'constants.py', 'function.py', 'backends.py', 'retry_.py', 'tasks.py', 'setup.py', 'manager.py', 'simple_model.dat', 'converted_notebook.py', 'test_gcs_access.py', 'util_.py', 'HousingTrain.py', 'test_high_level_apis.py', 'runtime_config.py', 'deployer.py'}\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding /home/jovyan/LinearModel/fairing/fairing/__init__.py at /app/fairing/__init__.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding /home/jovyan/LinearModel/fairing/fairing/runtime_config.py at /app/fairing/runtime_config.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding transport_pool_.py at /app/transport_pool_.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding main.py at /app/main.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding notebook_util.py at /app/notebook_util.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding HousingServe.py at /app/HousingServe.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding docker.py at /app/docker.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding cluster.py at /app/cluster.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding metadata_.py at /app/metadata_.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding test_gcp_creds.py at /app/test_gcp_creds.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding trained_ames_model.dat at /app/trained_ames_model.dat\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding docker_digest_.py at /app/docker_digest_.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding test_function_preprocessor.py at /app/test_function_preprocessor.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding test_full_notebook.py at /app/test_full_notebook.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding nested_.py at /app/nested_.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding gcs_context.py at /app/gcs_context.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding deploy_with_fairing.py at /app/deploy_with_fairing.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding context_source.py at /app/context_source.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding ipython_kernel_config.py at /app/ipython_kernel_config.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding test_gcp_high_level_apis.py at /app/test_gcp_high_level_apis.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding docker_creds_.py at /app/docker_creds_.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding docker_image_.py at /app/docker_image_.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding test_full_notebook_preprocessor.py at /app/test_full_notebook_preprocessor.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding test_base_preprocessor.py at /app/test_base_preprocessor.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding conftest.py at /app/conftest.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding test_kubeflow_training.py at /app/test_kubeflow_training.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding serving.py at /app/serving.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding config.py at /app/config.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding docker_name_.py at /app/docker_name_.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding test_gcpserving.py at /app/test_gcpserving.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding dockerfile.py at /app/dockerfile.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding demo_util.py at /app/demo_util.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding test_gcp.py at /app/test_gcp.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding __init__.py at /app/__init__.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding oci_compat_.py at /app/oci_compat_.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding docker_image_list_.py at /app/docker_image_list_.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding builder.py at /app/builder.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding test_running_in_notebooks.py at /app/test_running_in_notebooks.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding docker_session_.py at /app/docker_session_.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding gcpserving.py at /app/gcpserving.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding full_notebook.py at /app/full_notebook.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding config_test.py at /app/config_test.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding append.py at /app/append.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding append_.py at /app/append_.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding monitor_.py at /app/monitor_.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding gcp.py at /app/gcp.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding tfjob.py at /app/tfjob.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding docker_http_.py at /app/docker_http_.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding v1_compat_.py at /app/v1_compat_.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding base.py at /app/base.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding function_shim.py at /app/function_shim.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding base_builder.py at /app/base_builder.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding job.py at /app/job.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding test_dockerfile.py at /app/test_dockerfile.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding LabelPrediction.py at /app/LabelPrediction.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding v2_compat_.py at /app/v2_compat_.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding save_.py at /app/save_.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding utils.py at /app/utils.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding hello-world-gpu-pytorch.py at /app/hello-world-gpu-pytorch.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding constants.py at /app/constants.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding function.py at /app/function.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding backends.py at /app/backends.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding retry_.py at /app/retry_.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding tasks.py at /app/tasks.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding setup.py at /app/setup.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding manager.py at /app/manager.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding simple_model.dat at /app/simple_model.dat\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding converted_notebook.py at /app/converted_notebook.py\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding test_gcs_access.py at /app/test_gcs_access.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding util_.py at /app/util_.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding HousingTrain.py at /app/HousingTrain.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding test_high_level_apis.py at /app/test_high_level_apis.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding runtime_config.py at /app/runtime_config.py\n",
      "INFO:root:Context: /tmp/fairing.context.tar.gz, Adding deployer.py at /app/deployer.py\n",
      "INFO:root:Loading Docker credentials for repository 'gcr.io/kubeflow-images-public/fairing-base:v20190516'\n",
      "INFO:root:Invoking 'docker-credential-gcloud' to obtain Docker credentials.\n",
      "INFO:root:Successfully obtained Docker credentials.\n",
      "WARNING:fairing.builders.append.append:Image successfully built in 2.135091503034346s.\n",
      "WARNING:fairing.builders.append.append:Pushing image gcr.io/aliz-development/fairing-job/fairing-job:99578B64...\n",
      "INFO:root:Loading Docker credentials for repository 'gcr.io/aliz-development/fairing-job/fairing-job:99578B64'\n",
      "INFO:root:Invoking 'docker-credential-gcloud' to obtain Docker credentials.\n",
      "INFO:root:Successfully obtained Docker credentials.\n",
      "WARNING:fairing.builders.append.append:Uploading gcr.io/aliz-development/fairing-job/fairing-job:99578B64\n",
      "INFO:root:Layer sha256:da4d8c6b3f1d6f339ae31ab5624be041df02aff9d3f7f085bee0be70d5510be9 exists, skipping\n",
      "INFO:root:Layer sha256:54f7e8ac135a5f502a6ee9537ef3d64b1cd2fa570dc0a40b4d3b6f7ac81e7486 exists, skipping\n",
      "INFO:root:Layer sha256:2eeb5ce9b9240a928b0a799f9f2601027e2c6b7525394ae5c371f124058489d7 exists, skipping\n",
      "INFO:root:Layer sha256:df57ae687ef5b466a3e956951a4a61f879084f89228b7d34cf20eee4b730f160 exists, skipping\n",
      "INFO:root:Layer sha256:3685478d4d9b72bd0c88c3e3a415d3d25d19042452b672d2befb2b6e6f1d1bbc exists, skipping\n",
      "INFO:root:Layer sha256:620aea26e85367b08cdf1f6768491fb44df6a2a71f7d663f835b1692e849c3ee exists, skipping\n",
      "INFO:root:Layer sha256:425317e6171bfb0c70b2fa348dfcc0ae2166734273450aa05aa04be43b9278d7 exists, skipping\n",
      "INFO:root:Layer sha256:687ed2fb2a0d7da5503478759fd00c23970b65d02b317119b3fb9025038a2594 exists, skipping\n",
      "INFO:root:Layer sha256:0c1db95989906f161007d8ef2a6ef6e0ec64bc15bf2c993fd002edbdfc7aa7df exists, skipping\n",
      "INFO:root:Layer sha256:087a57faf9491b1b82a83e26bc8cc90c90c30e4a4d858b57ddd5b4c2c90095f6 exists, skipping\n",
      "INFO:root:Layer sha256:46b64f8f5cd275ca04071c8cd36170990c26b5cce6bcb440579cb4a9094fc7b4 exists, skipping\n",
      "INFO:root:Layer sha256:a86c8ce7a2ae5a681a1a702074121ed8c6df9b2f044e61c6aacd87e4c80f7299 exists, skipping\n",
      "INFO:root:Layer sha256:a8c5303780550b746a4781e5e4cd893121d8019e971414a2a1273d54486b4eb9 exists, skipping\n",
      "INFO:root:Layer sha256:1e33f14f5f4ae8acc90736eddcf5f1667c15829f343954164b6099d37e40c3e9 exists, skipping\n",
      "INFO:root:Layer sha256:d6341e30912f12f56e18564a3b582853f65376766f5f9d641a68a724ed6db88f exists, skipping\n",
      "INFO:root:Layer sha256:5d71636fb824265e30ff34bf20737c9cdc4f5af28b6bce86f08215c55b89bfab exists, skipping\n",
      "INFO:root:Layer sha256:8a4ffb3e73f0855e0f8617b09dfc9af19ae8e58046b09a9b528f6f47831f9703 pushed.\n",
      "INFO:root:Layer sha256:6bd932530b5849daf35fb127f7a2dd345f6688366e32e7997f4aa74c62aa222e pushed.\n",
      "INFO:root:Finished upload of: gcr.io/aliz-development/fairing-job/fairing-job:99578B64\n",
      "WARNING:fairing.builders.append.append:Pushed image gcr.io/aliz-development/fairing-job/fairing-job:99578B64 in 9.444158874975983s.\n",
      "WARNING:fairing.deployers.serving.serving:Deployment fairing-deployer-b8xmj launched.\n",
      "WARNING:fairing.deployers.serving.serving:In cluster Endpoint http://fairing-service-g7xrf.kubeflow.svc.cluster.local launched.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(<fairing.preprocessors.base.BasePreProcessor at 0x7f50fa510b38>,\n",
       " <fairing.builders.append.append.AppendBuilder at 0x7f513165b6d8>,\n",
       " <fairing.deployers.serving.serving.Serving at 0x7f50f251d6a0>)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# To disambiguate between different deployments.\n",
    "serving_label = SERVING_LABEL + '-' + uuid.uuid4().hex[:4]\n",
    "print('Deploying service with selector', serving_label)\n",
    "\n",
    "# TODO(gabrielwen): Re-deploy has permission issue.\n",
    "importlib.reload(deploy_with_fairing)\n",
    "deploy_with_fairing.deploy(DOCKER_REGISTRY, BASE_IMAGE, serving_label=serving_label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fetch fairing service endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fairing service: kubeflow/fairing-service-g7xrf\n"
     ]
    }
   ],
   "source": [
    "namespace, service_name = demo_util.get_fairing_endpoint(serving_label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Serving with Kubeflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "@retry(wait_exponential_multiplier=1000, wait_exponential_max=5000,\n",
    "       stop_max_delay=2*60*1000)\n",
    "def predict(url, id):\n",
    "    pdata={\n",
    "        'strData': id,\n",
    "    }\n",
    "    serialized_data = json.dumps(pdata)\n",
    "    r = requests.post(url, data={'json':serialized_data}, timeout=5)\n",
    "    return r\n",
    "\n",
    "def extract_prediction(content):\n",
    "    c = json.loads(content)\n",
    "    return c.get('binData', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prediction: 1494937.691618489\n"
     ]
    }
   ],
   "source": [
    "\n",
    "url = \"http://{service_name}.{namespace}.svc.cluster.local:5000/predict\".format(\n",
    "    service_name=service_name,\n",
    "    namespace=namespace)\n",
    "\n",
    "r = predict(url, 'CA 48958')\n",
    "prediction = extract_prediction(r.content)\n",
    "print('prediction:', prediction)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}