inference-tinyllama1.1b-gguf/tinyllama-gguf-online-endpoint.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azure.ai.ml import MLClient\n",
    "from azure.identity import (\n",
    "    DefaultAzureCredential,\n",
    ")\n",
    "from azure.ai.ml.entities import (\n",
    "    AmlCompute, Model, Environment, BuildContext, CodeConfiguration, \n",
    "    ManagedOnlineEndpoint, ManagedOnlineDeployment, OnlineRequestSettings, ProbeSettings\n",
    ")\n",
    "from azure.ai.ml.constants import AssetTypes\n",
    "\n",
    "import time, sys, json, os\n",
    "from IPython.display import display, JSON"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found the config file in: /config.json\n"
     ]
    }
   ],
   "source": [
    "credential = DefaultAzureCredential()\n",
    "\n",
    "workspace_ml_client = MLClient.from_config(\n",
    "    credential\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Register the model asset in workspace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model tinyllama-gguf already exists.\n"
     ]
    }
   ],
   "source": [
    "model_name = \"tinyllama-gguf\"\n",
    "\n",
    "# Make the models query fail-safe\n",
    "models = workspace_ml_client.models.list()\n",
    "model = next((m for m in models if m.name == model_name), None)\n",
    "\n",
    "if model is not None:\n",
    "    print(f\"Model {model.name} already exists.\")\n",
    "else:\n",
    "    # Register the foundation model from local as Model asset in ml workspace\n",
    "    model = Model(\n",
    "        path=\"models\",\n",
    "        type=AssetTypes.CUSTOM_MODEL,\n",
    "        name=model_name,\n",
    "        description=\"tinyllamagguf - gguf format model\",\n",
    "    )\n",
    "\n",
    "    workspace_ml_client.create_or_update(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Register the environment asset in workspace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Environment tinyllama-gguf-env already exists.\n"
     ]
    }
   ],
   "source": [
    "env_name = \"tinyllama-gguf-env\"\n",
    "\n",
    "# Make the environments query fail-safe\n",
    "envs = workspace_ml_client.environments.list()\n",
    "env = next((e for e in envs if e.name == env_name), None)\n",
    "\n",
    "# Flag to indicate if the environment is modified. \n",
    "# Set it manually:: If True, old env version is used. If False, new env version is created.\n",
    "is_env_earlierone = True\n",
    "\n",
    "if env is not None and is_env_earlierone:\n",
    "    print(f\"Environment {env_name} already exists.\")\n",
    "else:\n",
    "    print(\"new\")\n",
    "    # Create a new environment based on the curated one\n",
    "    custom_env = Environment(\n",
    "        name=env_name,\n",
    "        description=\"Custom environment with additional dependencies\",\n",
    "        build=BuildContext(path=\"../env/cpu\")\n",
    "    )\n",
    "\n",
    "    # Register the custom environment\n",
    "    workspace_ml_client.environments.create_or_update(custom_env)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create managed online endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create online endpoint - endpoint names need to be unique in a region, hence using timestamp to create unique endpoint name\n",
    "online_endpoint_name = \"tinyllama-gguf-ep\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# managed endpoint\n",
    "endpoint = ManagedOnlineEndpoint(\n",
    "    name=online_endpoint_name,\n",
    "    description=\"Online endpoint for tinyllama gguf\",\n",
    "    auth_mode=\"key\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# managed endpoint create async call\n",
    "workspace_ml_client.begin_create_or_update(endpoint).wait()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create managed online deployment for the endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "deployment_name = \"deploy01\"\n",
    "model = f\"{model_name}@latest\"\n",
    "env = f\"{env_name}@latest\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# managed endpoint deployment\n",
    "demo_deployment = ManagedOnlineDeployment(\n",
    "    name=deployment_name,\n",
    "    endpoint_name=online_endpoint_name,\n",
    "    model=model,\n",
    "    environment=env,\n",
    "    code_configuration=CodeConfiguration(\n",
    "        code=\"../onlinescoring\",\n",
    "        scoring_script=\"score.py\",\n",
    "    ),\n",
    "    instance_type=\"Standard_F16s_v2\",\n",
    "    instance_count=1,\n",
    "    request_settings=OnlineRequestSettings(\n",
    "        request_timeout_ms=120000,\n",
    "    ),\n",
    "    liveness_probe=ProbeSettings(\n",
    "        initial_delay=600\n",
    "    ),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Check: endpoint tinyllama-gguf-ep exists\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "..................................................................."
     ]
    }
   ],
   "source": [
    "# managed endpoint deplyment create async call\n",
    "workspace_ml_client.online_deployments.begin_create_or_update(deployment=demo_deployment).wait()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# update traffic to the deployment for 100%\n",
    "endpoint.traffic = {deployment_name: 100}\n",
    "workspace_ml_client.begin_create_or_update(endpoint).result()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test endpoint with sample data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"output\": \"1. Champs-Elys\\u00e9es\\n2. Eiffel Tower\\n3. Notre Dame Cathedral\\n4. Arc de Triomphe\\n5. Louvre Museum\\n6. Giverny Garden\\n7. The Seine River\\n8. Montmartre\\n9. Jardin des Plantes\\n10. Mus\\u00e9e d'Orsay\\n\\nRemember to always check the weather and safety rules before visiting these places, especially if you plan on seeing any landmarks or monuments.\"}\n"
     ]
    }
   ],
   "source": [
    "# score the request file using the online endpoint with the azureml endpoint invoke method\n",
    "response = workspace_ml_client.online_endpoints.invoke(\n",
    "    endpoint_name=online_endpoint_name,\n",
    "    request_file=\"../payload/request1.json\",\n",
    ")\n",
    "\n",
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"output\": \"The capital of India is New Delhi.\"}\n"
     ]
    }
   ],
   "source": [
    "# score the request file using the online endpoint with the azureml endpoint invoke method\n",
    "response = workspace_ml_client.online_endpoints.invoke(\n",
    "    endpoint_name=online_endpoint_name,\n",
    "    request_file=\"../payload/request3.json\",\n",
    ")\n",
    "\n",
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "azureml_py310_sdkv2",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}