RAT.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyPZD9mbtobIrZh9G6JQlxcQ",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/hdfeos/whomovedmycheese.io/blob/main/RAT.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "This is Amazon Sidewalk data processing example."
      ],
      "metadata": {
        "id": "f2mvJ53Sxifx"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 88,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        },
        "id": "fEnUM1QvIrce",
        "outputId": "a8c7ef3d-a840-4037-f93f-071240b499bc"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-06d056c7-552a-4030-b94a-9e53195fbc88\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-06d056c7-552a-4030-b94a-9e53195fbc88\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script>// Copyright 2017 Google LLC\n",
              "//\n",
              "// Licensed under the Apache License, Version 2.0 (the \"License\");\n",
              "// you may not use this file except in compliance with the License.\n",
              "// You may obtain a copy of the License at\n",
              "//\n",
              "//      http://www.apache.org/licenses/LICENSE-2.0\n",
              "//\n",
              "// Unless required by applicable law or agreed to in writing, software\n",
              "// distributed under the License is distributed on an \"AS IS\" BASIS,\n",
              "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
              "// See the License for the specific language governing permissions and\n",
              "// limitations under the License.\n",
              "\n",
              "/**\n",
              " * @fileoverview Helpers for google.colab Python module.\n",
              " */\n",
              "(function(scope) {\n",
              "function span(text, styleAttributes = {}) {\n",
              "  const element = document.createElement('span');\n",
              "  element.textContent = text;\n",
              "  for (const key of Object.keys(styleAttributes)) {\n",
              "    element.style[key] = styleAttributes[key];\n",
              "  }\n",
              "  return element;\n",
              "}\n",
              "\n",
              "// Max number of bytes which will be uploaded at a time.\n",
              "const MAX_PAYLOAD_SIZE = 100 * 1024;\n",
              "\n",
              "function _uploadFiles(inputId, outputId) {\n",
              "  const steps = uploadFilesStep(inputId, outputId);\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  // Cache steps on the outputElement to make it available for the next call\n",
              "  // to uploadFilesContinue from Python.\n",
              "  outputElement.steps = steps;\n",
              "\n",
              "  return _uploadFilesContinue(outputId);\n",
              "}\n",
              "\n",
              "// This is roughly an async generator (not supported in the browser yet),\n",
              "// where there are multiple asynchronous steps and the Python side is going\n",
              "// to poll for completion of each step.\n",
              "// This uses a Promise to block the python side on completion of each step,\n",
              "// then passes the result of the previous step as the input to the next step.\n",
              "function _uploadFilesContinue(outputId) {\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  const steps = outputElement.steps;\n",
              "\n",
              "  const next = steps.next(outputElement.lastPromiseValue);\n",
              "  return Promise.resolve(next.value.promise).then((value) => {\n",
              "    // Cache the last promise value to make it available to the next\n",
              "    // step of the generator.\n",
              "    outputElement.lastPromiseValue = value;\n",
              "    return next.value.response;\n",
              "  });\n",
              "}\n",
              "\n",
              "/**\n",
              " * Generator function which is called between each async step of the upload\n",
              " * process.\n",
              " * @param {string} inputId Element ID of the input file picker element.\n",
              " * @param {string} outputId Element ID of the output display.\n",
              " * @return {!Iterable<!Object>} Iterable of next steps.\n",
              " */\n",
              "function* uploadFilesStep(inputId, outputId) {\n",
              "  const inputElement = document.getElementById(inputId);\n",
              "  inputElement.disabled = false;\n",
              "\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  outputElement.innerHTML = '';\n",
              "\n",
              "  const pickedPromise = new Promise((resolve) => {\n",
              "    inputElement.addEventListener('change', (e) => {\n",
              "      resolve(e.target.files);\n",
              "    });\n",
              "  });\n",
              "\n",
              "  const cancel = document.createElement('button');\n",
              "  inputElement.parentElement.appendChild(cancel);\n",
              "  cancel.textContent = 'Cancel upload';\n",
              "  const cancelPromise = new Promise((resolve) => {\n",
              "    cancel.onclick = () => {\n",
              "      resolve(null);\n",
              "    };\n",
              "  });\n",
              "\n",
              "  // Wait for the user to pick the files.\n",
              "  const files = yield {\n",
              "    promise: Promise.race([pickedPromise, cancelPromise]),\n",
              "    response: {\n",
              "      action: 'starting',\n",
              "    }\n",
              "  };\n",
              "\n",
              "  cancel.remove();\n",
              "\n",
              "  // Disable the input element since further picks are not allowed.\n",
              "  inputElement.disabled = true;\n",
              "\n",
              "  if (!files) {\n",
              "    return {\n",
              "      response: {\n",
              "        action: 'complete',\n",
              "      }\n",
              "    };\n",
              "  }\n",
              "\n",
              "  for (const file of files) {\n",
              "    const li = document.createElement('li');\n",
              "    li.append(span(file.name, {fontWeight: 'bold'}));\n",
              "    li.append(span(\n",
              "        `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n",
              "        `last modified: ${\n",
              "            file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n",
              "                                    'n/a'} - `));\n",
              "    const percent = span('0% done');\n",
              "    li.appendChild(percent);\n",
              "\n",
              "    outputElement.appendChild(li);\n",
              "\n",
              "    const fileDataPromise = new Promise((resolve) => {\n",
              "      const reader = new FileReader();\n",
              "      reader.onload = (e) => {\n",
              "        resolve(e.target.result);\n",
              "      };\n",
              "      reader.readAsArrayBuffer(file);\n",
              "    });\n",
              "    // Wait for the data to be ready.\n",
              "    let fileData = yield {\n",
              "      promise: fileDataPromise,\n",
              "      response: {\n",
              "        action: 'continue',\n",
              "      }\n",
              "    };\n",
              "\n",
              "    // Use a chunked sending to avoid message size limits. See b/62115660.\n",
              "    let position = 0;\n",
              "    do {\n",
              "      const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n",
              "      const chunk = new Uint8Array(fileData, position, length);\n",
              "      position += length;\n",
              "\n",
              "      const base64 = btoa(String.fromCharCode.apply(null, chunk));\n",
              "      yield {\n",
              "        response: {\n",
              "          action: 'append',\n",
              "          file: file.name,\n",
              "          data: base64,\n",
              "        },\n",
              "      };\n",
              "\n",
              "      let percentDone = fileData.byteLength === 0 ?\n",
              "          100 :\n",
              "          Math.round((position / fileData.byteLength) * 100);\n",
              "      percent.textContent = `${percentDone}% done`;\n",
              "\n",
              "    } while (position < fileData.byteLength);\n",
              "  }\n",
              "\n",
              "  // All done.\n",
              "  yield {\n",
              "    response: {\n",
              "      action: 'complete',\n",
              "    }\n",
              "  };\n",
              "}\n",
              "\n",
              "scope.google = scope.google || {};\n",
              "scope.google.colab = scope.google.colab || {};\n",
              "scope.google.colab._files = {\n",
              "  _uploadFiles,\n",
              "  _uploadFilesContinue,\n",
              "};\n",
              "})(self);\n",
              "</script> "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving report.csv to report (1).csv\n"
          ]
        }
      ],
      "source": [
        "import pandas as pd\n",
        "\n",
        "from google.colab import files\n",
        "\n",
        "uploaded = files.upload()\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for fn in uploaded.keys():\n",
        "  df = pd.read_csv(fn)\n"
      ],
      "metadata": {
        "id": "nsV60z3RLFiF"
      },
      "execution_count": 89,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "display(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "h1UCgsD8Ormd",
        "outputId": "3ac87c54-1a4f-414a-d297-0c70de4f0281"
      },
      "execution_count": 90,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                timestamp  LQI  GPS_latitude  GPS_longitude  GPS_accuracy\n",
              "0     2023-04-07 18:23:11    4     40.061177     -88.272761          19.0\n",
              "1     2023-04-07 18:23:02    4     40.061209     -88.272832          21.0\n",
              "2     2023-04-07 18:22:51    4     40.061222     -88.272902          18.0\n",
              "3     2023-04-07 18:22:38    4     40.061152     -88.272832          15.0\n",
              "4     2023-04-07 18:22:32    4     40.061084     -88.272793          11.0\n",
              "...                   ...  ...           ...            ...           ...\n",
              "7223  2023-04-06 21:03:42    4           NaN            NaN           NaN\n",
              "7224  2023-04-06 21:03:27    4           NaN            NaN           NaN\n",
              "7225  2023-04-06 21:03:17    4           NaN            NaN           NaN\n",
              "7226  2023-04-06 21:03:07    4           NaN            NaN           NaN\n",
              "7227  2023-04-06 21:02:56    4           NaN            NaN           NaN\n",
              "\n",
              "[7228 rows x 5 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-0c11e780-2335-49f0-abac-7e68c38f5889\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>timestamp</th>\n",
              "      <th>LQI</th>\n",
              "      <th>GPS_latitude</th>\n",
              "      <th>GPS_longitude</th>\n",
              "      <th>GPS_accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2023-04-07 18:23:11</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061177</td>\n",
              "      <td>-88.272761</td>\n",
              "      <td>19.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2023-04-07 18:23:02</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061209</td>\n",
              "      <td>-88.272832</td>\n",
              "      <td>21.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2023-04-07 18:22:51</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061222</td>\n",
              "      <td>-88.272902</td>\n",
              "      <td>18.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2023-04-07 18:22:38</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061152</td>\n",
              "      <td>-88.272832</td>\n",
              "      <td>15.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2023-04-07 18:22:32</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061084</td>\n",
              "      <td>-88.272793</td>\n",
              "      <td>11.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7223</th>\n",
              "      <td>2023-04-06 21:03:42</td>\n",
              "      <td>4</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7224</th>\n",
              "      <td>2023-04-06 21:03:27</td>\n",
              "      <td>4</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7225</th>\n",
              "      <td>2023-04-06 21:03:17</td>\n",
              "      <td>4</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7226</th>\n",
              "      <td>2023-04-06 21:03:07</td>\n",
              "      <td>4</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7227</th>\n",
              "      <td>2023-04-06 21:02:56</td>\n",
              "      <td>4</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>7228 rows × 5 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0c11e780-2335-49f0-abac-7e68c38f5889')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-0c11e780-2335-49f0-abac-7e68c38f5889 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-0c11e780-2335-49f0-abac-7e68c38f5889');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from numpy import NaN\n",
        " # Filter NaN and filter by date\n",
        "df_n = df[df.GPS_latitude.notnull() & (df.timestamp < '2023-04-07 00:00:00')]\n",
        "display(df_n)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "CGOw376pPhy5",
        "outputId": "5d57115c-8836-490e-c741-0c5c313d03dd"
      },
      "execution_count": 91,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                timestamp  LQI  GPS_latitude  GPS_longitude  GPS_accuracy\n",
              "6166  2023-04-06 23:59:53    4     40.061161     -88.272774          12.0\n",
              "6167  2023-04-06 23:59:43    4     40.061171     -88.272780          18.0\n",
              "6168  2023-04-06 23:59:36    4     40.061168     -88.272780          13.0\n",
              "6169  2023-04-06 23:59:24    4     40.061142     -88.272774          11.0\n",
              "6170  2023-04-06 23:59:17    4     40.061139     -88.272774           6.0\n",
              "...                   ...  ...           ...            ...           ...\n",
              "7218  2023-04-06 21:04:23    4     40.061110     -88.272825          16.0\n",
              "7219  2023-04-06 21:04:16    4     40.061110     -88.272825          22.0\n",
              "7220  2023-04-06 21:04:07    4     40.061088     -88.272780          33.0\n",
              "7221  2023-04-06 21:04:01    4     40.061123     -88.272787          60.0\n",
              "7222  2023-04-06 21:03:52    4     40.061065     -88.272665         106.0\n",
              "\n",
              "[880 rows x 5 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8d4bb2e1-abf3-44f4-ab9a-9ab95e1edef6\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>timestamp</th>\n",
              "      <th>LQI</th>\n",
              "      <th>GPS_latitude</th>\n",
              "      <th>GPS_longitude</th>\n",
              "      <th>GPS_accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>6166</th>\n",
              "      <td>2023-04-06 23:59:53</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061161</td>\n",
              "      <td>-88.272774</td>\n",
              "      <td>12.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6167</th>\n",
              "      <td>2023-04-06 23:59:43</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061171</td>\n",
              "      <td>-88.272780</td>\n",
              "      <td>18.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6168</th>\n",
              "      <td>2023-04-06 23:59:36</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061168</td>\n",
              "      <td>-88.272780</td>\n",
              "      <td>13.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6169</th>\n",
              "      <td>2023-04-06 23:59:24</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061142</td>\n",
              "      <td>-88.272774</td>\n",
              "      <td>11.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6170</th>\n",
              "      <td>2023-04-06 23:59:17</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061139</td>\n",
              "      <td>-88.272774</td>\n",
              "      <td>6.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7218</th>\n",
              "      <td>2023-04-06 21:04:23</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061110</td>\n",
              "      <td>-88.272825</td>\n",
              "      <td>16.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7219</th>\n",
              "      <td>2023-04-06 21:04:16</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061110</td>\n",
              "      <td>-88.272825</td>\n",
              "      <td>22.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7220</th>\n",
              "      <td>2023-04-06 21:04:07</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061088</td>\n",
              "      <td>-88.272780</td>\n",
              "      <td>33.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7221</th>\n",
              "      <td>2023-04-06 21:04:01</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061123</td>\n",
              "      <td>-88.272787</td>\n",
              "      <td>60.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7222</th>\n",
              "      <td>2023-04-06 21:03:52</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061065</td>\n",
              "      <td>-88.272665</td>\n",
              "      <td>106.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>880 rows × 5 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8d4bb2e1-abf3-44f4-ab9a-9ab95e1edef6')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8d4bb2e1-abf3-44f4-ab9a-9ab95e1edef6 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8d4bb2e1-abf3-44f4-ab9a-9ab95e1edef6');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Sort data by time.\n",
        "df_s = df_n.sort_values('timestamp')\n",
        "display(df_s)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "G_yczI5Cx5Qy",
        "outputId": "cdc52d6f-c90a-473c-8bcc-9bd9144210ac"
      },
      "execution_count": 92,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                timestamp  LQI  GPS_latitude  GPS_longitude  GPS_accuracy\n",
              "7222  2023-04-06 21:03:52    4     40.061065     -88.272665         106.0\n",
              "7221  2023-04-06 21:04:01    4     40.061123     -88.272787          60.0\n",
              "7220  2023-04-06 21:04:07    4     40.061088     -88.272780          33.0\n",
              "7219  2023-04-06 21:04:16    4     40.061110     -88.272825          22.0\n",
              "7218  2023-04-06 21:04:23    4     40.061110     -88.272825          16.0\n",
              "...                   ...  ...           ...            ...           ...\n",
              "6170  2023-04-06 23:59:17    4     40.061139     -88.272774           6.0\n",
              "6169  2023-04-06 23:59:24    4     40.061142     -88.272774          11.0\n",
              "6168  2023-04-06 23:59:36    4     40.061168     -88.272780          13.0\n",
              "6167  2023-04-06 23:59:43    4     40.061171     -88.272780          18.0\n",
              "6166  2023-04-06 23:59:53    4     40.061161     -88.272774          12.0\n",
              "\n",
              "[880 rows x 5 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-df1e27b6-f4ec-4e88-95ea-303a3c1ccc55\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>timestamp</th>\n",
              "      <th>LQI</th>\n",
              "      <th>GPS_latitude</th>\n",
              "      <th>GPS_longitude</th>\n",
              "      <th>GPS_accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>7222</th>\n",
              "      <td>2023-04-06 21:03:52</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061065</td>\n",
              "      <td>-88.272665</td>\n",
              "      <td>106.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7221</th>\n",
              "      <td>2023-04-06 21:04:01</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061123</td>\n",
              "      <td>-88.272787</td>\n",
              "      <td>60.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7220</th>\n",
              "      <td>2023-04-06 21:04:07</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061088</td>\n",
              "      <td>-88.272780</td>\n",
              "      <td>33.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7219</th>\n",
              "      <td>2023-04-06 21:04:16</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061110</td>\n",
              "      <td>-88.272825</td>\n",
              "      <td>22.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7218</th>\n",
              "      <td>2023-04-06 21:04:23</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061110</td>\n",
              "      <td>-88.272825</td>\n",
              "      <td>16.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6170</th>\n",
              "      <td>2023-04-06 23:59:17</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061139</td>\n",
              "      <td>-88.272774</td>\n",
              "      <td>6.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6169</th>\n",
              "      <td>2023-04-06 23:59:24</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061142</td>\n",
              "      <td>-88.272774</td>\n",
              "      <td>11.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6168</th>\n",
              "      <td>2023-04-06 23:59:36</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061168</td>\n",
              "      <td>-88.272780</td>\n",
              "      <td>13.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6167</th>\n",
              "      <td>2023-04-06 23:59:43</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061171</td>\n",
              "      <td>-88.272780</td>\n",
              "      <td>18.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6166</th>\n",
              "      <td>2023-04-06 23:59:53</td>\n",
              "      <td>4</td>\n",
              "      <td>40.061161</td>\n",
              "      <td>-88.272774</td>\n",
              "      <td>12.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>880 rows × 5 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-df1e27b6-f4ec-4e88-95ea-303a3c1ccc55')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-df1e27b6-f4ec-4e88-95ea-303a3c1ccc55 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-df1e27b6-f4ec-4e88-95ea-303a3c1ccc55');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pandas.core.internals.base import default_index\n",
        "import datetime\n",
        "s = pd.to_datetime(df_n['timestamp']).astype(int) / 10**9\n",
        "display(s)\n",
        "\n",
        "import h5py\n",
        "with h5py.File('RAT_demo.h5', 'w') as f:\n",
        "            f['Time'] = s\n",
        "            f['Time'].attrs['units'] = \"seconds since 1970-01-01 00:00:00\"\n",
        "            f['Latitude'] = df_n.GPS_latitude\n",
        "            f['Latitude'].attrs['units'] = \"degrees_north\"\n",
        "            f['Longitude'] = df_n.GPS_longitude\n",
        "            f['Longitude'].attrs['units'] = \"degrees_east\"\n",
        "            f['Altitude'] = df_n.GPS_accuracy\n",
        "            f['Altitude'].attrs['units'] = \"m\"\n",
        "            f['LQI'] = df_n.LQI #  Link Quality Indication\n",
        "            \n",
        "            # Creating dimension scales\n",
        "            h5py.h5ds.set_scale(f['Time'].id)\n",
        "\n",
        "            f['Altitude'].dims[0].attach_scale(f['Time'])\n",
        "\n",
        "            f['Latitude'].dims[0].attach_scale(f['Time'])\n",
        "\n",
        "            f['Longitude'].dims[0].attach_scale(f['Time'])\n",
        "\n",
        "            f['LQI'].dims[0].attach_scale(f['Time'])"
      ],
      "metadata": {
        "id": "CmJGnA4EK2Ws",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 231
        },
        "outputId": "e36c6b4f-51c7-4978-efac-293778c813a5"
      },
      "execution_count": 103,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "6166    1.680826e+09\n",
              "6167    1.680826e+09\n",
              "6168    1.680826e+09\n",
              "6169    1.680826e+09\n",
              "6170    1.680826e+09\n",
              "            ...     \n",
              "7218    1.680815e+09\n",
              "7219    1.680815e+09\n",
              "7220    1.680815e+09\n",
              "7221    1.680815e+09\n",
              "7222    1.680815e+09\n",
              "Name: timestamp, Length: 880, dtype: float64"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "files.download('RAT_demo.h5')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "_zslrekyMdDC",
        "outputId": "c40bece9-246f-4aac-f468-72f3808de5f2"
      },
      "execution_count": 104,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_463d7025-0c4f-412c-aeb2-ac609249a4b1\", \"RAT_demo.h5\", 43392)"
            ]
          },
          "metadata": {}
        }
      ]
    }
  ]
}