move credentials to an env file

cloud-annotations · Jan 13, 2021 · 619f97f · 619f97f
1 parent 008752d
commit 619f97f
Show file tree

Hide file tree

Showing 4 changed files with 134 additions and 17 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,6 @@
+data/
+model/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

diff --git a/prepare-data.ipynb b/prepare-data.ipynb
@@ -3,30 +3,77 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "papermill": {
+     "duration": 1.488152,
+     "end_time": "2021-01-13T20:44:49.387618",
+     "exception": false,
+     "start_time": "2021-01-13T20:44:47.899466",
+     "status": "completed"
+    },
+    "tags": []
+   },
    "outputs": [],
    "source": [
-    "!pip install ibm-cos-sdk"
+    "!pip install ibm-cos-sdk python-dotenv"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "papermill": {
+     "duration": 0.091494,
+     "end_time": "2021-01-13T20:44:49.483776",
+     "exception": false,
+     "start_time": "2021-01-13T20:44:49.392282",
+     "status": "completed"
+    },
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "import os\n",
     "import shutil\n",
     "import json\n",
     "import uuid\n",
     "\n",
-    "import ibm_boto3"
+    "import ibm_boto3\n",
+    "\n",
+    "from dotenv import load_dotenv"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "papermill": {
+     "duration": 0.016471,
+     "end_time": "2021-01-13T20:44:49.504840",
+     "exception": false,
+     "start_time": "2021-01-13T20:44:49.488369",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "load_dotenv()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "papermill": {
+     "duration": 0.010609,
+     "end_time": "2021-01-13T20:44:49.520260",
+     "exception": false,
+     "start_time": "2021-01-13T20:44:49.509651",
+     "status": "completed"
+    },
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "bucket = os.getenv(\"BUCKET\", \"\")\n",
@@ -38,7 +85,16 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "papermill": {
+     "duration": 0.864643,
+     "end_time": "2021-01-13T20:44:50.389077",
+     "exception": false,
+     "start_time": "2021-01-13T20:44:49.524434",
+     "status": "completed"
+    },
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "cos = ibm_boto3.resource(\"s3\",\n",
@@ -57,7 +113,16 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "papermill": {
+     "duration": 14.03486,
+     "end_time": "2021-01-13T20:45:04.430418",
+     "exception": false,
+     "start_time": "2021-01-13T20:44:50.395558",
+     "status": "completed"
+    },
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "data_dir = \"data\"\n",
@@ -101,6 +166,18 @@
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
    "version": "3.7.9"
+  },
+  "papermill": {
+   "default_parameters": {},
+   "duration": 17.417327,
+   "end_time": "2021-01-13T20:45:04.652484",
+   "environment_variables": {},
+   "exception": null,
+   "input_path": "/Users/niko/Documents/code/elyra-classification/prepare-data.ipynb",
+   "output_path": "/Users/niko/Documents/code/elyra-classification/prepare-data.ipynb",
+   "parameters": {},
+   "start_time": "2021-01-13T20:44:47.235157",
+   "version": "2.2.2"
   }
  },
  "nbformat": 4,

diff --git a/train.ipynb b/train.ipynb
@@ -3,7 +3,16 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "papermill": {
+     "duration": 1.167889,
+     "end_time": "2021-01-13T20:45:06.435383",
+     "exception": false,
+     "start_time": "2021-01-13T20:45:05.267494",
+     "status": "completed"
+    },
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "!pip install \"tensorflow~=2.0\" \"tensorflow-hub[make_image_classifier]~=0.6\""
@@ -12,7 +21,16 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "papermill": {
+     "duration": 22.252066,
+     "end_time": "2021-01-13T20:45:28.691929",
+     "exception": false,
+     "start_time": "2021-01-13T20:45:06.439863",
+     "status": "completed"
+    },
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "!make_image_classifier \\\n",
@@ -36,7 +54,16 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "papermill": {
+     "duration": 0.133484,
+     "end_time": "2021-01-13T20:45:28.835616",
+     "exception": false,
+     "start_time": "2021-01-13T20:45:28.702132",
+     "status": "completed"
+    },
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "!mv labels.txt model"
@@ -60,8 +87,20 @@
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
    "version": "3.7.9"
+  },
+  "papermill": {
+   "default_parameters": {},
+   "duration": 24.388299,
+   "end_time": "2021-01-13T20:45:29.055848",
+   "environment_variables": {},
+   "exception": null,
+   "input_path": "/Users/niko/Documents/code/elyra-classification/train.ipynb",
+   "output_path": "/Users/niko/Documents/code/elyra-classification/train.ipynb",
+   "parameters": {},
+   "start_time": "2021-01-13T20:45:04.667549",
+   "version": "2.2.2"
   }
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}
diff --git a/train.pipeline b/train.pipeline
@@ -15,17 +15,15 @@
           "app_data": {
             "filename": "prepare-data.ipynb",
             "runtime_image": "tensorflow/tensorflow:2.3.0",
-            "env_vars": [
-              "BUCKET=",
-              "ACCESS_KEY_ID=",
-              "SECRET_ACCESS_KEY=",
-              "ENDPOINT_URL="
-            ],
+            "env_vars": [],
             "include_subdirectories": false,
             "outputs": [
               "data"
             ],
             "invalidNodeError": null,
+            "dependencies": [
+              ".env"
+            ],
             "ui_data": {
               "label": "prepare-data.ipynb",
               "image": "data:image/svg+xml;utf8,%3Csvg%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20width%3D%2216%22%20viewBox%3D%220%200%2022%2022%22%3E%0A%20%20%3Cg%20class%3D%22jp-icon-warn0%20jp-icon-selectable%22%20fill%3D%22%23EF6C00%22%3E%0A%20%20%20%20%3Cpath%20d%3D%22M18.7%203.3v15.4H3.3V3.3h15.4m1.5-1.5H1.8v18.3h18.3l.1-18.3z%22%2F%3E%0A%20%20%20%20%3Cpath%20d%3D%22M16.5%2016.5l-5.4-4.3-5.6%204.3v-11h11z%22%2F%3E%0A%20%20%3C%2Fg%3E%0A%3C%2Fsvg%3E%0A",