Housekeeping of Feast CI, docker compose, and notebooks (#916)

* Add caching to core and serving builds * Fix flaky e2e tests * Simplify docker-compose setup * Add kafka broker to docker compose test * Rename batch to historical * Update troubleshooting.md * Fix casing for FEAST_HISTORICAL_SERVING_URL * Remove f-string prefix * Fix missing key in docker compose setup * Fix documentation typos * Add jupyter to load test * Add docker compose back to load-test * Clean up FEAST_VERSION in load test * Fix artifact output for load test * Revert e2e test order
feast-dev · Aug 2, 2020 · ffd0647 · ffd0647
1 parent d3295a2
commit ffd0647
Show file tree

Hide file tree

Showing 25 changed files with 268 additions and 200 deletions.
diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml
@@ -7,17 +7,23 @@ jobs:
     runs-on: [self-hosted]
     strategy:
       matrix:
-        component: [core, serving]
+        component: [core, serving, jupyter]
     env:
       GITHUB_PR_SHA: ${{ github.event.pull_request.head.sha }}
       REGISTRY: gcr.io/kf-feast
+      MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2019-10-24.tar
     steps:
       - uses: actions/checkout@v2
       - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
         with:
           version: '290.0.1'
           export_default_credentials: true
       - run: gcloud auth configure-docker --quiet
+      - name: Get m2 cache
+        run: |
+          infra/scripts/download-maven-cache.sh \
+          --archive-uri ${MAVEN_CACHE} \
+          --output-dir .
       - name: Build image
         run: make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA}
       - name: Push image

diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml
@@ -12,13 +12,20 @@ jobs:
     strategy:
       matrix:
         component: [core, serving, jupyter, ci]
+    env:
+      MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2019-10-24.tar
     steps:
       - uses: actions/checkout@v2
       - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
         with:
           version: '290.0.1'
           export_default_credentials: true
       - run: gcloud auth configure-docker --quiet
+      - name: Get m2 cache
+        run: |
+          infra/scripts/download-maven-cache.sh \
+          --archive-uri ${MAVEN_CACHE} \
+          --output-dir .
       - name: Build image
         run: make build-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA}
       - name: Push image

diff --git a/docs/administration/troubleshooting.md b/docs/administration/troubleshooting.md
@@ -15,8 +15,8 @@ docker ps
 ```text
 CONTAINER ID        IMAGE                                  COMMAND                  CREATED             STATUS              PORTS                                            NAMES
 d7447205bced        jupyter/datascience-notebook:latest    "tini -g -- start-no…"   2 minutes ago       Up 2 minutes        0.0.0.0:8888->8888/tcp                           feast_jupyter_1
-8e49dbe81b92        gcr.io/kf-feast/feast-serving:latest   "java -Xms1024m -Xmx…"   2 minutes ago       Up 5 seconds        0.0.0.0:6567->6567/tcp                           feast_batch-serving_1
-b859494bd33a        gcr.io/kf-feast/feast-serving:latest   "java -jar /opt/feas…"   2 minutes ago       Up About a minute   0.0.0.0:6566->6566/tcp                           feast_online-serving_1
+8e49dbe81b92        gcr.io/kf-feast/feast-serving:latest   "java -Xms1024m -Xmx…"   2 minutes ago       Up 5 seconds        0.0.0.0:6567->6567/tcp                           feast_historical_serving_1
+b859494bd33a        gcr.io/kf-feast/feast-serving:latest   "java -jar /opt/feas…"   2 minutes ago       Up About a minute   0.0.0.0:6566->6566/tcp                           feast_online_serving_1
 5c4962811767        gcr.io/kf-feast/feast-core:latest      "java -jar /opt/feas…"   2 minutes ago       Up 2 minutes        0.0.0.0:6565->6565/tcp                           feast_core_1
 1ba7239e0ae0        confluentinc/cp-kafka:5.2.1            "/etc/confluent/dock…"   2 minutes ago       Up 2 minutes        0.0.0.0:9092->9092/tcp, 0.0.0.0:9094->9094/tcp   feast_kafka_1
 e2779672735c        confluentinc/cp-zookeeper:5.2.1        "/etc/confluent/dock…"   2 minutes ago       Up 2 minutes        2181/tcp, 2888/tcp, 3888/tcp                     feast_zookeeper_1
@@ -59,8 +59,8 @@ You will probably need to connect using the hostnames of services and standard F
 
 ```bash
 export FEAST_CORE_URL=core:6565
-export FEAST_ONLINE_SERVING_URL=online-serving:6566
-export FEAST_BATCH_SERVING_URL=batch-serving:6567
+export FEAST_ONLINE_SERVING_URL=online_serving:6566
+export FEAST_HISTORICAL_SERVING_URL=historical_serving:6567
 ```
 
 ### **Docker Compose \(from outside the docker cluster\)**
@@ -70,7 +70,7 @@ You will probably need to connect using `localhost` and standard ports:
 ```bash
 export FEAST_CORE_URL=localhost:6565
 export FEAST_ONLINE_SERVING_URL=localhost:6566
-export FEAST_BATCH_SERVING_URL=localhost:6567
+export FEAST_HISTORICAL_SERVING_URL=localhost:6567
 ```
 
 ### **Google Kubernetes Engine \(GKE\)**
@@ -81,7 +81,7 @@ You will need to find the external IP of one of the nodes as well as the NodePor
 export FEAST_IP=$(kubectl describe nodes | grep ExternalIP | awk '{print $2}' | head -n 1)
 export FEAST_CORE_URL=${FEAST_IP}:32090
 export FEAST_ONLINE_SERVING_URL=${FEAST_IP}:32091
-export FEAST_BATCH_SERVING_URL=${FEAST_IP}:32092
+export FEAST_HISTORICAL_SERVING_URL=${FEAST_IP}:32092
 ```
 
 `netcat`, `telnet`, or even `curl` can be used to test whether all services are available and ports are open, but `grpc_cli` is the most powerful. It can be installed from [here](https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md).
@@ -107,7 +107,7 @@ ListProjects
 ### Testing Feast Batch Serving and Online Serving
 
 ```bash
-grpc_cli ls ${FEAST_BATCH_SERVING_URL} feast.serving.ServingService
+grpc_cli ls ${FEAST_HISTORICAL_SERVING_URL} feast.serving.ServingService
 ```
 
 ```text
@@ -145,11 +145,11 @@ In order to print the logs from these services, please run the commands below.
 ```
 
 ```text
-docker logs -f feast_batch-serving_1
+docker logs -f feast_historical_serving_1
 ```
 
 ```text
-docker logs -f feast_online-serving_1
+docker logs -f feast_online_serving_1
 ```
 
 ### Google Kubernetes Engine

diff --git a/docs/getting-started/deploying-feast/docker-compose.md b/docs/getting-started/deploying-feast/docker-compose.md
@@ -80,12 +80,12 @@ Configure the `.env` file based on your environment. At the very least you have
 | Parameter | Description |
 | :--- | :--- |
 | `FEAST_CORE_GCP_SERVICE_ACCOUNT_KEY` | This should be your service account file name, for example `key.json`. |
-| `FEAST_BATCH_SERVING_GCP_SERVICE_ACCOUNT_KEY` | This should be your service account file name, for example `key.json` |
+| `FEAST_HISTORICAL_SERVING_GCP_SERVICE_ACCOUNT_KEY` | This should be your service account file name, for example `key.json` |
 | `FEAST_JUPYTER_GCP_SERVICE_ACCOUNT_KEY` | This should be your service account file name, for example `key.json` |
 
 ### 3.3 Configure Historical Serving
 
-We will also need to configure the `batch-serving.yml` file inside `infra/docker-compose/serving/`. This configuration is used to retrieve training datasets from Feast. At a minimum you will need to set:
+We will also need to configure the `historical-serving.yml` file inside `infra/docker-compose/serving/`. This configuration is used to retrieve training datasets from Feast. At a minimum you will need to set:
 
 | Parameter | Description |
 | :--- | :--- |

diff --git a/docs/installation/docker-compose.md b/docs/installation/docker-compose.md
@@ -85,7 +85,7 @@ Configure the `.env` file based on your environment. At the very least you have
 | Parameter | Description |
 | :--- | :--- |
 | FEAST\_CORE\_GCP\_SERVICE\_ACCOUNT\_KEY | This should be your service account file name, for example `key.json`. |
-| FEAST\_BATCH\_SERVING\_GCP\_SERVICE\_ACCOUNT\_KEY | This should be your service account file name, for example `key.json` |
+| FEAST\_HISTORICAL\_SERVING\_GCP\_SERVICE\_ACCOUNT\_KEY | This should be your service account file name, for example `key.json` |
 | FEAST\_JUPYTER\_GCP\_SERVICE\_ACCOUNT\_KEY | This should be your service account file name, for example `key.json` |
 | FEAST\_JOB\_STAGING\_LOCATION | Google Cloud Storage bucket that Feast will use to stage data exports and batch retrieval requests, for example `gs://your-gcs-bucket/staging` |
 

diff --git a/docs/installation/gke.md b/docs/installation/gke.md
@@ -80,7 +80,7 @@ For this guide we will use `NodePort` for exposing Feast services. In order to d
 export FEAST_IP=$(kubectl describe nodes | grep ExternalIP | awk '{print $2}' | head -n 1)
 export FEAST_CORE_URL=${FEAST_IP}:32090
 export FEAST_ONLINE_SERVING_URL=${FEAST_IP}:32091
-export FEAST_BATCH_SERVING_URL=${FEAST_IP}:32092
+export FEAST_HISTORICAL_SERVING_URL=${FEAST_IP}:32092
 ```
 
 Add firewall rules to open up ports on your Google Cloud Platform project:

diff --git a/docs/user-guide/feature-retrieval.md b/docs/user-guide/feature-retrieval.md
@@ -116,7 +116,7 @@ The computation of statistics is not enabled by default. To indicate to Feast th
 ```python
 dataset = client.get_historical_features(
     feature_refs=features, 
-    entity_rows=entity_df 
+    entity_rows=entity_df,
     compute_statistics=True
 )
 

diff --git a/examples/basic/basic.ipynb b/examples/basic/basic.ipynb
@@ -43,7 +43,7 @@
     "FEAST_ONLINE_SERVING_URL = os.getenv('FEAST_ONLINE_SERVING_URL', 'localhost:6566')\n",
     "\n",
     "# Feast Batch Serving allows for the retrieval of historical feature data\n",
-    "FEAST_BATCH_SERVING_URL = os.getenv('FEAST_BATCH_SERVING_URL', 'localhost:6567')"
+    "FEAST_HISTORICAL_SERVING_URL = os.getenv('FEAST_HISTORICAL_SERVING_URL', 'localhost:6567')"
    ]
   },
   {
@@ -466,8 +466,8 @@
    "source": [
     "online_features = client.get_online_features(\n",
     "    feature_refs=[\n",
-    "        f\"daily_transactions\",\n",
-    "        f\"total_transactions\",\n",
+    "        \"daily_transactions\",\n",
+    "        \"total_transactions\",\n",
     "    ],\n",
     "    entity_rows=[\n",
     "        {\n",
@@ -592,7 +592,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "batch_client = Client(core_url=FEAST_CORE_URL, serving_url=FEAST_BATCH_SERVING_URL)"
+    "batch_client = Client(core_url=FEAST_CORE_URL, serving_url=FEAST_HISTORICAL_SERVING_URL)"
    ]
   },
   {
@@ -614,8 +614,8 @@
    "source": [
     "job = batch_client.get_historical_features(\n",
     "                            feature_refs=[\n",
-    "                                f\"daily_transactions\", \n",
-    "                                f\"total_transactions\", \n",
+    "                                \"daily_transactions\",\n",
+    "                                \"total_transactions\",\n",
     "                               ],\n",
     "                            entity_rows=entity_rows\n",
     "                         )"
@@ -754,4 +754,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}
diff --git a/...hurn-prediction-tutorial/Telecom Customer Churn Prediction (with Feast and XGBoost).ipynb b/...hurn-prediction-tutorial/Telecom Customer Churn Prediction (with Feast and XGBoost).ipynb
@@ -6752,7 +6752,7 @@
     "FEAST_ONLINE_SERVING_URL = os.getenv('FEAST_ONLINE_SERVING_URL', 'localhost:6566')\n",
     "\n",
     "# Feast Batch Serving allows for the retrieval of historical feature data\n",
-    "FEAST_BATCH_SERVING_URL = os.getenv('FEAST_BATCH_SERVING_URL', 'localhost:6567')"
+    "FEAST_HISTORICAL_SERVING_URL = os.getenv('FEAST_HISTORICAL_SERVING_URL', 'localhost:6567')"
    ]
   },
   {
@@ -7187,7 +7187,7 @@
     "    def __init__(self, features, target, model_path=None):\n",
     "        # Set up Feast clients to retrieve training and online serving data\n",
     "        self._feast_online_client = Client(serving_url=os.environ['FEAST_ONLINE_SERVING_URL'])\n",
-    "        self._feast_batch_client = Client(serving_url=os.environ['FEAST_BATCH_SERVING_URL'],\n",
+    "        self._feast_batch_client = Client(serving_url=os.environ['FEAST_HISTORICAL_SERVING_URL'],\n",
     "                                          core_url=os.environ['FEAST_CORE_URL'])\n",
     "        \n",
     "        # Path to either save models after training or load models for serving\n",

diff --git a/infra/charts/feast/charts/feast-jupyter/templates/deployment.yaml b/infra/charts/feast/charts/feast-jupyter/templates/deployment.yaml
@@ -38,7 +38,7 @@ spec:
               value: "{{ .Release.Name }}-feast-core:6565"
             - name: FEAST_ONLINE_SERVING_URL
               value: "{{ .Release.Name }}-feast-online-serving:6566"
-            - name: FEAST_BATCH_SERVING_URL
+            - name: FEAST_HISTORICAL_SERVING_URL
               value: "{{ .Release.Name }}-feast-batch-serving:6566"
             {{- if .Values.gcpServiceAccount.enabled }}
             - name: GOOGLE_APPLICATION_CREDENTIALS

diff --git a/infra/charts/feast/templates/tests/test-feast-batch-serving.yaml b/infra/charts/feast/templates/tests/test-feast-batch-serving.yaml
@@ -70,14 +70,14 @@ spec:
       entity_rows_df = df.copy(deep=True).rename(columns={"datetime": "event_timestamp"})[["event_timestamp", "customer_id"]]
       pandavro.to_avro("entity_rows.avro", entity_rows_df)
 
-      batch_serving_job = client.get_historical_features(
+      historical_serving_job = client.get_historical_features(
           entity_rows="file://entity_rows.avro", 
           feature_refs=[
               f"{project}/daily_transactions:1",
               f"{project}/total_transactions:1",
           ]
       )
-      result_df = batch_serving_job.to_dataframe()
+      result_df = historical_serving_job.to_dataframe()
 
       print("Retrieved dataframe: ")
       print(result_df)

diff --git a/infra/docker-compose/.env.sample b/infra/docker-compose/.env.sample
@@ -1,20 +1,7 @@
-# General
 COMPOSE_PROJECT_NAME=feast
-FEAST_VERSION=latest
-FEAST_REPOSITORY_VERSION=v0.5-branch
-
-# Feast Core
-FEAST_CORE_IMAGE=gcr.io/kf-feast/feast-core
-FEAST_CORE_CONFIG=core.yml
-FEAST_CORE_GCP_SERVICE_ACCOUNT_KEY=placeholder.json
-
-# Feast Serving
-FEAST_SERVING_IMAGE=gcr.io/kf-feast/feast-serving
-# Feast Serving - Batch (BigQuery)
-FEAST_BATCH_SERVING_CONFIG=batch-serving.yml
-FEAST_BATCH_SERVING_GCP_SERVICE_ACCOUNT_KEY=placeholder.json
-# Feast Serving - Online (Redis)
-FEAST_ONLINE_SERVING_CONFIG=online-serving.yml
-
-# Jupyter
-FEAST_JUPYTER_GCP_SERVICE_ACCOUNT_KEY=placeholder.json
+FEAST_VERSION=0.6.1
+GCP_SERVICE_ACCOUNT=./gcp-service-accounts/key.json
+FEAST_CORE_CONFIG=./core/core.yml
+FEAST_HISTORICAL_SERVING_CONFIG=./serving/historical-serving.yml
+FEAST_HISTORICAL_SERVING_ENABLED=false
+FEAST_ONLINE_SERVING_CONFIG=./serving/online-serving.yml
diff --git a/infra/docker-compose/core/core.yml b/infra/docker-compose/core/core.yml
@@ -1,12 +1,13 @@
 feast:
   jobs:
-    polling_interval_milliseconds: 30000
+    polling_interval_milliseconds: 20000
     job_update_timeout_seconds: 240
     active_runner: direct
     runners:
       - name: direct
         type: DirectRunner
-        options: {}
+        options:
+          tempLocation: gs://bucket/tempLocation
   stream:
     type: kafka
     options:

diff --git a/infra/docker-compose/docker-compose.batch.yml b/infra/docker-compose/docker-compose.batch.yml
diff --git a/infra/docker-compose/docker-compose.dev.yml b/infra/docker-compose/docker-compose.dev.yml
@@ -25,21 +25,18 @@ services:
       - spring-boot:run
 
   jupyter:
-    image: jupyter/minimal-notebook:619e9cc2fc07
+    image: gcr.io/kf-feast/feast-jupyter:${FEAST_VERSION}
     volumes:
-      - ./gcp-service-accounts/${FEAST_JUPYTER_GCP_SERVICE_ACCOUNT_KEY}:/etc/gcloud/service-accounts/key.json
-      - ./jupyter/startup.sh:/etc/startup.sh
+      - ${GCP_SERVICE_ACCOUNT}:/etc/gcloud/service-accounts/key.json
     depends_on:
       - core
     environment:
       FEAST_CORE_URL: core:6565
-      FEAST_ONLINE_SERVING_URL: online-serving:6566
-      FEAST_BATCH_SERVING_URL: batch-serving:6567
+      FEAST_ONLINE_SERVING_URL: online_serving:6566
+      FEAST_HISTORICAL_SERVING_URL: historical_serving:6567
       GOOGLE_APPLICATION_CREDENTIALS: /etc/gcloud/service-accounts/key.json
-      FEAST_REPOSITORY_VERSION: ${FEAST_REPOSITORY_VERSION}
     ports:
       - 8888:8888
-    command: ["/etc/startup.sh"]
 
   kafka:
     image: confluentinc/cp-kafka:5.2.1

diff --git a/infra/docker-compose/docker-compose.online.yml b/infra/docker-compose/docker-compose.online.yml