Add docker-compose file with Jupyer notebook (#328)

* Add docker-compose file with Jupyer notebook

* Add development version of dockerfile for Feast developer, to avoid mvn dependencies download

* Add end to end test runner

* Add documentation

* Add batch serving functionality to docker compose setup

.dockerignore

@@ -0,0 +1,2 @@
+docs
+charts

docs/getting-started/installing-feast.md

@@ -2,11 +2,50 @@
 
 ## Overview
 
-This installation guide will demonstrate two ways of installing Feast:
+This installation guide will demonstrate three ways of installing Feast:
 
+* [**Docker Compose \(Quickstart\)**](install-feast.md#docker-compose)**:** Allow quick testing on Feast via a sample Jupyter notebook pre-installed with Feast Python SDK.
 * [**Minikube \(Minimal\)**](installing-feast.md#minikube)**:** This installation has no external dependencies, but does not have a historical feature store installed. It allows users to quickly get a feel for Feast.
 * [**Google Kubernetes Engine \(Recommended\):**](installing-feast.md#google-kubernetes-engine) This guide installs a single cluster Feast installation on Google's GKE. It has Google Cloud specific dependencies like BigQuery, Dataflow, and Google Cloud Storage.
 
+
+## Docker Compose
+
+### Overview
+
+A docker compose file is provided to quickly test Feast with official docker images. GCP dependency is optional.
+
+* Define and register feature set
+* Feature ingestion
+* Retrieve features for online serving
+* Updating the feature set
+
+The docker compose setup uses Direct Runner for the Apache Beam jobs.
+
+### 0. Requirements
+
+1. [Docker compose](https://docs.docker.com/compose/install/) should be installed.
+2. Port 6565, 6566 and 8888, 9094 are not in used. Otherwise, modify the port mappings in  `infra/docker-compose/docker-compose.yml` to use unoccupied ports.
+3. For batch serving, you will also need a service account key that has access to GCS and BigQuery. Port 6567 will be used for the batch serving endpoint.
+
+### 1. Step-by-step guide (Online serving)
+1. Navigate to `infra/docker-compose`.
+2. Copy `.env.sample` to `.env`.
+3. `docker-compose up -d`
+4. A jupyter notebook server should be accessible via `localhost:8888`
+5. Please wait a minute or two for the Feast services to be ready before running the notebook. You will know that the services are ready when port `6565` and `6566` starts listening.
+
+### 2. Step-by-step guide (Batch serving)
+1. Navigate to `infra/docker-compose`.
+2. Copy `.env.sample` to `.env`.
+3. Copy your GCP account service key(s) to `infra/docker-compose/gcp-service-accounts`.
+4. Modify the value of `FEAST_<SERVICE_NAME>_GCP_SERVICE_ACCOUNT_KEY` in your `.env` file. It should be the json file name without extension.
+5. Modify the value of `infra/docker-compose/serving/bq-store.yml`. Alternatively, you can also point to a different store configuration file by modifying `FEAST_BATCH_STORE_CONFIG` in your `.env` file.
+5. `docker-compose -f docker-compose.yml -f docker-compose.batch.yml up -d`
+6. A jupyter notebook server should be accessible via `localhost:8888`
+7. Please wait a minute or two for the Feast services to be ready before running the notebook. You will know that the services are ready when port `6565` and `6567` starts listening.
+8. When you are done, run `docker-compose -f docker-compose.yml -f docker-compose.batch.yml down` to shutdown the services.
+
 ## Minikube
 
 ### Overview

infra/docker-compose/.env.sample

@@ -0,0 +1,19 @@
+COMPOSE_PROJECT_NAME=feast
+
+FEAST_VERSION=latest
+
+FEAST_CORE_IMAGE=gcr.io/kf-feast/feast-core
+FEAST_CORE_CONFIG=direct-runner
+FEAST_CORE_GCP_SERVICE_ACCOUNT_KEY=placeholder
+
+FEAST_SERVING_IMAGE=gcr.io/kf-feast/feast-serving
+FEAST_ONLINE_SERVING_CONFIG=online-serving
+FEAST_ONLINE_STORE_CONFIG=redis-store
+FEAST_BATCH_SERVING_CONFIG=batch-serving
+FEAST_BATCH_STORE_CONFIG=bq-store
+FEAST_BATCH_SERVING_GCP_SERVICE_ACCOUNT_KEY=placeholder
+FEAST_JOB_STAGING_LOCATION=gs://your-gcp-project/bucket
+
+FEAST_JUPYTER_IMAGE=gcr.io/kf-feast/feast-jupyter
+FEAST_JUPYTER_GCP_SERVICE_ACCOUNT_KEY=placeholder
+

infra/docker-compose/core/direct-runner.yml

@@ -0,0 +1,3 @@
+feast:
+  jobs:
+    runner: DirectRunner

infra/docker-compose/docker-compose.batch.yml

@@ -0,0 +1,25 @@
+version: "3.7"
+
+services:
+  batch-serving:
+    image: ${FEAST_SERVING_IMAGE}:${FEAST_VERSION}
+    volumes:
+      - ./serving/${FEAST_BATCH_SERVING_CONFIG}.yml:/etc/feast/application.yml
+      - ./serving/${FEAST_BATCH_STORE_CONFIG}.yml:/etc/feast/store.yml
+      - ./gcp-service-accounts/${FEAST_BATCH_SERVING_GCP_SERVICE_ACCOUNT_KEY}.json:/etc/gcloud/service-accounts/key.json
+    depends_on:
+      - core
+      - redis
+    ports:
+      - 6567:6567
+    restart: on-failure
+    environment:
+      GOOGLE_APPLICATION_CREDENTIALS: /etc/gcloud/service-accounts/key.json
+      FEAST_JOB_STAGING_LOCATION: ${FEAST_JOB_STAGING_LOCATION}
+    command:
+      - "java"
+      - "-Xms1024m"
+      - "-Xmx1024m"
+      - "-jar"
+      - "/opt/feast/feast-serving.jar"
+      - "--spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml"

infra/docker-compose/docker-compose.yml

@@ -0,0 +1,84 @@
+version: "3.7"
+
+services:
+  core:
+    image: ${FEAST_CORE_IMAGE}:${FEAST_VERSION}
+    volumes:
+      - ./core/${FEAST_CORE_CONFIG}.yml:/etc/feast/application.yml
+      - ./gcp-service-accounts/${FEAST_CORE_GCP_SERVICE_ACCOUNT_KEY}.json:/etc/gcloud/service-accounts/key.json
+    environment:
+      DB_HOST: db
+      GOOGLE_APPLICATION_CREDENTIALS: /etc/gcloud/service-accounts/key.json
+    restart: on-failure
+    depends_on:
+      - db
+      - kafka
+    ports:
+      - 6565:6565
+    command:
+      - java
+      - -jar
+      - /opt/feast/feast-core.jar
+      - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yaml
+
+  online-serving:
+    image: ${FEAST_SERVING_IMAGE}:${FEAST_VERSION}
+    volumes:
+      - ./serving/${FEAST_ONLINE_SERVING_CONFIG}.yml:/etc/feast/application.yml
+      - ./serving/${FEAST_ONLINE_STORE_CONFIG}.yml:/etc/feast/store.yml
+    depends_on:
+      - core
+      - redis
+    ports:
+      - 6566:6566
+    restart: on-failure
+    command:
+      - java
+      - -jar
+      - /opt/feast/feast-serving.jar
+      - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml
+
+  jupyter:
+    image: ${FEAST_JUPYTER_IMAGE}:${FEAST_VERSION}
+    volumes:
+      - ./jupyter/notebooks:/home/jovyan/feast-notebooks
+      - ./jupyter/features:/home/jovyan/features
+      - ./gcp-service-accounts/${FEAST_JUPYTER_GCP_SERVICE_ACCOUNT_KEY}.json:/etc/gcloud/service-accounts/key.json
+    depends_on:
+      - core
+      - online-serving
+    environment:
+      FEAST_CORE_URL: core:6565
+      FEAST_SERVING_URL: online-serving:6566
+      GOOGLE_APPLICATION_CREDENTIALS: /etc/gcloud/service-accounts/key.json
+    ports:
+      - 8888:8888
+    command:
+      - start-notebook.sh
+      - --NotebookApp.token=''
+
+  redis:
+    image: redis:5-alpine
+
+  kafka:
+    image: confluentinc/cp-kafka:5.2.1
+    environment:
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka:9092,OUTSIDE://localhost:9094
+      KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
+    ports:
+      - 9094:9092
+
+    depends_on:
+      - zookeeper
+
+  zookeeper:
+    image: confluentinc/cp-zookeeper:5.2.1
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+
+  db:
+    image: postgres:12-alpine

infra/docker-compose/gcp-service-accounts/placeholder.json

@@ -0,0 +1 @@
+{}

infra/docker-compose/jupyter/features/cust_trans_fs.yaml

@@ -0,0 +1,11 @@
+name: customer_transactions
+kind: feature_set
+entities:
+- name: customer_id
+  valueType: INT64
+features:
+- name: daily_transactions
+  valueType: FLOAT
+- name: total_transactions
+  valueType: FLOAT
+maxAge: 3600s

infra/docker-compose/jupyter/features/cust_trans_fs_updated.yaml

@@ -0,0 +1,13 @@
+name: customer_transactions
+kind: feature_set
+entities:
+- name: customer_id
+  valueType: INT64
+features:
+- name: daily_transactions
+  valueType: FLOAT
+- name: total_transactions
+  valueType: FLOAT
+- name: discounts
+  valueType: FLOAT
+maxAge: 3600s