Jobs on GPU

.circleci/config.yml

@@ -68,14 +68,12 @@ jobs:
     steps:
       - ci_steps:
           platform: debian
-
   # build-macos:
   #   macos:
   #     xcode: 10.2.1
   #   steps:
   #     - ci_steps:
   #         platform: macosx
-
   build-multiarch-docker:
     machine:
       enabled: true
@@ -100,7 +98,32 @@ jobs:
             sudo docker login -u redisfab -p $DOCKER_REDISFAB_PWD
             make -C opt/build/docker build
             sudo make -C opt/build/docker publish
-
+  build-gpu:
+    machine:
+      enabled: true
+      docker_layer_caching: true
+      resource_class: gpu.nvidia.small
+      image: ubuntu-1604-cuda-10.1:201909-23
+    steps:
+      - checkout
+#      - run:
+#          name: Checkout LFS
+#          command: |
+#            curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
+#            sudo apt-get install -y git-lfs
+#            git lfs install
+#            git lfs pull
+      - run:
+          name: Build
+          command: |
+            docker build -f Dockerfile.gpu-test -t redisai-gpu:latest-x64-bionic-test .
+      - run:
+          name: Test
+          command: |
+            mkdir -p ~/workspace/tests
+            docker run --gpus all -v $HOME/workspace/tests:/build/test/logs -it --rm redisai-gpu:latest-x64-bionic-test
+      - store_test_results:
+          path: ~/workspace/tests
   deploy_package:
     parameters:
       package:
@@ -143,9 +166,13 @@ workflows:
       #- build-multiarch-docker:
       #    filters:
       #      tags:
-      #        only: /.*/
+      #        only: /^v[0-9].*/ 
       #      branches:
-      #        only: master
+      #        ignore: /.*/
+      - build-gpu:
+          filters:
+            tags:
+              only: /.*/
       - deploy_package:
           name: deploy_branch
           package: branch

CMakeLists.txt

@@ -96,7 +96,8 @@ ADD_DEFINITIONS(-DREDISMODULE_EXPERIMENTAL_API)
 
 IF(BUILD_TF)
     FIND_LIBRARY(TF_LIBRARIES NAMES tensorflow
-            PATHS ${depsAbs}/libtensorflow/lib)
+        PATHS ${depsAbs}/libtensorflow/lib)
+    MESSAGE(STATUS "Found TensorFlow Libraries: \"${TF_LIBRARIES}\")")
     IF (NOT TF_LIBRARIES)
         MESSAGE(WARNING "Could not find TensorFlow in ${depsAbs}/libtensorflow/lib. Trying find_package method")
         FIND_PACKAGE(TensorFlow REQUIRED)
@@ -112,8 +113,11 @@ ENDIF()
 #----------------------------------------------------------------------------------------------
 
 IF(BUILD_TFLITE)
-    FIND_LIBRARY(TFLITE_LIBRARIES NAMES tensorflow-lite
+    FIND_LIBRARY(TFLITE_LIBRARIES_1 NAMES tensorflow-lite
             PATHS ${depsAbs}/libtensorflow-lite/lib)
+    FIND_LIBRARY(TFLITE_LIBRARIES_2 NAMES benchmark-lib.a
+            PATHS ${depsAbs}/libtensorflow-lite/lib)
+    SET(TFLITE_LIBRARIES ${TFLITE_LIBRARIES_1} ${TFLITE_LIBRARIES_2})
     MESSAGE(STATUS "Found TensorFlow Lite Libraries: \"${TFLITE_LIBRARIES}\")")
     IF (NOT TFLITE_LIBRARIES)
         MESSAGE(FATAL_ERROR "Could not find TensorFlow Lite")

Dockerfile

@@ -1,4 +1,4 @@
-# BUILD redisfab/redisai-cpu-${OSNICK}:M.m.b-${ARCH}
+# BUILD redisfab/redisai-cpu-${OSNICK}:${VERSION}-${ARCH}
 
 ARG REDIS_VER=5.0.7
 

Dockerfile.gpu

@@ -1,57 +1,57 @@
-FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04 AS builder
+# BUILD redisfab/redisai-gpu:${VERSION}-${ARCH}-${OSNICK}
 
-ENV DEPS "build-essential git ca-certificates curl unzip wget libgomp1 patchelf"
+ARG REDIS_VER=5.0.7
+
+# OSNICK=bionic|centos7|centos6
+ARG OSNICK=bionic
+
+# ARCH=x64|arm64v8|arm32v7
+ARG ARCH=x64
+
+#----------------------------------------------------------------------------------------------
+FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis
+FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04 AS builder
+
+# ENV DEPS "git ca-certificates wget unzip cmake libgomp1 patchelf coreutils tcl libjemalloc-dev"
 ENV NVIDIA_VISIBLE_DEVICES all
 ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
 
-RUN apt update && apt -y upgrade
-RUN apt-get install -y libgomp1
-RUN apt-get install -y wget
-
-RUN apt install -y build-essential tcl libjemalloc-dev
-RUN wget http://download.redis.io/redis-stable.tar.gz
-RUN tar xzvf redis-stable.tar.gz && cd redis-stable && make && make install && cd .. && rm redis-stable.tar.gz
-
-# install latest cmake
-ADD https://cmake.org/files/v3.12/cmake-3.12.4-Linux-x86_64.sh /cmake-3.12.4-Linux-x86_64.sh
-RUN mkdir /opt/cmake
-RUN sh /cmake-3.12.4-Linux-x86_64.sh --prefix=/opt/cmake --skip-license
-RUN ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake
-RUN cmake --version
-
-# Set up a build environment
-RUN set -ex;\
-    deps="$DEPS";\
-    apt-get update;\
-    apt-get install -y --no-install-recommends $deps
-
-# Get the dependencies
-WORKDIR /redisai
-ADD ./ /redisai
-RUN set -ex;\
-    mkdir -p deps;\
-    DEPS_DIRECTORY=deps bash ./get_deps.sh gpu
-
-# Configurations
-ADD http://download.redis.io/redis-stable/redis.conf /usr/local/etc/redis/redis.conf
-RUN sed -i 's/bind 127.0.0.1/bind 0.0.0.0/g' /usr/local/etc/redis/redis.conf
-
-# Build the source
-RUN set -ex;\
-    rm -rf build;\
-    mkdir -p build;\
-    cd build;\
-    cmake -DDEVICE=gpu ..;\
-    make && make install;\
-    cd ..
-
-# Package the runner
-FROM builder
-
-RUN mkdir -p /usr/lib/redis/modules/
-
-COPY --from=builder /redisai/install-gpu/ /usr/lib/redis/modules/
+WORKDIR /build
+COPY --from=redis /usr/local/ /usr/local/
+
+COPY ./opt/ opt/
+COPY ./test/test_requirements.txt test/
+
+RUN ./opt/readies/bin/getpy
+RUN ./opt/system-setup.py
+
+ARG DEPS_ARGS=""
+COPY ./get_deps.sh .
+RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh gpu; else env $DEPS_ARGS ./get_deps.sh gpu; fi
+
+ARG BUILD_ARGS=""
+ADD ./ /build
+RUN make -C opt build GPU=1 $BUILD_ARGS SHOW=1
+
+ARG PACK=0
+ARG TEST=0
+
+RUN if [ "$PACK" = "1" ]; then make -C opt pack GPU=1; fi
+RUN if [ "$TEST" = "1" ]; then TEST="" make -C opt test GPU=1 $BUILD_ARGS NO_LFS=1; fi
+
+#----------------------------------------------------------------------------------------------
+FROM nvidia/cuda:10.0-cudnn7-runtime-ubuntu18.04
+
+RUN set -e; apt-get -qq update; apt-get -q install -y libgomp1
+
+ENV REDIS_MODULES /usr/lib/redis/modules
+RUN mkdir -p $REDIS_MODULES/
+
+COPY --from=redis /usr/local/ /usr/local/
+COPY --from=builder /build/install-gpu/ $REDIS_MODULES/
+# COPY --from=builder /usr/local/etc/redis/redis.conf /usr/local/etc/redis/
 
 WORKDIR /data
 EXPOSE 6379
-CMD ["redis-server", "/usr/local/etc/redis/redis.conf", "--loadmodule", "/usr/lib/redis/modules/redisai.so"]
+# CMD ["/usr/local/bin/redis-server", "/usr/local/etc/redis/redis.conf", "--loadmodule", "/usr/lib/redis/modules/redisai.so"]
+CMD ["/usr/local/bin/redis-server", "--loadmodule", "/usr/lib/redis/modules/redisai.so"]

Dockerfile.gpu-test

@@ -0,0 +1,41 @@
+# BUILD redisfab/redisai-gpu:${VERSION}-${ARCH}-${OSNICK}-test
+
+ARG REDIS_VER=5.0.7
+
+# OSNICK=bionic|centos7|centos6
+ARG OSNICK=bionic
+
+# ARCH=x64|arm64v8|arm32v7
+ARG ARCH=x64
+
+#----------------------------------------------------------------------------------------------
+FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK} AS redis
+FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04 AS builder
+
+ENV NVIDIA_VISIBLE_DEVICES all
+ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
+
+WORKDIR /build
+COPY --from=redis /usr/local/ /usr/local/
+
+COPY ./opt/ opt/
+COPY ./test/test_requirements.txt test/
+
+RUN ./opt/readies/bin/getpy
+RUN ./opt/system-setup.py
+
+ARG DEPS_ARGS=""
+COPY ./get_deps.sh .
+RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh gpu; else env $DEPS_ARGS ./get_deps.sh gpu; fi
+
+ARG BUILD_ARGS=""
+ADD ./ /build
+RUN make -C opt build GPU=1 $BUILD_ARGS SHOW=1
+
+ARG PACK=1
+
+RUN if [ "$PACK" = "1" ]; then make -C opt pack GPU=1; fi
+
+RUN git remote set-url origin https://github.com/RedisAI/RedisAI
+
+CMD ["bash", "-c", "make -C opt test GPU=1 SHOW=1"]

get_deps.sh

@@ -30,13 +30,13 @@ set -e
 [[ $VERBOSE == 1 ]] && set -x
 
 if [[ "$1" == "cpu" ]]; then
-	GPU=no
+	GPU=0
 	DEVICE=cpu
 elif [[ "$1" == "gpu" ]]; then
-	GPU=yes
+	GPU=1
 	DEVICE=gpu
 else
-	GPU=${GPU:-no}
+	GPU=${GPU:-0}
 	if [[ $GPU == 1 ]]; then
 		DEVICE=gpu
 	else
@@ -92,7 +92,7 @@ if [[ $WITH_TF != 0 ]]; then
 
 		if [[ $OS == linux ]]; then
 			TF_OS="linux"
-			if [[ $GPU == no ]]; then
+			if [[ $GPU != 1 ]]; then
 				TF_BUILD="cpu"
 			else
 				TF_BUILD="gpu"
@@ -148,7 +148,7 @@ if [[ $WITH_TFLITE != 0 ]]; then
 		LIBTF_URL_BASE=https://s3.amazonaws.com/redismodules/tensorflow
 		if [[ $OS == linux ]]; then
 			TFLITE_OS="linux"
-			# if [[ $GPU == no ]]; then
+			# if [[ $GPU != 1 ]]; then
 			# 	TFLITE_BUILD="cpu"
 			# else
 			# 	TFLITE_BUILD="gpu"
@@ -169,7 +169,7 @@ if [[ $WITH_TFLITE != 0 ]]; then
 
 		LIBTFLITE_ARCHIVE=libtensorflowlite-${TFLITE_OS}-${TFLITE_ARCH}-${TFLITE_VERSION}.tar.gz
 
-		[[ ! -f $LIBTFLITE_ARCHIVE || $FORCE == 1 ]] && wget --quiet $LIBTF_URL_BASE/$LIBTFLITE_ARCHIVE
+		[[ ! -f $LIBTFLITE_ARCHIVE || $FORCE == 1 ]] && wget $LIBTF_URL_BASE/$LIBTFLITE_ARCHIVE
 
 		rm -rf $LIBTFLITE.x
 		mkdir $LIBTFLITE.x
@@ -195,13 +195,13 @@ if [[ $WITH_PT != 0 ]]; then
 		echo "Installing libtorch ..."
 
 		PT_REPACK=0
-		
+
 		if [[ $OS == linux ]]; then
 			PT_OS=linux
-			if [[ $GPU == no ]]; then
+			if [[ $GPU != 1 ]]; then
 				PT_BUILD=cpu
 			else
-				PT_BUILD=cu100
+				PT_BUILD=cu101
 			fi
 			if [[ $ARCH == x64 ]]; then
 				PT_ARCH=x86_64
@@ -223,7 +223,7 @@ if [[ $WITH_PT != 0 ]]; then
 		LIBTORCH_ARCHIVE=libtorch-${PT_BUILD}-${PT_OS}-${PT_ARCH}-${PT_VERSION}.tar.gz
 
 		if [[ $PT_REPACK == 1 ]]; then
-			PT_VERSION=$PT_VERSION $HERE/opt/build/libtorch/repack.sh
+			PT_VERSION=$PT_VERSION GPU=$GPU $HERE/opt/build/libtorch/repack.sh
 		else
 			LIBTORCH_URL=https://s3.amazonaws.com/redismodules/pytorch/$LIBTORCH_ARCHIVE
 
@@ -280,7 +280,7 @@ if [[ $WITH_ORT != 0 ]]; then
 
 		if [[ $OS == linux ]]; then
 			ORT_OS=linux
-			if [[ $GPU == no ]]; then
+			if [[ $GPU != 1 ]]; then
 				ORT_BUILD=""
 			else
 				ORT_BUILD="-gpu"

opt/Makefile

@@ -22,6 +22,11 @@ fetch and build options:
   WITH_TFLITE=0 # SKip TensofFlowLite
   WITH_PT=0     # Skip PyTorch
   WITH_ORT=0    # SKip ONNXRuntime
+
+device selection options (fetch, build, and test):
+  CPU=1         # build for CPU
+  GPU=1         # build for GPU
+  CUDA=1        # build for GPU
 endef
 
 #---------------------------------------------------------------------------------------------- 
@@ -127,17 +132,22 @@ endif
 
 TEST_REPORT_DIR ?= $(PWD)
 
+ifeq ($(TEST),)
+TEST=basic_tests.py
+endif
+
 test:
 ifneq ($(NO_LFS),1)
 	$(SHOW)if [ "$(git lfs env > /dev/null 2>&1 ; echo $?)" != "0" ]; then cd $(ROOT); git lfs install; fi
 	$(SHOW)cd $(ROOT); git lfs pull
 endif
 	$(SHOW)set -e ;\
 	cd $(ROOT)/test ;\
-	python3 -m RLTest $(TEST_ARGS) --test basic_tests.py --module $(INSTALL_DIR)/redisai.so ;\
-	python3 -m RLTest $(TEST_ARGS) --test basic_tests.py --module $(INSTALL_DIR)/redisai.so --use-slaves ;\
-	python3 -m RLTest $(TEST_ARGS) --test basic_tests.py --module $(INSTALL_DIR)/redisai.so --use-aof 
-        # Commented until the MacOS CI is available
+	export DEVICE=$(DEVICE) ;\
+	python3 -m RLTest $(TEST_ARGS) --test $(TEST) --module $(INSTALL_DIR)/redisai.so ;\
+	python3 -m RLTest $(TEST_ARGS) --test $(TEST) --module $(INSTALL_DIR)/redisai.so --use-slaves ;\
+	python3 -m RLTest $(TEST_ARGS) --test $(TEST) --module $(INSTALL_DIR)/redisai.so --use-aof 
+	# Commented until the MacOS CI is available
 	# python3 -m RLTest $(TEST_ARGS) --test double-panda.py --module $(INSTALL_DIR)/redisai.so
 
 #----------------------------------------------------------------------------------------------

opt/build/libtorch/Makefile

@@ -1,7 +1,7 @@
 
 ROOT=../../..
 
-VERSION ?= 1.2.0
+VERSION ?= 1.3.1
 OSNICK ?= buster
 
 #----------------------------------------------------------------------------------------------
@@ -100,7 +100,7 @@ $(eval $(call publish_arm,arm64v8,arm64))
 $(eval $(call publish_arm,arm32v7,arm))
 
 repack:
-	@PT_VERSION=$(VERSION) ./repack.sh
+	@PT_VERSION=$(VERSION) GPU=$(GPU) ./repack.sh
 
 help:
 	@echo "make [build|repack|publish] [X64=1|ARM7=1|ARM8=1]"