Makefile: added pack and test targets (#169)

* Makefile: added pack and test targets

* Added deps/readies and system-setup.py

* Fixed docker functionality, fixed tests, setup changes

- Added deps/readies and system-setup.py to provide standard system setup,
- Docker: load all backends,
- Tests: load backends upfront to avoid rdb problems,
- Removed Python virtual environment setup in Makefile to avoid setup
  nightmares in various platforms,
- Docker: build using debian:buster, then copy binaries into redis:latest.

* CircleCI: refactoring

* CircleCI: refactoring #2

* CircleCI: github ssh keys for git lfs

* CircleCI: revert to old image

* CircleCI: disabled cache restore

* CircleCI: added sudo

* CircleCI: fixed git lfs handling

* CircleCI: try redis:5.0.5-buster image once again

* CircleCI: so maybe python:3.7.4-buster

* Moved Paella to Python 3, bumped version to 0.3.2

* Added getpy to readies/paella

* Build fix #1

* Build fix #2

* Dockerfile sh syntax, DEBUG flag in Makefile

* system-setup: added popen.communicate()

* Build: better cpu/gpu separation

* Docker fixes

* Build fix #3

* Build fix #4

* Fixes from review, updated README

* Fixes from review #2

- Restored README.md
- Moved Makefile and system-setup.py to automation/
- Removed setup and show-setup from Makefile
- Removed LD_LIBRARY_PATH from Makefile
- Moved deps/readies into automation/
- Will provice automation/README.md in a subsequent PR
- Added default value to DEVICE in CMakeLists.txt

* Fixes from review #3

* Fixes from review #4

* Fixes from review #5

* Fixes from review #6

* Fixes from review #7

* CircleCI cache hacking

* CircleCI cache: cancelled fallback

Author: rafie

.circleci/config.yml

@@ -1,8 +1,8 @@
-version: 2
+version: 2.1
 jobs:
   build:
     docker:
-      - image: circleci/python:3.6.8
+      - image: circleci/python:3.7.4-buster
 
     steps:
       - checkout
@@ -12,58 +12,38 @@ jobs:
           keys:
           - v1-dependencies-{{ checksum "get_deps.sh" }}
           # fallback to using the latest cache if no exact match is found
-          - v1-dependencies-
+          # - v1-dependencies-
 
       - run:
           name: install dependencies
           command: |
-            curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
-            sudo apt-get install -y git-lfs patchelf
-            git lfs install
-            curl -o cmake-3.12.4-Linux-x86_64.sh https://cmake.org/files/v3.12/cmake-3.12.4-Linux-x86_64.sh
-            mkdir /home/circleci/project/cmake-bin
-            sh cmake-3.12.4-Linux-x86_64.sh --prefix=/home/circleci/project/cmake-bin --skip-license
-            sudo ln -s /home/circleci/project/cmake-bin/bin/cmake /usr/local/bin/cmake
-            echo `cmake --version`
-            rm -rf /home/circleci/project/deps
-            bash get_deps.sh cpu
+            sudo ./automation/readies/bin/getpy
+            sudo ./automation/system-setup.py
+            ./get_deps.sh cpu
             git clone git://github.com/antirez/redis.git --branch 5.0.3
             (cd redis && make malloc=libc -j4 && sudo make install)
 
       - save_cache:
           paths:
             - deps
-          key: v1-dependencies-{{ checksum "get_deps.sh" }}
+          key: v2-dependencies-{{ checksum "get_deps.sh" }}
 
       - run:
           name: Build
           command: |
-            rm -rf build
-            mkdir build
-            cd build
-            cmake -DDEPS_PATH=../deps/install ..
-            make && make install
-            cd ..
-            make rlec_runpath_fix
+            make -C automation all
+            make -C automation pack
 
       - run:
           name: Test
-          command: |
-            python3 -m venv venv
-            . venv/bin/activate
-            pip -q uninstall -y RLTest || true
-            pip -q install git+https://github.com/RedisLabsModules/RLTest@py3
-            pip -q install -r test/test_requirements.txt
-            git lfs pull
-            python3 -m RLTest --test test/basic_tests.py --module install/redisai.so
+          command: make -C automation test
 
       - run:
           name: Persist Artifacts
           command: |
             mkdir -p ~/workspace/build
-            cp install/ramp/$MODULE_ARTIFACT ~/workspace/
-            # cp deps/install/lib/*.so*  ~/workspace/
-            cp -r install/backends ~/workspace/
+            cp install-cpu/$MODULE_ARTIFACT ~/workspace/
+            cp -r install-cpu/backends ~/workspace/
             cp ramp.yml ~/workspace/
       - persist_to_workspace:
           root: ~/workspace

.dockerignore

@@ -1,4 +1,7 @@
-deps/*
-install/*
+/deps/
+/build/
+/install*/
 **/*.o
-**/*.so
+**/*.so
+.venv/
+venv*/

.gitignore

@@ -7,11 +7,16 @@ __pycache__
 *.pyc
 
 # Dirs
-install/
-deps/
-build/
+/install*/
+/deps/*
+/build/
 examples/js/node_modules/
 examples/js/package-lock.json
+/test/venv/
+/test/logs/
+/1/
+/venv/
+/.venv/
 
 # Eclipse (if that's your jam...)
 .project

CMakeLists.txt

@@ -7,6 +7,10 @@ IF (NOT DEPS_PATH)
     # MESSAGE(FATAL_ERROR "DEPS PATH Missing!")
 ENDIF()
 
+IF (NOT DEVICE)
+    SET(DEVICE cpu)
+ENDIF()
+
 FUNCTION(ADD_LDFLAGS _TARGET NEW_FLAGS)
     GET_TARGET_PROPERTY(LD_FLAGS ${_TARGET} LINK_FLAGS)
     IF(LD_FLAGS)
@@ -19,6 +23,8 @@ GET_FILENAME_COMPONENT(depsAbs
     "${DEPS_PATH}" REALPATH
     BASE_DIR "${CMAKE_BINARY_DIR}")
 
+# SET(CUDA_TOOLKIT_ROOT_DIR /usr/local/cuda-9.0)
+
 INCLUDE_DIRECTORIES(${depsAbs}/dlpack/include)
 INCLUDE_DIRECTORIES(${depsAbs}/libtensorflow/include)
 INCLUDE_DIRECTORIES(${depsAbs}/libtorch/include)
@@ -60,7 +66,7 @@ IF (APPLE)
         LINK_FLAGS "-undefined dynamic_lookup")
 ENDIF()
 
-SET(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/install)
+SET(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/install-${DEVICE})
 
 INSTALL(TARGETS redisai LIBRARY DESTINATION .)
 

Dockerfile

@@ -1,45 +1,38 @@
-FROM redis AS builder
+ARG OS=debian:buster
 
-ENV DEPS "build-essential git ca-certificates curl unzip wget libgomp1 patchelf"
+#----------------------------------------------------------------------------------------------
+FROM redis AS redis
+FROM ${OS} AS builder
 
-# install latest cmake
-ADD https://cmake.org/files/v3.12/cmake-3.12.4-Linux-x86_64.sh /cmake-3.12.4-Linux-x86_64.sh
-RUN mkdir /opt/cmake
-RUN sh /cmake-3.12.4-Linux-x86_64.sh --prefix=/opt/cmake --skip-license
-RUN ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake
-RUN cmake --version
+WORKDIR /redisai
+COPY --from=redis /usr/local/ /usr/local/
 
-# Set up a build environment
-RUN set -ex;\
-    deps="$DEPS";\
-    apt-get update;\
-    apt-get install -y --no-install-recommends $deps
+COPY ./automation/ automation/
+COPY ./test/test_requirements.txt test/
+
+RUN ./automation/readies/bin/getpy
+RUN ./automation/system-setup.py
+
+COPY ./get_deps.sh .
+RUN ./get_deps.sh cpu
 
-# Get the dependencies
-WORKDIR /redisai
 ADD ./ /redisai
-RUN set -ex;\
-    mkdir -p deps;\
-    DEPS_DIRECTORY=deps bash ./get_deps.sh cpu
-
-# Build the source
-RUN set -ex;\
-    rm -rf build;\
-    mkdir -p build;\
-    cd build;\
-    cmake -DDEPS_PATH=../deps/install ..;\
-    make && make install;\
-    cd ..
-
-# Package the runner
+RUN make -C automation all
+
+ARG PACK=0
+ARG TEST=0
+
+RUN if [ "$PACK" = "1" ]; then make -C automation pack; fi
+RUN if [ "$TEST" = "1" ]; then make -C automation test; fi
+
+#----------------------------------------------------------------------------------------------
 FROM redis
 
 RUN set -e; apt-get -qq update; apt-get install -y libgomp1
 
-RUN set -ex;\
-    mkdir -p /usr/lib/redis/modules/;
+RUN mkdir -p /usr/lib/redis/modules/
 
-COPY --from=builder /redisai/install/ /usr/lib/redis/modules/
+COPY --from=builder /redisai/install-cpu/ /usr/lib/redis/modules/
 
 WORKDIR /data
 EXPOSE 6379

Makefile

@@ -0,0 +1,103 @@
+
+ROOT:=..
+
+override GPU:=$(or $(findstring $(CUDA),1),$(findstring $(GPU),1))
+
+ifeq ($(GPU),1)
+ifeq ($(CPU),1)
+$(error CPU=1 and GPU=1 (or CUDA=1) are conflicting)
+endif
+DEPS_FLAGS=gpu
+DEVICE=gpu
+else
+DEPS_FLAGS=cpu
+DEVICE=cpu
+endif
+
+GIT_BRANCH:=$(shell git rev-parse --abbrev-ref HEAD)
+GIT_COMMIT:=$(shell git describe --always --abbrev=7 --dirty="+")
+
+ifeq ($(VERSION),)
+PACK_VER:=$(GIT_BRANCH)-$(GIT_COMMIT)
+else
+PACK_VER:=$(VERSION)
+endif
+
+BINDIR=$(PWD)/build
+INSTALL_DIR=$(PWD)/install-$(DEVICE)
+
+BACKENDS_PATH ?= $(INSTALL_DIR)/backends
+
+RAMP:=ramp
+
+ifeq ($(DEBUG),1)
+CMAKE_FLAGS += -DCMAKE_BUILD_TYPE=Debug
+endif
+
+#----------------------------------------------------------------------------------------------
+
+.PHONY: all setup build clean deps pack pack_ramp pack_deps test
+
+all: build
+
+build:
+ifeq ($(wildcard build/Makefile),)
+	mkdir -p $(ROOT)/build
+	cd $(ROOT)/build; \
+	cmake -DDEVICE=$(DEVICE) -DDEPS_PATH=$(ROOT)/deps/install $(CMAKE_FLAGS) $(ROOT)
+endif
+	$(MAKE) -C $(ROOT)/build
+	$(MAKE) -C $(ROOT)/build install
+	cd $(ROOT) ;\
+	[ ! -e install ] && ln -sf install-$(DEVICE) install
+
+clean:
+ifeq ($(ALL),1)
+	cd $(ROOT) ;\
+	rm -rf build install deps/dlpack deps/install-$(DEVICE) deps/*.tar.gz deps/*.zip deps/*.tgz
+else
+	$(MAKE) -C $(ROOT)/build clean
+endif
+
+deps fetch:
+	@echo Fetching dependencies...
+	$(ROOT)/get_deps.sh $(DEPS_FLAGS)
+
+pack: pack_ramp pack_deps
+
+pack_ramp:
+	@[ ! -z `command -v redis-server` ] || { echo "Cannot find redis-server - aborting."; exit 1; }
+	@echo "Building RAMP file ..."
+	@set -e ;\
+	cd $(ROOT) ;\
+	RAMPOUT=$$(mktemp /tmp/ramp.XXXXXX) ;\
+	$(RAMP) pack -m $(PWD)/ramp.yml -o "build/redisai.{os}-{architecture}.{semantic_version}.zip" $(INSTALL_DIR)/redisai.so 2> /dev/null | grep '.zip' > $$RAMPOUT ;\
+	tail -1 $$RAMPOUT > $(BINDIR)/PACKAGE ;\
+	cat $(BINDIR)/PACKAGE | sed -e "s/[^.]*\.[^.]*\.\(.*\)\.zip/\1/" > $(BINDIR)/VERSION ;\
+	VERSION=$$(cat $(BINDIR)/VERSION) ;\
+	$(RAMP) pack -m $(PWD)/ramp.yml -o "build/redisai.{os}-{architecture}.{semantic_version}.zip" \
+		-c "BACKENDSPATH /opt/redislabs/lib/redisai-cpu-$$VERSION/backends" $(INSTALL_DIR)/redisai.so 2> /dev/null | grep '.zip' > $$RAMPOUT ;\
+	rm -f $$RAMPOUT ;\
+	echo "Done."
+
+pack_deps: pack_ramp
+	@echo "Building dependencies file ..."
+	@set -e ;\
+	cd $(ROOT) ;\
+	PACK_FNAME=$$(basename `cat $(BINDIR)/PACKAGE`) ;\
+	ARCHOSVER=$$(echo "$$PACK_FNAME" | sed -e "s/^redisai\.\([^.]*\..*\)\.zip/\1/") ;\
+	VERSION=$$(cat $(BINDIR)/VERSION) ;\
+	cd install-$(DEVICE) ;\
+	rm -rf redisai-$(DEVICE)-$$VERSION ;\
+	mkdir redisai-$(DEVICE)-$$VERSION ;\
+	mv backends redisai-$(DEVICE)-$$VERSION ;\
+	ln -sf redisai-$(DEVICE)-$$VERSION/backends backends ;\
+	find redisai-$(DEVICE)-$$VERSION -name "*.so*" | xargs tar pczf redisai-$(DEVICE)-dependencies.$$ARCHOSVER.tgz ;\
+	echo "Done."
+
+test:
+	@git lfs install
+	@git lfs pull
+	@set -e ;\
+	cd $(ROOT)/test ;\
+	python3 -m RLTest $(TEST_ARGS) --test basic_tests.py --module $(INSTALL_DIR)/redisai.so