From 6c2ff5f69c31c489ff40d3ad84473c00fd4331cd Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Tue, 30 Jul 2019 15:35:15 +0300
Subject: [PATCH 01/31] Makefile: added pack and test targets

---
 .circleci/config.yml |  8 +++---
 .gitignore           |  8 +++---
 Makefile             | 59 +++++++++++++++++++++++++++++---------------
 3 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index da95b42ed..9e5ecdb1d 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -42,9 +42,7 @@ jobs:
             mkdir build
             cd build
             cmake -DDEPS_PATH=../deps/install ..
-            make && make install
-            cd ..
-            make rlec_runpath_fix
+            make
 
       - run:
           name: Test
@@ -52,7 +50,7 @@ jobs:
             python3 -m venv venv
             . venv/bin/activate
             pip -q uninstall -y RLTest || true
-            pip -q install git+https://github.com/RedisLabsModules/RLTest@py3
+            pip -q install git+https://github.com/RedisLabsModules/RLTest@master
             pip -q install -r test/test_requirements.txt
             git lfs pull
             python3 -m RLTest --test test/basic_tests.py --module install/redisai.so
@@ -61,7 +59,7 @@ jobs:
           name: Persist Artifacts
           command: |
             mkdir -p ~/workspace/build
-            cp install/ramp/$MODULE_ARTIFACT ~/workspace/
+            cp install/$MODULE_ARTIFACT ~/workspace/
             # cp deps/install/lib/*.so*  ~/workspace/
             cp -r install/backends ~/workspace/
             cp ramp.yml ~/workspace/
diff --git a/.gitignore b/.gitignore
index 7d100019b..bf7c8c6df 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,11 +7,13 @@ __pycache__
 *.pyc
 
 # Dirs
-install/
-deps/
-build/
+/install/
+/deps/
+/build/
 examples/js/node_modules/
 examples/js/package-lock.json
+/test/venv/
+/test/logs/
 
 # Eclipse (if that's your jam...)
 .project
diff --git a/Makefile b/Makefile
index 642bc4cb5..f0399b8c2 100755
--- a/Makefile
+++ b/Makefile
@@ -18,15 +18,21 @@ endif
 
 BINDIR=$(PWD)/install
 
-.PHONY: all clean deps pack rlec_runpath_fix
+.PHONY: all build clean deps pack test
 
-all:
+all: build
+
+version:
+	@echo $(PACK_VER)
+
+build:
 ifeq ($(wildcard build/.),)
 	mkdir -p build
 	cd build; \
 	cmake -DDEPS_PATH=../deps/install ..
 endif
-	$(MAKE) -C build && $(MAKE) -C build install
+	$(MAKE) -C build
+	$(MAKE) -C build install
 
 clean:
 ifeq ($(ALL),1)
@@ -39,16 +45,8 @@ deps:
 	@echo Fetching dependencies...
 	@./get_deps.sh $(DEPS_FLAGS)
 
-# in pack: create ramp/redisai.so with RUNPATH set to /opt/redislabs/lib for RLEC compliance
-rlec_runpath_fix: 
-	@echo Fixing RLEC RUNPATH...
-	@mkdir -p $(BINDIR)/ramp
-	@cp -f $(BINDIR)/redisai.so $(BINDIR)/ramp/
-	# @patchelf --set-rpath $(REDIS_ENT_LIB_PATH) $(BINDIR)/ramp/redisai.so
-
-pack: rlec_runpath_fix
+pack:
 	@[ ! -z `command -v redis-server` ] || { echo "Cannot find redis-server - aborting."; exit 1; }
-	@[ ! -e $(REDIS_ENT_LIB_PATH) ] || { echo "$(REDIS_ENT_LIB_PATH) exists - aborting."; exit 1; }
 ifeq ($(wildcard build/pyenv/.),)
 	@virtualenv build/pyenv ;\
 	. ./build/pyenv/bin/activate ;\
@@ -57,11 +55,32 @@ endif
 	@echo "Building RAMP file ..."
 	@set -e ;\
 	. ./build/pyenv/bin/activate ;\
-	ln -fs $(PWD)/deps/install/lib/ $(REDIS_ENT_LIB_PATH) ;\
-	ramp pack -m $(PWD)/ramp.yml -o "build/redisai.{os}-{architecture}.${PACK_VER}.zip" $(BINDIR)/ramp/redisai.so 2>&1 > /dev/null ;\
-	rm $(REDIS_ENT_LIB_PATH)
-	@echo Done.
-	@echo "Building dependencies file redisai-dependencies.${PACK_VER}.tgz ..."
-	@cd deps/install/lib; \
-	tar pczf ../../../build/redisai-dependencies.${PACK_VER}.tgz *.so*
-	@echo Done.
+	RAMPOUT=$$(mktemp /tmp/ramp.XXXXXX) ;\
+	LD_LIBRARY_PATH=$(PWD)/deps/install/lib \
+	ramp pack -m $(PWD)/ramp.yml -o "build/redisai.{os}-{architecture}.{semantic_version}.zip" $(BINDIR)/redisai.so 2> /dev/null | grep '.zip' > $$RAMPOUT ;\
+	tail -1 $$RAMPOUT > $(BINDIR)/PACKAGE ;\
+	rm -f $RAMPOUT ;\
+	echo "Done."
+	@echo "Building dependencies file ..."
+	@set -e ;\
+	PACK_FNAME=$$(basename `cat $(BINDIR)/PACKAGE`) ;\
+	ARCHOSVER=$$(echo "$$PACK_FNAME" | sed -e "s/^redisai\.\([^.]*\..*\)\.zip/\1/") ;\
+	cd install ;\
+	find backends -name "*.so*" | xargs tar pczf redisai-dependencies.$$ARCHOSVER.tgz
+	@echo "Done."
+
+test:
+ifeq ($(wildcard venv/.),)
+	@python3 -m venv test/venv
+	@set -e ;\
+	cd test ;\
+	. venv/bin/activate ;\
+	pip -q install git+https://github.com/RedisLabsModules/RLTest@master ;\
+	pip -q install -r test_requirements.txt
+	@git lfs pull
+endif
+	@set -e ;\
+	cd test ;\
+	. venv/bin/activate ;\
+	LD_LIBRARY_PATH=$(PWD)/install \
+	python3 -m RLTest --test basic_tests.py --module $(BINDIR)/redisai.so

From 70736cc16aac86ec641284d30752ad88eb93c3dd Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Wed, 31 Jul 2019 14:27:54 +0300
Subject: [PATCH 02/31] Added deps/readies and system-setup.py

---
 .gitignore                      |   5 +-
 deps/readies/.gitignore         |  36 ++++++++
 deps/readies/LICENSE            |  29 ++++++
 deps/readies/README.md          |   9 ++
 deps/readies/bin/getpy2         |  26 ++++++
 deps/readies/bin/getredis5      |  56 +++++++++++
 deps/readies/bin/platform       |  40 ++++++++
 deps/readies/bin/python2        |   7 ++
 deps/readies/cetara/diag/gdb.c  | 130 ++++++++++++++++++++++++++
 deps/readies/cetara/diag/gdb.h  |  12 +++
 deps/readies/mk/bindirs.defs    |   8 ++
 deps/readies/mk/bindirs.rules   |  10 ++
 deps/readies/mk/common.defs     |  14 +++
 deps/readies/mk/common.rules    |   6 ++
 deps/readies/mk/configure.defs  |   2 +
 deps/readies/mk/configure.rules |  13 +++
 deps/readies/mk/defs            |   6 ++
 deps/readies/mk/help.defs       |   4 +
 deps/readies/mk/help.rules      |  11 +++
 deps/readies/mk/rules           |   4 +
 deps/readies/mk/variant.defs    |  42 +++++++++
 deps/readies/mk/variant.rules   |   3 +
 deps/readies/paella/__init__.py |  32 +++++++
 deps/readies/paella/debug.py    |  11 +++
 deps/readies/paella/docopt1.py  |  25 +++++
 deps/readies/paella/files.py    |  23 +++++
 deps/readies/paella/log.py      |   6 ++
 deps/readies/paella/platform.py | 158 ++++++++++++++++++++++++++++++++
 deps/readies/paella/setup.py    | 143 +++++++++++++++++++++++++++++
 deps/readies/paella/utils.py    |   7 ++
 deps/readies/paella/utils2.py   |   5 +
 deps/readies/paella/utils3.py   |   5 +
 deps/readies/shibumi/functions  |  21 +++++
 system-setup.py                 |  60 ++++++++++++
 34 files changed, 968 insertions(+), 1 deletion(-)
 create mode 100755 deps/readies/.gitignore
 create mode 100755 deps/readies/LICENSE
 create mode 100755 deps/readies/README.md
 create mode 100755 deps/readies/bin/getpy2
 create mode 100755 deps/readies/bin/getredis5
 create mode 100755 deps/readies/bin/platform
 create mode 100755 deps/readies/bin/python2
 create mode 100755 deps/readies/cetara/diag/gdb.c
 create mode 100755 deps/readies/cetara/diag/gdb.h
 create mode 100755 deps/readies/mk/bindirs.defs
 create mode 100755 deps/readies/mk/bindirs.rules
 create mode 100755 deps/readies/mk/common.defs
 create mode 100755 deps/readies/mk/common.rules
 create mode 100755 deps/readies/mk/configure.defs
 create mode 100755 deps/readies/mk/configure.rules
 create mode 100755 deps/readies/mk/defs
 create mode 100755 deps/readies/mk/help.defs
 create mode 100755 deps/readies/mk/help.rules
 create mode 100755 deps/readies/mk/rules
 create mode 100755 deps/readies/mk/variant.defs
 create mode 100755 deps/readies/mk/variant.rules
 create mode 100755 deps/readies/paella/__init__.py
 create mode 100755 deps/readies/paella/debug.py
 create mode 100755 deps/readies/paella/docopt1.py
 create mode 100755 deps/readies/paella/files.py
 create mode 100755 deps/readies/paella/log.py
 create mode 100755 deps/readies/paella/platform.py
 create mode 100755 deps/readies/paella/setup.py
 create mode 100755 deps/readies/paella/utils.py
 create mode 100755 deps/readies/paella/utils2.py
 create mode 100755 deps/readies/paella/utils3.py
 create mode 100755 deps/readies/shibumi/functions
 create mode 100755 system-setup.py

diff --git a/.gitignore b/.gitignore
index bf7c8c6df..6730af9f2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,12 +8,15 @@ __pycache__
 
 # Dirs
 /install/
-/deps/
+/deps/*
+!/deps/readies/
+!/deps/*.py
 /build/
 examples/js/node_modules/
 examples/js/package-lock.json
 /test/venv/
 /test/logs/
+/1/
 
 # Eclipse (if that's your jam...)
 .project
diff --git a/deps/readies/.gitignore b/deps/readies/.gitignore
new file mode 100755
index 000000000..5ff250e4c
--- /dev/null
+++ b/deps/readies/.gitignore
@@ -0,0 +1,36 @@
+# Object files
+*.o
+*.ko
+*.obj
+*.elf
+*.lib
+*.a
+*.la
+*.lo
+*.dll
+*.so
+*.so.*
+*.dylib
+*.exe
+*.out
+*.app
+*.i*86
+*.x86_64
+*.hex
+
+# C/C++
+*.gch
+*.pch
+*.d
+
+# Python
+*.pyc
+
+# CMake
+CMakeFiles/
+*.cmake
+CMakeCache.txt
+
+# Eclipse
+.cproject
+.project
diff --git a/deps/readies/LICENSE b/deps/readies/LICENSE
new file mode 100755
index 000000000..bf39fd1a1
--- /dev/null
+++ b/deps/readies/LICENSE
@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2019, Redis Labs Modules
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/deps/readies/README.md b/deps/readies/README.md
new file mode 100755
index 000000000..a490b311a
--- /dev/null
+++ b/deps/readies/README.md
@@ -0,0 +1,9 @@
+# Library cluster of common Redis Modules code
+
+Cetara: C/C++
+
+Paella: Python
+
+Shibumi: Bash
+
+mk: GNU Make
diff --git a/deps/readies/bin/getpy2 b/deps/readies/bin/getpy2
new file mode 100755
index 000000000..ea12aff36
--- /dev/null
+++ b/deps/readies/bin/getpy2
@@ -0,0 +1,26 @@
+#!/bin/sh
+
+if [ ! -z $(command -v python) ]; then
+	[ "$(python --version 2>&1 | cut -d" " -f2 | cut -d. -f1)" = "2" ] && exit 0
+fi
+
+[ ! -z $(command -v python2) ] && exit 0
+
+if [ ! -z $(command -v apt-get) ]; then
+	apt-get -qq update
+	apt-get -qq install -y python
+elif [ ! -z $(command -v dnf) ]; then
+	dnf install -y python2
+elif [ ! -z $(command -v yum) ]; then
+	yum install -y python2
+elif [ ! -z $(command -v apk) ]; then
+	apk update
+	apk add python2
+elif [ ! -z $(command -v brew) ]; then
+	brew install python2
+fi
+
+if [ -z $(command -v python) ]; then
+	>&2 echo "Cannot install Python2. Aborting."
+	exit 1
+fi
diff --git a/deps/readies/bin/getredis5 b/deps/readies/bin/getredis5
new file mode 100755
index 000000000..883c8058e
--- /dev/null
+++ b/deps/readies/bin/getredis5
@@ -0,0 +1,56 @@
+#!/usr/bin/env python2
+
+import sys
+import os
+import popen2
+import argparse
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "deps/readies"))
+import paella
+
+os.environ["PYTHONWARNINGS"] = 'ignore:DEPRECATION::pip._internal.cli.base_command'
+
+#----------------------------------------------------------------------------------------------
+
+class Redis5Setup(paella.Setup):
+    def __init__(self, nop=False):
+        paella.Setup.__init__(self, nop)
+
+    def common_first(self):
+        pass
+
+    def debian_compat(self):
+        # https://chilts.org/installing-redis-from-chris-leas-ppa/
+        self.run("add-apt-repository -y ppa:chris-lea/redis-server")
+        self.install("redis-server")
+
+    def redhat_compat(self):
+        # https://linuxize.com/post/how-to-install-and-configure-redis-on-centos-7/
+        self.install("epel-release yum-utils")
+
+        self.install("http://rpms.remirepo.net/enterprise/remi-release-7.rpm")
+        self.run("yum-config-manager -y --enable remi")
+        self.install("redis")
+
+    def fedora(self):
+        self.install("dnf-plugins-core")
+        
+        self.install("https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm")
+        self.install("--allowerasing https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm")
+        self.install("http://rpms.remirepo.net/enterprise/remi-release-7.rpm")
+        self.run("dnf config-manager -y --set-enabled remi")
+        self.install("redis")
+
+    def macosx(self):
+        self.install("redis")
+
+    def common_last(self):
+        pass
+
+#----------------------------------------------------------------------------------------------
+
+parser = argparse.ArgumentParser(description='Set up system for build.')
+parser.add_argument('-n', '--nop', action="store_true", help='no operation')
+args = parser.parse_args()
+
+RedisTimeSeriesSetup(nop = args.nop).setup()
diff --git a/deps/readies/bin/platform b/deps/readies/bin/platform
new file mode 100755
index 000000000..b07a605b0
--- /dev/null
+++ b/deps/readies/bin/platform
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+
+import sys
+import os
+import argparse
+
+READIES_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), ".."))
+sys.path.insert(0, READIES_PATH)
+from paella import Platform
+
+parser = argparse.ArgumentParser(description='Report platform characteristics.')
+parser.add_argument('--os', action="store_true", help='Operating system')
+parser.add_argument('--version', action="store_true", help='OS/Distribution version')
+parser.add_argument('--dist', action="store_true", help='Linux distribution (if applicable)')
+parser.add_argument('--arch', action="store_true", help='CPU Architecture')
+parser.add_argument('--kernel', action="store_true", help='Kernel version (if applicable)')
+parser.add_argument('--glibc', action="store_true", help='GLIBC version (if applicable)')
+args = parser.parse_args()
+
+platform = Platform()
+ret = ""
+if args.os:
+	ret += " " + platform.os
+if args.dist:
+	ret += " " + platform.dist
+if args.version:
+	ret += " " + platform.os_ver
+if args.arch:
+	ret += " " + platform.arch
+if args.kernel:
+	pass
+if args.glibc:
+	pass
+if ret == "":
+    os = platform.os
+    dist = platform.dist
+    if dist != "":
+        os = dist + " " + os
+    ret = os + " " + platform.os_ver + " " + platform.arch
+print(ret.strip())
diff --git a/deps/readies/bin/python2 b/deps/readies/bin/python2
new file mode 100755
index 000000000..f46ed5a8e
--- /dev/null
+++ b/deps/readies/bin/python2
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+if [ ! -z $(command -v python2) ]; then
+	python2 "$@"
+else
+	python "$@"
+fi
diff --git a/deps/readies/cetara/diag/gdb.c b/deps/readies/cetara/diag/gdb.c
new file mode 100755
index 000000000..8afa1697d
--- /dev/null
+++ b/deps/readies/cetara/diag/gdb.c
@@ -0,0 +1,130 @@
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "readies/cetara/diag/gdb.h"
+
+#include <string.h>
+#include <errno.h>
+#include <unistd.h>
+#include <stdint.h>
+#include <sys/ptrace.h>
+#include <sys/wait.h>
+
+///////////////////////////////////////////////////////////////////////////////////////////////
+
+#ifdef __linux__
+
+#if 1
+
+static inline bool _via_gdb()
+{
+    int pid;
+    int from_child[2] = {-1, -1};
+
+    if (pipe(from_child) < 0) {
+        fprintf(stderr, "Debugger check failed: Error opening internal pipe: %s", strerror(errno));
+        return false;
+    }
+
+    pid = fork();
+    if (pid == -1) {
+        fprintf(stderr, "Debugger check failed: Error forking: %s", strerror(errno));
+        return false;
+    }
+
+    if (pid == 0) // child
+	{
+        uint8_t ret = 0;
+        int ppid = getppid();
+
+        close(from_child[0]); // close parent's side
+
+        if (ptrace(PTRACE_ATTACH, ppid, NULL, NULL) == 0) 
+		{
+            waitpid(ppid, NULL, 0); // wait for the parent to stop
+            write(from_child[1], &ret, sizeof(ret)); // tell the parent what happened
+
+			ptrace(PTRACE_DETACH, ppid, NULL, NULL);
+            exit(0);
+        }
+
+        ret = 1;
+        write(from_child[1], &ret, sizeof(ret)); // tell the parent what happened
+
+        exit(0);
+   
+    }
+	else // parent
+	{ 
+        uint8_t ret = -1;
+
+        // child writes a 1 if pattach failed else 0.
+        // read may be interrupted by pattach, hence the loop.
+        while (read(from_child[0], &ret, sizeof(ret)) < 0 && errno == EINTR)
+			;
+
+        // ret not updated
+        if (ret < 0)
+            fprintf(stderr, "Debugger check failed: Error getting status from child: %s", strerror(errno));
+
+        // close the pipes here, to avoid races with pattach (if we did it above)
+        close(from_child[1]);
+        close(from_child[0]);
+
+        waitpid(pid, NULL, 0); // collect the status of the child
+
+        return ret == 1;
+    }
+}
+
+#else
+
+static inline bool _via_gdb()
+{
+    const int status_fd = open("/proc/self/status", O_RDONLY);
+    if (status_fd == -1)
+        return false;
+
+    char buf[4096];
+    const ssize_t num_read = read(status_fd, buf, sizeof(buf) - 1);
+    if (num_read <= 0)
+        return false;
+
+    buf[num_read] = '\0';
+    constexpr char tracer_pid[] = "TracerPid:";
+    const auto tracer_pid_p = strstr(buf, tracer_pid);
+    if (!tracer_pid_p)
+        return false;
+
+    for (const char *p = tracer_pid_p + sizeof(tracer_pid) - 1; p <= buf + num_read; ++p)
+    {
+        if (isspace(*p))
+            continue;
+        return isdigit(*p) && *p != '0';
+    }
+
+    return false;
+}
+
+#endif // 1
+
+#elif defined(__APPLE__)
+
+static inline bool _via_gdb()
+{
+}
+
+#endif
+
+//---------------------------------------------------------------------------------------------
+
+bool __via_gdb = false;
+
+__attribute__((constructor))
+static void initialize(void) 
+{
+    __via_gdb = _via_gdb();
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/deps/readies/cetara/diag/gdb.h b/deps/readies/cetara/diag/gdb.h
new file mode 100755
index 000000000..458a7e15b
--- /dev/null
+++ b/deps/readies/cetara/diag/gdb.h
@@ -0,0 +1,12 @@
+
+#pragma once
+
+#include <stdbool.h>
+
+extern bool __via_gdb;
+
+#ifdef __arm__
+#define BB do { if (__via_gdb) { __asm__("trap"); } } while(0)
+#else
+#define BB do { if (__via_gdb) { __asm__("int $3"); } } while(0)
+#endif
\ No newline at end of file
diff --git a/deps/readies/mk/bindirs.defs b/deps/readies/mk/bindirs.defs
new file mode 100755
index 000000000..1ff57c788
--- /dev/null
+++ b/deps/readies/mk/bindirs.defs
@@ -0,0 +1,8 @@
+
+BINROOT=$(ROOT)/bin/$(FULL_VARIANT)
+BIN_DIRS=$(sort $(patsubst %/,%,$(BINDIR) $(dir $(OBJECTS))))
+
+define mkdir_rule
+$(1):
+	$$(SHOW)mkdir -p $(1)
+endef
diff --git a/deps/readies/mk/bindirs.rules b/deps/readies/mk/bindirs.rules
new file mode 100755
index 000000000..4b57e8d49
--- /dev/null
+++ b/deps/readies/mk/bindirs.rules
@@ -0,0 +1,10 @@
+
+ifeq ($(BINDIR),)
+$(error BINDIR is undefined)
+endif
+
+.PHONY: bindirs
+
+bindirs: $(BIN_DIRS)
+
+$(foreach DIR,$(BIN_DIRS),$(eval $(call mkdir_rule,$(DIR))))
diff --git a/deps/readies/mk/common.defs b/deps/readies/mk/common.defs
new file mode 100755
index 000000000..206cd969d
--- /dev/null
+++ b/deps/readies/mk/common.defs
@@ -0,0 +1,14 @@
+
+_SHOW:=$(SHOW)
+ifeq ($(SHOW),1)
+override SHOW:=
+else
+override SHOW:=@
+endif
+
+MAKEFLAGS += --no-builtin-rules  --no-print-directory
+# --no-builtin-variables
+
+define __SEP
+import os; rows, cols = os.popen('stty size', 'r').read().split(); print(\"\n\" + '-' * (int(cols) - 1) + \"\n\")
+endef
diff --git a/deps/readies/mk/common.rules b/deps/readies/mk/common.rules
new file mode 100755
index 000000000..633cf2f6b
--- /dev/null
+++ b/deps/readies/mk/common.rules
@@ -0,0 +1,6 @@
+
+.PHONY: __sep
+
+__sep: ;
+#	@python -c "$(__SEP)"
+
diff --git a/deps/readies/mk/configure.defs b/deps/readies/mk/configure.defs
new file mode 100755
index 000000000..da576887b
--- /dev/null
+++ b/deps/readies/mk/configure.defs
@@ -0,0 +1,2 @@
+
+CONFIGURE_FLAGS=
diff --git a/deps/readies/mk/configure.rules b/deps/readies/mk/configure.rules
new file mode 100755
index 000000000..baf4ce4d4
--- /dev/null
+++ b/deps/readies/mk/configure.rules
@@ -0,0 +1,13 @@
+
+build: __sep bindirs
+ifeq (,$(wildcard $(BUILD_DIR)/Makefile))
+	$(SHOW)cd $(BUILD_DIR); $(realpath $(SRCDIR))/configure $(CONFIGURE_FLAGS)
+endif
+	@make -C $(BUILD_DIR)
+
+clean:
+ifeq ($(ALL),1) 
+	$(SHOW)rm -rf $(BINDIR)
+else
+	$(SHOW)$(MAKE) clean -C $(BUILD_DIR)
+endif
diff --git a/deps/readies/mk/defs b/deps/readies/mk/defs
new file mode 100755
index 000000000..27059268b
--- /dev/null
+++ b/deps/readies/mk/defs
@@ -0,0 +1,6 @@
+
+MK=$(ROOT)/deps/readies/mk
+
+include $(MK)/common.defs
+include $(MK)/variant.defs
+include $(MK)/bindirs.defs
diff --git a/deps/readies/mk/help.defs b/deps/readies/mk/help.defs
new file mode 100755
index 000000000..0641d1684
--- /dev/null
+++ b/deps/readies/mk/help.defs
@@ -0,0 +1,4 @@
+
+ifneq ($(HELP),) 
+HELPFILE:=$(shell mktemp /tmp/make.help.XXXX)
+endif
diff --git a/deps/readies/mk/help.rules b/deps/readies/mk/help.rules
new file mode 100755
index 000000000..eae2fed21
--- /dev/null
+++ b/deps/readies/mk/help.rules
@@ -0,0 +1,11 @@
+
+ifneq ($(HELP),) 
+
+.PHONY: help
+
+help:
+	$(file >$(HELPFILE),$(HELP))
+	@cat $(HELPFILE)
+	@-rm -f $(HELPFILE)
+
+endif
diff --git a/deps/readies/mk/rules b/deps/readies/mk/rules
new file mode 100755
index 000000000..1aa2985e3
--- /dev/null
+++ b/deps/readies/mk/rules
@@ -0,0 +1,4 @@
+
+include $(MK)/common.rules
+include $(MK)/variant.rules
+include $(MK)/bindirs.rules
diff --git a/deps/readies/mk/variant.defs b/deps/readies/mk/variant.defs
new file mode 100755
index 000000000..a1c242702
--- /dev/null
+++ b/deps/readies/mk/variant.defs
@@ -0,0 +1,42 @@
+
+OS:=$(shell $(ROOT)/deps/readies/bin/platform --os)
+# ifeq ($(OS),linux)
+# OS:=$(shell $(ROOT)/deps/readies/bin/platform --dist)
+# endif
+
+ARCH=$(shell $(ROOT)/deps/readies/bin/platform --arch)
+
+#----------------------------------------------------------------------------------------------
+
+GIT_SHA := $(shell git rev-parse HEAD)
+GIT_COMMIT := $(shell git describe --always --abbrev=7 --dirty="+")
+
+#----------------------------------------------------------------------------------------------
+
+ifeq ($(DEBUG),1)
+FLAVOR=debug
+else
+FLAVOR=release
+endif
+
+#----------------------------------------------------------------------------------------------
+
+__VARIANT__=$(shell if [ -f $(ROOT)/VARIANT ]; then cat $(ROOT)/VARIANT; fi)
+
+ifeq ($(origin VARIANT),undefined)
+ifneq ($(__VARIANT__),)
+VARIANT:=$(__VARIANT__)
+endif
+endif
+
+ifeq ($(VARIANT),)
+__VARIANT:=
+else
+__VARIANT:=-$(VARIANT)
+endif
+FULL_VARIANT:=$(OS)-$(ARCH)-$(FLAVOR)$(__VARIANT)
+FULL_VARIANT_REL:=$(OS)-$(ARCH)-release$(__VARIANT)
+
+ifneq ($(origin VARIANT),)
+$(eval $(shell if [ -z $(VARIANT) ]; then rm -f $(ROOT)/VARIANT; else echo $(VARIANT)>$(ROOT)/VARIANT; fi))
+endif
diff --git a/deps/readies/mk/variant.rules b/deps/readies/mk/variant.rules
new file mode 100755
index 000000000..c7580000e
--- /dev/null
+++ b/deps/readies/mk/variant.rules
@@ -0,0 +1,3 @@
+
+show-variant:
+	@cat $(ROOT)/VARIANT
\ No newline at end of file
diff --git a/deps/readies/paella/__init__.py b/deps/readies/paella/__init__.py
new file mode 100755
index 000000000..286fecdd3
--- /dev/null
+++ b/deps/readies/paella/__init__.py
@@ -0,0 +1,32 @@
+
+from .debug import *
+from .utils import *
+from .files import *
+# from .docopt import docopt
+from .log import *
+from .platform import *
+from .setup import *
+
+#----------------------------------------------------------------------------------------------
+
+import sys
+
+class global_injector:
+    def __init__(self):
+        try:
+            # Python 2
+            self.__dict__['builtin'] = sys.modules['__builtin__'].__dict__
+        except KeyError:
+            # Python 3
+            self.__dict__['builtin'] = sys.modules['builtins'].__dict__
+    def __setattr__(self,name,value):
+        self.builtin[name] = value
+
+Global = global_injector()
+
+#----------------------------------------------------------------------------------------------
+
+Global.bb = bb
+Global.eprint = eprint
+Global.fatal = fatal
+Global.cwd = cwd
diff --git a/deps/readies/paella/debug.py b/deps/readies/paella/debug.py
new file mode 100755
index 000000000..1b60ab7a9
--- /dev/null
+++ b/deps/readies/paella/debug.py
@@ -0,0 +1,11 @@
+
+import os
+
+#----------------------------------------------------------------------------------------------
+
+if 'PYDEBUG' in os.environ:
+	from pdb import set_trace as bb
+else:
+	def bb(): pass
+
+#----------------------------------------------------------------------------------------------
diff --git a/deps/readies/paella/docopt1.py b/deps/readies/paella/docopt1.py
new file mode 100755
index 000000000..dc08ee3c5
--- /dev/null
+++ b/deps/readies/paella/docopt1.py
@@ -0,0 +1,25 @@
+
+from docopt import docopt as docopt1
+from collections import namedtuple
+import re
+
+def dict_to_obj(d):
+	def add_to_dict_if_match(r, k, v, d):
+		m = re.match(r, k)
+		if m:
+			d[m[1].replace('-', '_')] = v
+		return not not m
+
+	d1 = dict()
+	for k, v in d.items():
+		if isinstance(v, dict):
+			d1[k] = dict_to_obj(v)
+		elif add_to_dict_if_match('--(.*)', k, v, d1):
+			pass
+		elif add_to_dict_if_match('\<(.*)\>', k, v, d1):
+			pass
+	return namedtuple('object', d1.keys())(*d1.values())
+
+def docopt(*args, **kwargs):
+	a = docopt1(*args, **kwargs)
+	return dict_to_obj(a)
diff --git a/deps/readies/paella/files.py b/deps/readies/paella/files.py
new file mode 100755
index 000000000..b38199803
--- /dev/null
+++ b/deps/readies/paella/files.py
@@ -0,0 +1,23 @@
+
+from contextlib import contextmanager
+import os
+
+def fread(fname, mode = 'rb'):
+	with open(fname, mode) as file:
+		return file.read()
+
+def flines(fname, mode = 'rb'):
+	return [line.rstrip() for line in open(fname)]
+
+@contextmanager
+def cwd(path):
+    d0 = os.getcwd()
+    os.chdir(str(path))
+    try:
+        yield
+    finally:
+        os.chdir(d0)
+
+def mkdir_p(dir):
+    if dir != '':
+        os.makedirs(dir, exist_ok=True)
diff --git a/deps/readies/paella/log.py b/deps/readies/paella/log.py
new file mode 100755
index 000000000..3edaec3ee
--- /dev/null
+++ b/deps/readies/paella/log.py
@@ -0,0 +1,6 @@
+
+import os
+
+def fatal(text):
+	eprint("%s: %s" %(os.path.basename(__file__), text))
+	exit(1)
diff --git a/deps/readies/paella/platform.py b/deps/readies/paella/platform.py
new file mode 100755
index 000000000..d8496ceb8
--- /dev/null
+++ b/deps/readies/paella/platform.py
@@ -0,0 +1,158 @@
+
+from __future__ import absolute_import
+import platform
+
+#----------------------------------------------------------------------------------------------
+
+class Platform:
+    def __init__(self):
+        self.os = self.dist = self.os_ver = self.full_os_ver = self.os_nick = self.arch = '?'
+    
+        self.os = platform.system().lower()
+        dist = platform.linux_distribution()
+        distname = dist[0].lower()
+        self.os_ver = self.full_os_ver = dist[1]
+        if self.os == 'linux':
+            if distname == 'fedora' or distname == 'ubuntu' or  distname == 'debian' or distname == 'arch':
+                pass
+            elif distname == 'centos linux':
+                distname = 'centos'
+            elif distname.startswith('redhat'):
+                distname = 'redhat'
+            elif distname.startswith('suse'):
+                distname = 'suse'
+            else:
+                Assert(False), "Cannot determine distribution"
+            self.dist = distname
+        elif self.os == 'darwin':
+            self.os = 'macosx'
+            self.dist = ''
+            mac_ver = platform.mac_ver()
+            self.full_os_ver = mac_ver[0] # e.g. 10.14, but also 10.5.8
+            self.os_ver = '.'.join(self.full_os_ver.split('.')[:2]) # major.minor
+            # self.arch = mac_ver[2] # e.g. x64_64
+        elif self.os == 'windows':
+            self.dist = self.os
+            self.os_ver = platform.release()
+            self.full_os_ver = os.version()
+        elif self.os == 'sunos':
+            self.os = 'solaris'
+            self.os_ver = ''
+            self.dist = ''
+        else:
+            Assert(False), "Cannot determine OS"
+
+        self.arch = platform.machine().lower()
+        if self.arch == 'amd64' or self.arch == 'x86_64':
+            self.arch = 'x64'
+        elif self.arch == 'i386' or self.arch == 'i686' or self.arch == 'i86pc':
+            self.arch = 'x86'
+        elif self.arch == 'aarch64':
+            self.arch = 'arm64v8'
+        elif self.arch == 'armv7l':
+            self.arch = 'arm32v7'
+
+    def is_debian_compat(self):
+        return self.dist == 'debian' or self.dist == 'ubuntu'
+    
+    def is_redhat_compat(self):
+        return self.dist == 'redhat' or self.dist == 'centos'
+    
+    def report(self):
+        print("This system is " + self.distname + " " + self.distver + ".\n")
+
+#----------------------------------------------------------------------------------------------
+
+class OnPlatform:
+    def __init__(self):
+        self.stages = [0]
+        self.platform = Platform()
+
+    def invoke(self):
+        os = self.os = self.platform.os
+        dist = self.dist = self.platform.dist
+        self.ver = self.platform.os_ver
+        self.common_first()
+
+        for stage in self.stages:
+            self.stage = stage
+            self.common()
+            if os == 'linux':
+                self.linux()
+                
+                if self.platform.is_debian_compat():
+                    self.debian_compat()
+                if self.platform.is_redhat_compat():
+                    self.redhat_compat()
+                
+                if dist == 'fedora':
+                    self.fedora()
+                elif dist == 'ubuntu':
+                    self.ubuntu()
+                elif dist == 'debian':
+                    self.debian()
+                elif dist == 'centos':
+                    self.centos()
+                elif dist == 'redhat':
+                    self.redhat()
+                elif dist == 'suse':
+                    self.suse()
+                elif dist == 'arch':
+                    self.arch()
+                else:
+                    Assert(False), "Cannot determine installer"
+            elif os == 'macosx':
+                self.macosx()
+
+        self.common_last()
+
+    def common(self):
+        pass
+
+    def common_first(self):
+        pass
+
+    def common_last(self):
+        pass
+
+    def linux(self):
+        pass
+
+    def arch(self):
+        pass
+
+    def debian_compat(self): # debian, ubuntu, etc
+        pass
+    
+    def debian(self):
+        pass
+    
+    def centos(self):
+        pass
+        
+    def fedora(self):
+        pass
+
+    def redhat_compat(self): # centos, rhel
+        pass
+
+    def redhat(self):
+        pass
+        
+    def ubuntu(self):
+        pass
+
+    def suse(self):
+        pass
+
+    def macosx(self):
+        pass
+
+    def windows(self):
+        pass
+
+    def bsd_compat(self):
+        pass
+
+    def freebsd(self):
+        pass
diff --git a/deps/readies/paella/setup.py b/deps/readies/paella/setup.py
new file mode 100755
index 000000000..65928faee
--- /dev/null
+++ b/deps/readies/paella/setup.py
@@ -0,0 +1,143 @@
+
+import os
+import sys
+from .platform import OnPlatform, Platform
+
+#----------------------------------------------------------------------------------------------
+
+class Runner:
+    def __init__(self, nop=False):
+        self.nop = nop
+
+    def run(self, cmd):
+        if self.nop:
+            print(cmd)
+            return
+        rc = os.system(cmd)
+        if rc > 0:
+            eprint("command failed: " + cmd)
+            sys.exit(1)
+
+    def has_command(self, cmd):
+        return os.system("command -v " + cmd + " > /dev/null") == 0
+
+#----------------------------------------------------------------------------------------------
+
+class RepoRefresh(OnPlatform):
+    def __init__(self, runner):
+        OnPlatform.__init__(self)
+        self.runner = runner
+
+    def redhat_compat(self):
+        pass
+    
+    def debian_compat(self):
+        self.runner.run("apt-get -qq update -y")
+
+#----------------------------------------------------------------------------------------------
+
+class Setup(OnPlatform):
+    def __init__(self, nop=False):
+        OnPlatform.__init__(self)
+        self.runner = Runner(nop)
+        self.stages = [0]
+        self.platform = Platform()
+        self.os = self.platform.os
+        self.dist = self.platform.dist
+        self.ver = self.platform.os_ver
+        
+        if self.has_command("python"):
+            self.python = "python"
+        elif self.has_command("python2"):
+            self.python = "python2"
+
+        if self.os == 'macosx':
+            # this is required because osx pip installed are done with --user
+            os.environ["PATH"] = os.environ["PATH"] + ':' + '$HOME/Library/Python/2.7/bin'
+        
+        if self.platform.is_debian_compat():
+            # prevents apt-get from interactively prompting
+            os.environ["DEBIAN_FRONTEND"] = 'noninteractive'
+        
+        os.environ["PYTHONWARNINGS"] = 'ignore:DEPRECATION::pip._internal.cli.base_command'
+
+    def setup(self):
+        RepoRefresh(self.runner).invoke()
+        self.invoke()
+
+    def run(self, cmd):
+        return self.runner.run(cmd)
+
+    def has_command(self, cmd):
+        return self.runner.has_command(cmd)
+
+    #------------------------------------------------------------------------------------------
+
+    def apt_install(self, packs, group=False):
+        self.run("apt-get -qq install -y " + packs)
+
+    def yum_install(self, packs, group=False):
+        if not group:
+            self.run("yum install -q -y " + packs)
+        else:
+            self.run("yum groupinstall -y " + packs)
+
+    def dnf_install(self, packs, group=False):
+        if not group:
+            self.run("dnf install -y " + packs)
+        else:
+            self.run("dnf groupinstall -y " + packs)
+
+    def zypper_install(self, packs, group=False):
+        self.run("zipper --non-interactive install " + packs)
+
+    def pacman_install(self, packs, group=False):
+        self.run("pacman --noconfirm -S " + packs)
+
+    def brew_install(self, packs, group=False):
+        self.run('brew install ' + packs)
+
+    def install(self, packs, group=False):
+        if self.os == 'linux':
+            if self.dist == 'fedora':
+                self.dnf_install(packs, group)
+            elif self.dist == 'ubuntu' or self.dist == 'debian':
+                self.apt_install(packs, group)
+            elif self.dist == 'centos' or self.dist == 'redhat':
+                self.yum_install(packs, group)
+            elif self.dist == 'suse':
+                self.zypper_install(packs, group)
+            elif self.dist == 'arch':
+                self.pacman_install(packs, group)
+            else:
+                Assert(False), "Cannot determine installer"
+        elif self.os == 'macosx':
+            self.brew_install(packs, group)
+        else:
+            Assert(False), "Cannot determine installer"
+
+    def group_install(self, packs):
+        self.install(packs, group=True)
+
+    #------------------------------------------------------------------------------------------
+
+    def pip_install(self, cmd):
+        pip_user = ''
+        if self.os == 'macosx':
+            pip_user = '--user '
+        self.run("pip install --disable-pip-version-check " + pip_user + cmd)
+
+    def pip3_install(self, cmd):
+        pip_user = ''
+        if self.os == 'macosx':
+            pip_user = '--user '
+        self.run("pip3 install --disable-pip-version-check " + pip_user + cmd)
+
+    def setup_pip(self):
+        get_pip = "set -e; cd /tmp; curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py"
+        if not self.has_command("pip"):
+            self.install("curl")
+            self.run(get_pip + "; " + self.python + " get-pip.py")
+        ## fails on ubuntu 18:
+        # if not has_command("pip3") and has_command("python3"):
+        #     run(get_pip + "; python3 get-pip.py")
diff --git a/deps/readies/paella/utils.py b/deps/readies/paella/utils.py
new file mode 100755
index 000000000..a46d97576
--- /dev/null
+++ b/deps/readies/paella/utils.py
@@ -0,0 +1,7 @@
+
+import sys
+
+if (sys.version_info > (3, 0)):
+	from .utils3 import *
+else:
+	from .utils2 import *
diff --git a/deps/readies/paella/utils2.py b/deps/readies/paella/utils2.py
new file mode 100755
index 000000000..bad63520e
--- /dev/null
+++ b/deps/readies/paella/utils2.py
@@ -0,0 +1,5 @@
+
+import sys
+
+def eprint(*args, **kwargs):
+	print >> sys.stderr, ' '.join(map(lambda x: "%s" % x, args))
diff --git a/deps/readies/paella/utils3.py b/deps/readies/paella/utils3.py
new file mode 100755
index 000000000..2f552cd34
--- /dev/null
+++ b/deps/readies/paella/utils3.py
@@ -0,0 +1,5 @@
+
+import sys
+
+def eprint(*args, **kwargs):
+	print(*args, file = sys.stderr, **kwargs)
diff --git a/deps/readies/shibumi/functions b/deps/readies/shibumi/functions
new file mode 100755
index 000000000..6eb9450dd
--- /dev/null
+++ b/deps/readies/shibumi/functions
@@ -0,0 +1,21 @@
+
+platform_os() {
+	case "$OSTYPE" in
+		linux*)   echo "linux" ;;
+		solaris*) echo "solaris" ;;
+		darwin*)  echo "macosx" ;; 
+		bsd*)     echo "bsd" ;;
+		msys*)    echo "windows" ;;
+		*)        echo "?" ;;
+	esac
+}
+
+#----------------------------------------------------------------------------------------------
+
+if [[ $(platform_os) == mac ]]; then
+
+realpath() {
+	[[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}"
+}
+
+fi
diff --git a/system-setup.py b/system-setup.py
new file mode 100755
index 000000000..258ac12e3
--- /dev/null
+++ b/system-setup.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python2
+
+import sys
+import os
+import popen2
+import argparse
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "deps/readies"))
+import paella
+
+#----------------------------------------------------------------------------------------------
+
+class RedisGearsSetup(paella.Setup):
+    def __init__(self, nop=False):
+        paella.Setup.__init__(self, nop)
+
+    def common_first(self):
+        self.setup_pip()
+        self.pip_install("wheel")
+        self.pip_install("setuptools --upgrade")
+        
+        self.install("git python3 cmake ca-certificates curl unzip wget patchelf awscli")
+
+    def debian_compat(self):
+        # self.run("curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash")
+        # self.install("git-lfs")
+        # self.run("git lfs install")
+
+        self.install("build-essential")
+        self.install("python-psutil")
+
+    def redhat_compat(self):
+        self.group_install("'Development Tools'")
+        self.install("redhat-lsb-core")
+
+        # uninstall and install psutil (order is important), otherwise RLTest fails
+        self.run("pip uninstall -y psutil")
+        self.install("python2-psutil")
+
+    def fedora(self):
+        self.group_install("'Development Tools'")
+
+    def macosx(self):
+        r, w, e = popen2.popen3('xcode-select -p')
+        if r.readlines() == []:
+            fatal("Xcode tools are not installed. Please run xcode-select --install.")
+
+    def common_last(self):
+        if not self.has_command("ramp"):
+            self.pip_install("git+https://github.com/RedisLabs/RAMP --upgrade")
+        if not self.has_command("RLTest"):
+            self.pip_install("git+https://github.com/RedisLabsModules/RLTest.git@master")
+
+#----------------------------------------------------------------------------------------------
+
+parser = argparse.ArgumentParser(description='Set up system for build.')
+parser.add_argument('-n', '--nop', action="store_true", help='no operation')
+args = parser.parse_args()
+
+RedisGearsSetup(nop = args.nop).setup()

From b5c47bee4879539a706bb9683e0b59e43fad60ec Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Wed, 31 Jul 2019 21:36:41 +0300
Subject: [PATCH 03/31] Fixed docker functionality, fixed tests, setup changes

- Added deps/readies and system-setup.py to provide standard system setup,
- Docker: load all backends,
- Tests: load backends upfront to avoid rdb problems,
- Removed Python virtual environment setup in Makefile to avoid setup
  nightmares in various platforms,
- Docker: build using debian:buster, then copy binaries into redis:latest.
---
 .dockerignore                   |  9 ++++--
 Dockerfile                      | 55 ++++++++++++++-------------------
 Makefile                        | 31 +++++--------------
 deps/readies/bin/getpy2         | 22 ++++++++-----
 deps/readies/bin/getredis5      |  2 ++
 deps/readies/paella/platform.py |  9 +++++-
 deps/readies/paella/setup.py    | 40 +++++++++++++++---------
 deps/readies/shibumi/here       |  3 ++
 get_deps.sh                     |  8 ++---
 system-setup.py                 | 20 ++++++------
 test/basic_tests.py             | 22 ++++++-------
 11 files changed, 114 insertions(+), 107 deletions(-)
 mode change 100644 => 100755 Dockerfile
 create mode 100755 deps/readies/shibumi/here

diff --git a/.dockerignore b/.dockerignore
index caac77506..8a0cea68e 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,4 +1,7 @@
-deps/*
-install/*
+/deps/*
+!/deps/readies/
+!/deps/*.py
+/build/
+/install/
 **/*.o
-**/*.so
\ No newline at end of file
+**/*.so
diff --git a/Dockerfile b/Dockerfile
old mode 100644
new mode 100755
index 24889bc42..6f4742c3c
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,46 +1,37 @@
-FROM redis AS builder
+#----------------------------------------------------------------------------------------------
+FROM redis AS redis
+FROM debian:buster AS builder
 
-ENV DEPS "build-essential git ca-certificates curl unzip wget libgomp1 patchelf"
+WORKDIR /redisai
+COPY --from=redis /usr/local/ /usr/local/
 
-# install latest cmake
-ADD https://cmake.org/files/v3.12/cmake-3.12.4-Linux-x86_64.sh /cmake-3.12.4-Linux-x86_64.sh
-RUN mkdir /opt/cmake
-RUN sh /cmake-3.12.4-Linux-x86_64.sh --prefix=/opt/cmake --skip-license
-RUN ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake
-RUN cmake --version
+COPY ./deps/readies/ deps/readies/
+COPY ./system-setup.py .
+COPY ./test/test_requirements.txt test/
 
-# Set up a build environment
-RUN set -ex;\
-    deps="$DEPS";\
-    apt-get update;\
-    apt-get install -y --no-install-recommends $deps
+RUN ./deps/readies/bin/getpy2
+RUN ./system-setup.py
+
+COPY ./get_deps.sh .
+RUN ./get_deps.sh cpu
 
-# Get the dependencies
-WORKDIR /redisai
 ADD ./ /redisai
-RUN set -ex;\
-    mkdir -p deps;\
-    DEPS_DIRECTORY=deps bash ./get_deps.sh cpu
-
-# Build the source
-RUN set -ex;\
-    rm -rf build;\
-    mkdir -p build;\
-    cd build;\
-    cmake -DDEPS_PATH=../deps/install ..;\
-    make && make install;\
-    cd ..
-
-# Package the runner
+RUN make all
+# RUN make pack
+# RUN make test
+
+#----------------------------------------------------------------------------------------------
 FROM redis
 
 RUN set -e; apt-get -qq update; apt-get install -y libgomp1
 
-RUN set -ex;\
-    mkdir -p /usr/lib/redis/modules/;
+RUN mkdir -p /usr/lib/redis/modules/
 
 COPY --from=builder /redisai/install/ /usr/lib/redis/modules/
 
 WORKDIR /data
 EXPOSE 6379
-CMD ["--loadmodule", "/usr/lib/redis/modules/redisai.so"]
+CMD ["--loadmodule", "/usr/lib/redis/modules/redisai.so", \
+        "TF", "/usr/lib/redis/modules//backends/redisai_tensorflow/redisai_tensorflow.so", \
+        "TORCH", "/usr/lib/redis/modules//backends/redisai_torch/redisai_torch.so", \
+        "ONNX", "/usr/lib/redis/modules//backends/redisai_onnxruntime/redisai_onnxruntime.so"]
diff --git a/Makefile b/Makefile
index f0399b8c2..efa06faf1 100755
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,3 @@
-
 ifeq ($(CUDA),1)
 DEPS_FLAGS=
 else
@@ -22,9 +21,6 @@ BINDIR=$(PWD)/install
 
 all: build
 
-version:
-	@echo $(PACK_VER)
-
 build:
 ifeq ($(wildcard build/.),)
 	mkdir -p build
@@ -36,7 +32,7 @@ endif
 
 clean:
 ifeq ($(ALL),1)
-	rm -rf build deps
+	rm -rf build deps/install install
 else
 	$(MAKE) -C build clean
 endif
@@ -47,14 +43,8 @@ deps:
 
 pack:
 	@[ ! -z `command -v redis-server` ] || { echo "Cannot find redis-server - aborting."; exit 1; }
-ifeq ($(wildcard build/pyenv/.),)
-	@virtualenv build/pyenv ;\
-	. ./build/pyenv/bin/activate ;\
-	pip install git+https://github.com/RedisLabs/RAMP
-endif
 	@echo "Building RAMP file ..."
 	@set -e ;\
-	. ./build/pyenv/bin/activate ;\
 	RAMPOUT=$$(mktemp /tmp/ramp.XXXXXX) ;\
 	LD_LIBRARY_PATH=$(PWD)/deps/install/lib \
 	ramp pack -m $(PWD)/ramp.yml -o "build/redisai.{os}-{architecture}.{semantic_version}.zip" $(BINDIR)/redisai.so 2> /dev/null | grep '.zip' > $$RAMPOUT ;\
@@ -66,21 +56,14 @@ endif
 	PACK_FNAME=$$(basename `cat $(BINDIR)/PACKAGE`) ;\
 	ARCHOSVER=$$(echo "$$PACK_FNAME" | sed -e "s/^redisai\.\([^.]*\..*\)\.zip/\1/") ;\
 	cd install ;\
-	find backends -name "*.so*" | xargs tar pczf redisai-dependencies.$$ARCHOSVER.tgz
-	@echo "Done."
+	find backends -name "*.so*" | xargs tar pczf redisai-dependencies.$$ARCHOSVER.tgz ;\
+	echo "Done."
 
 test:
-ifeq ($(wildcard venv/.),)
-	@python3 -m venv test/venv
-	@set -e ;\
-	cd test ;\
-	. venv/bin/activate ;\
-	pip -q install git+https://github.com/RedisLabsModules/RLTest@master ;\
-	pip -q install -r test_requirements.txt
 	@git lfs pull
-endif
 	@set -e ;\
 	cd test ;\
-	. venv/bin/activate ;\
-	LD_LIBRARY_PATH=$(PWD)/install \
-	python3 -m RLTest --test basic_tests.py --module $(BINDIR)/redisai.so
+	python3 -m RLTest --test basic_tests.py --module $(BINDIR)/redisai.so \
+		--module-args "TF $(BINDIR)/backends/redisai_tensorflow/redisai_tensorflow.so ONNX $(BINDIR)/backends/redisai_onnxruntime/redisai_onnxruntime.so TORCH $(BINDIR)/backends/redisai_torch/redisai_torch.so"
+
+# LD_LIBRARY_PATH=$(PWD)/install
\ No newline at end of file
diff --git a/deps/readies/bin/getpy2 b/deps/readies/bin/getpy2
index ea12aff36..aa1908e64 100755
--- a/deps/readies/bin/getpy2
+++ b/deps/readies/bin/getpy2
@@ -1,4 +1,10 @@
-#!/bin/sh
+#!/bin/bash
+
+show_if_error() {
+	{ "${@:1}"; } > /tmp/py2.log  2>&1
+	[ $? != 0 ] && cat /tmp/py2.log
+	rm -f /tmp/py2.log
+}
 
 if [ ! -z $(command -v python) ]; then
 	[ "$(python --version 2>&1 | cut -d" " -f2 | cut -d. -f1)" = "2" ] && exit 0
@@ -7,17 +13,17 @@ fi
 [ ! -z $(command -v python2) ] && exit 0
 
 if [ ! -z $(command -v apt-get) ]; then
-	apt-get -qq update
-	apt-get -qq install -y python
+	show_if_error apt-get -qq update
+	show_if_error apt-get -qq install -y python
 elif [ ! -z $(command -v dnf) ]; then
-	dnf install -y python2
+	show_if_error dnf install -y python2
 elif [ ! -z $(command -v yum) ]; then
-	yum install -y python2
+	show_if_error yum install -y python2
 elif [ ! -z $(command -v apk) ]; then
-	apk update
-	apk add python2
+	show_if_error apk update
+	show_if_error apk add python2
 elif [ ! -z $(command -v brew) ]; then
-	brew install python2
+	show_if_error brew install python2
 fi
 
 if [ -z $(command -v python) ]; then
diff --git a/deps/readies/bin/getredis5 b/deps/readies/bin/getredis5
index 883c8058e..3a0cd973b 100755
--- a/deps/readies/bin/getredis5
+++ b/deps/readies/bin/getredis5
@@ -23,6 +23,8 @@ class Redis5Setup(paella.Setup):
         # https://chilts.org/installing-redis-from-chris-leas-ppa/
         self.run("add-apt-repository -y ppa:chris-lea/redis-server")
         self.install("redis-server")
+        # if not removed, might break apt-get update
+        self.run("add-apt-repository -r -y ppa:chris-lea/redis-server")
 
     def redhat_compat(self):
         # https://linuxize.com/post/how-to-install-and-configure-redis-on-centos-7/
diff --git a/deps/readies/paella/platform.py b/deps/readies/paella/platform.py
index d8496ceb8..e748cf728 100755
--- a/deps/readies/paella/platform.py
+++ b/deps/readies/paella/platform.py
@@ -57,7 +57,14 @@ def is_debian_compat(self):
     
     def is_redhat_compat(self):
         return self.dist == 'redhat' or self.dist == 'centos'
-    
+
+    def is_container(self):
+        with open('/proc/1/cgroups', 'r') as conf:
+            for line in conf:
+                if re.search('docker', line):
+                    return True
+        return False
+
     def report(self):
         print("This system is " + self.distname + " " + self.distver + ".\n")
 
diff --git a/deps/readies/paella/setup.py b/deps/readies/paella/setup.py
index 65928faee..7fa976d14 100755
--- a/deps/readies/paella/setup.py
+++ b/deps/readies/paella/setup.py
@@ -1,6 +1,7 @@
 
 import os
 import sys
+import tempfile
 from .platform import OnPlatform, Platform
 
 #----------------------------------------------------------------------------------------------
@@ -9,13 +10,22 @@ class Runner:
     def __init__(self, nop=False):
         self.nop = nop
 
-    def run(self, cmd):
+    def run(self, cmd, output_on_error=False):
+        print(cmd)
+        sys.stdout.flush()
         if self.nop:
-            print(cmd)
             return
+        if output_on_error:
+            fd, temppath = tempfile.mkstemp()
+            os.close(fd)
+            cmd = "{{ {}; }} >{} 2>&1".format(cmd, temppath)
         rc = os.system(cmd)
         if rc > 0:
+            if output_on_error:
+                os.system("cat {}".format(temppath))
+                os.remove(temppath)
             eprint("command failed: " + cmd)
+            sys.stderr.flush()
             sys.exit(1)
 
     def has_command(self, cmd):
@@ -65,8 +75,8 @@ def setup(self):
         RepoRefresh(self.runner).invoke()
         self.invoke()
 
-    def run(self, cmd):
-        return self.runner.run(cmd)
+    def run(self, cmd, output_on_error=False):
+        return self.runner.run(cmd, output_on_error=output_on_error)
 
     def has_command(self, cmd):
         return self.runner.has_command(cmd)
@@ -74,28 +84,28 @@ def has_command(self, cmd):
     #------------------------------------------------------------------------------------------
 
     def apt_install(self, packs, group=False):
-        self.run("apt-get -qq install -y " + packs)
+        self.run("apt-get -qq install -y " + packs, output_on_error=True)
 
     def yum_install(self, packs, group=False):
         if not group:
-            self.run("yum install -q -y " + packs)
+            self.run("yum install -q -y " + packs, output_on_error=True)
         else:
-            self.run("yum groupinstall -y " + packs)
+            self.run("yum groupinstall -y " + packs, output_on_error=True)
 
     def dnf_install(self, packs, group=False):
         if not group:
-            self.run("dnf install -y " + packs)
+            self.run("dnf install -y " + packs, output_on_error=True)
         else:
-            self.run("dnf groupinstall -y " + packs)
+            self.run("dnf groupinstall -y " + packs, output_on_error=True)
 
     def zypper_install(self, packs, group=False):
-        self.run("zipper --non-interactive install " + packs)
+        self.run("zipper --non-interactive install " + packs, output_on_error=True)
 
     def pacman_install(self, packs, group=False):
-        self.run("pacman --noconfirm -S " + packs)
+        self.run("pacman --noconfirm -S " + packs, output_on_error=True)
 
     def brew_install(self, packs, group=False):
-        self.run('brew install ' + packs)
+        self.run('brew install ' + packs, output_on_error=True)
 
     def install(self, packs, group=False):
         if self.os == 'linux':
@@ -125,19 +135,19 @@ def pip_install(self, cmd):
         pip_user = ''
         if self.os == 'macosx':
             pip_user = '--user '
-        self.run("pip install --disable-pip-version-check " + pip_user + cmd)
+        self.run("pip install --disable-pip-version-check " + pip_user + cmd, output_on_error=True)
 
     def pip3_install(self, cmd):
         pip_user = ''
         if self.os == 'macosx':
             pip_user = '--user '
-        self.run("pip3 install --disable-pip-version-check " + pip_user + cmd)
+        self.run("pip3 install --disable-pip-version-check " + pip_user + cmd, output_on_error=True)
 
     def setup_pip(self):
         get_pip = "set -e; cd /tmp; curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py"
         if not self.has_command("pip"):
             self.install("curl")
-            self.run(get_pip + "; " + self.python + " get-pip.py")
+            self.run(get_pip + "; " + self.python + " get-pip.py", output_on_error=True)
         ## fails on ubuntu 18:
         # if not has_command("pip3") and has_command("python3"):
         #     run(get_pip + "; python3 get-pip.py")
diff --git a/deps/readies/shibumi/here b/deps/readies/shibumi/here
new file mode 100755
index 000000000..a6e56964e
--- /dev/null
+++ b/deps/readies/shibumi/here
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" 
diff --git a/get_deps.sh b/get_deps.sh
index 468997e94..892986952 100755
--- a/get_deps.sh
+++ b/get_deps.sh
@@ -62,7 +62,7 @@ LIBTF_ARCHIVE=libtensorflow-${TF_BUILD}-${TF_OS}-x86_64-${TF_VERSION}.tar.gz
 
 if [ ! -e ${LIBTF_ARCHIVE} ]; then
   echo "Downloading libtensorflow ${TF_VERSION} ${TF_BUILD}"
-  wget https://storage.googleapis.com/tensorflow/libtensorflow/${LIBTF_ARCHIVE}
+  wget -q https://storage.googleapis.com/tensorflow/libtensorflow/${LIBTF_ARCHIVE}
 fi
 
 tar xf ${LIBTF_ARCHIVE} --no-same-owner --strip-components=1 -C ${TF_PREFIX}
@@ -98,7 +98,7 @@ LIBTORCH_ARCHIVE=libtorch-${PT_OS}-${PT_BUILD}-${PT_VERSION}.zip
 
 if [ ! -e "${LIBTORCH_ARCHIVE}" ]; then
   echo "Downloading libtorch ${PT_VERSION} ${PT_BUILD}"
-  curl -L ${LIBTORCH_URL} > ${LIBTORCH_ARCHIVE}
+  curl -s -L ${LIBTORCH_URL} > ${LIBTORCH_ARCHIVE}
 fi
 
 unzip -o ${LIBTORCH_ARCHIVE}
@@ -109,7 +109,7 @@ if [[ "${PT_OS}" == "macos" ]]; then
   # also download mkl
   MKL_BUNDLE=mklml_mac_2019.0.3.20190220
   if [ ! -e "${MKL_BUNDLE}.tgz" ]; then
-    wget "https://github.com/intel/mkl-dnn/releases/download/v0.18/${MKL_BUNDLE}.tgz"
+    wget -q "https://github.com/intel/mkl-dnn/releases/download/v0.18/${MKL_BUNDLE}.tgz"
   fi
   tar xf ${MKL_BUNDLE}.tgz --no-same-owner --strip-components=1 -C ${TORCH_PREFIX}
   tar xf ${MKL_BUNDLE}.tgz --no-same-owner --strip-components=1 -C ${ORT_PREFIX}
@@ -134,7 +134,7 @@ ORT_ARCHIVE=onnxruntime-${ORT_OS}-${ORT_VERSION}.tgz
 
 if [ ! -e ${ORT_ARCHIVE} ]; then
   echo "Downloading ONNXRuntime ${ORT_VERSION} ${ORT_BUILD}"
-  wget https://github.com/Microsoft/onnxruntime/releases/download/v${ORT_VERSION}/${ORT_ARCHIVE}
+  wget -q https://github.com/Microsoft/onnxruntime/releases/download/v${ORT_VERSION}/${ORT_ARCHIVE}
 fi
 
 tar xf ${ORT_ARCHIVE} --no-same-owner --strip-components=1 -C ${ORT_PREFIX}
diff --git a/system-setup.py b/system-setup.py
index 258ac12e3..bfe6effd1 100755
--- a/system-setup.py
+++ b/system-setup.py
@@ -10,7 +10,7 @@
 
 #----------------------------------------------------------------------------------------------
 
-class RedisGearsSetup(paella.Setup):
+class RedisAISetup(paella.Setup):
     def __init__(self, nop=False):
         paella.Setup.__init__(self, nop)
 
@@ -19,15 +19,16 @@ def common_first(self):
         self.pip_install("wheel")
         self.pip_install("setuptools --upgrade")
         
-        self.install("git python3 cmake ca-certificates curl unzip wget patchelf awscli")
+        self.install("git cmake ca-certificates curl unzip wget patchelf awscli")
+        self.install("python3 python3-pip python3-venv python3-psutil python3-networkx")
 
     def debian_compat(self):
-        # self.run("curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash")
-        # self.install("git-lfs")
-        # self.run("git lfs install")
+        self.run("curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash")
+        self.install("git-lfs")
+        self.run("git lfs install")
 
         self.install("build-essential")
-        self.install("python-psutil")
+        # self.install("python-psutil")
 
     def redhat_compat(self):
         self.group_install("'Development Tools'")
@@ -47,9 +48,10 @@ def macosx(self):
 
     def common_last(self):
         if not self.has_command("ramp"):
-            self.pip_install("git+https://github.com/RedisLabs/RAMP --upgrade")
+            self.pip3_install("git+https://github.com/RedisLabs/RAMP --upgrade")
         if not self.has_command("RLTest"):
-            self.pip_install("git+https://github.com/RedisLabsModules/RLTest.git@master")
+            self.pip3_install("git+https://github.com/RedisLabsModules/RLTest.git@master")
+        self.pip3_install("-r test/test_requirements.txt")
 
 #----------------------------------------------------------------------------------------------
 
@@ -57,4 +59,4 @@ def common_last(self):
 parser.add_argument('-n', '--nop', action="store_true", help='no operation')
 args = parser.parse_args()
 
-RedisGearsSetup(nop = args.nop).setup()
+RedisAISetup(nop = args.nop).setup()
diff --git a/test/basic_tests.py b/test/basic_tests.py
index 588e8c2ef..00eb82b42 100644
--- a/test/basic_tests.py
+++ b/test/basic_tests.py
@@ -111,7 +111,7 @@ def test_del_tf_model(env):
 
     con = env
 
-    con.execute_command('AI.CONFIG', 'LOADBACKEND', 'TF', os.path.join(BACKENDS_PATH, 'redisai_tensorflow/redisai_tensorflow.so'))
+    # con.execute_command('AI.CONFIG', 'LOADBACKEND', 'TF', os.path.join(BACKENDS_PATH, 'redisai_tensorflow/redisai_tensorflow.so'))
 
     ret = con.execute_command('AI.MODELSET', 'm', 'TF', 'CPU',
                               'INPUTS', 'a', 'b', 'OUTPUTS', 'mul', model_pb)
@@ -134,7 +134,7 @@ def test_run_tf_model(env):
 
     con = env
 
-    con.execute_command('AI.CONFIG', 'LOADBACKEND', 'TF', os.path.join(BACKENDS_PATH, 'redisai_tensorflow/redisai_tensorflow.so'))
+    # con.execute_command('AI.CONFIG', 'LOADBACKEND', 'TF', os.path.join(BACKENDS_PATH, 'redisai_tensorflow/redisai_tensorflow.so'))
 
     ret = con.execute_command('AI.MODELSET', 'm', 'TF', 'CPU',
                               'INPUTS', 'a', 'b', 'OUTPUTS', 'mul', model_pb)
@@ -257,7 +257,7 @@ def test_run_torch_model(env):
 
     con = env
 
-    con.execute_command('AI.CONFIG', 'LOADBACKEND', 'TORCH', os.path.join(BACKENDS_PATH, 'redisai_torch/redisai_torch.so'))
+    # con.execute_command('AI.CONFIG', 'LOADBACKEND', 'TORCH', os.path.join(BACKENDS_PATH, 'redisai_torch/redisai_torch.so'))
 
     ret = con.execute_command('AI.MODELSET', 'm', 'TORCH', 'CPU', model_pb)
     con.assertEqual(ret, b'OK')
@@ -361,7 +361,7 @@ def test_run_onnx_model(env):
 
     con = env
 
-    con.execute_command('AI.CONFIG', 'LOADBACKEND', 'ONNX', os.path.join(BACKENDS_PATH, 'redisai_onnxruntime/redisai_onnxruntime.so'))
+    # con.execute_command('AI.CONFIG', 'LOADBACKEND', 'ONNX', os.path.join(BACKENDS_PATH, 'redisai_onnxruntime/redisai_onnxruntime.so'))
 
     ret = con.execute_command('AI.MODELSET', 'm', 'ONNX', 'CPU', model_pb)
     con.assertEqual(ret, b'OK')
@@ -461,7 +461,7 @@ def test_run_onnxml_model(env):
 
     con = env
 
-    con.execute_command('AI.CONFIG', 'LOADBACKEND', 'ONNX', os.path.join(BACKENDS_PATH, 'redisai_onnxruntime/redisai_onnxruntime.so'))
+    # con.execute_command('AI.CONFIG', 'LOADBACKEND', 'ONNX', os.path.join(BACKENDS_PATH, 'redisai_onnxruntime/redisai_onnxruntime.so'))
 
     ret = con.execute_command('AI.MODELSET', 'linear', 'ONNX', 'CPU', linear_model)
     con.assertEqual(ret, b'OK')
@@ -522,7 +522,7 @@ def test_run_mobilenet(env):
     output_var = 'MobilenetV2/Predictions/Reshape_1'
     con = env
 
-    con.execute_command('AI.CONFIG', 'LOADBACKEND', 'TF', os.path.join(BACKENDS_PATH, 'redisai_tensorflow/redisai_tensorflow.so'))
+    # con.execute_command('AI.CONFIG', 'LOADBACKEND', 'TF', os.path.join(BACKENDS_PATH, 'redisai_tensorflow/redisai_tensorflow.so'))
 
     model_pb, labels, img = load_mobilenet_test_data()
 
@@ -565,7 +565,7 @@ def test_run_mobilenet_multiproc(env):
 
     con = env
 
-    con.execute_command('AI.CONFIG', 'LOADBACKEND', 'TF', os.path.join(BACKENDS_PATH, 'redisai_tensorflow/redisai_tensorflow.so'))
+    # con.execute_command('AI.CONFIG', 'LOADBACKEND', 'TF', os.path.join(BACKENDS_PATH, 'redisai_tensorflow/redisai_tensorflow.so'))
 
     model_pb, labels, img = load_mobilenet_test_data()
     con.execute_command('AI.MODELSET', 'mobilenet', 'TF', 'CPU',
@@ -587,7 +587,7 @@ def test_run_mobilenet_multiproc(env):
 
 
 def test_set_incorrect_script(env):
-    env.execute_command('AI.CONFIG', 'LOADBACKEND', 'TORCH', os.path.join(BACKENDS_PATH, 'redisai_torch/redisai_torch.so'))
+    # env.execute_command('AI.CONFIG', 'LOADBACKEND', 'TORCH', os.path.join(BACKENDS_PATH, 'redisai_torch/redisai_torch.so'))
 
     try:
         env.execute_command('AI.SCRIPTSET', 'ket', 'CPU', 'return 1')
@@ -609,7 +609,7 @@ def test_set_incorrect_script(env):
 
 
 def test_set_correct_script(env):
-    env.execute_command('AI.CONFIG', 'LOADBACKEND', 'TORCH', os.path.join(BACKENDS_PATH, 'redisai_torch/redisai_torch.so'))
+    # env.execute_command('AI.CONFIG', 'LOADBACKEND', 'TORCH', os.path.join(BACKENDS_PATH, 'redisai_torch/redisai_torch.so'))
 
     test_data_path = os.path.join(os.path.dirname(__file__), 'test_data')
     script_filename = os.path.join(test_data_path, 'script.txt')
@@ -630,7 +630,7 @@ def test_del_script(env):
     with open(script_filename, 'rb') as f:
         script = f.read()
 
-    env.execute_command('AI.CONFIG', 'LOADBACKEND', 'TORCH', os.path.join(BACKENDS_PATH, 'redisai_torch/redisai_torch.so'))
+    # env.execute_command('AI.CONFIG', 'LOADBACKEND', 'TORCH', os.path.join(BACKENDS_PATH, 'redisai_torch/redisai_torch.so'))
 
     ret = env.execute_command('AI.SCRIPTSET', 'ket', 'CPU', script)
     env.assertEqual(ret, b'OK')
@@ -640,7 +640,7 @@ def test_del_script(env):
 
 
 def test_run_script(env):
-    env.execute_command('AI.CONFIG', 'LOADBACKEND', 'TORCH', os.path.join(BACKENDS_PATH, 'redisai_torch/redisai_torch.so'))
+    # env.execute_command('AI.CONFIG', 'LOADBACKEND', 'TORCH', os.path.join(BACKENDS_PATH, 'redisai_torch/redisai_torch.so'))
 
     test_data_path = os.path.join(os.path.dirname(__file__), 'test_data')
     script_filename = os.path.join(test_data_path, 'script.txt')

From dc3c06db3bd4d174ad4c0c59f2182dc1953ffa2e Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Wed, 31 Jul 2019 21:52:23 +0300
Subject: [PATCH 04/31] CircleCI: refactoring

---
 .circleci/config.yml | 32 ++++++++------------------------
 1 file changed, 8 insertions(+), 24 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 9e5ecdb1d..3fdac959c 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1,8 +1,8 @@
-version: 2
+version: 2.1
 jobs:
   build:
     docker:
-      - image: circleci/python:3.6.8
+      - image: circleci/buster
 
     steps:
       - checkout
@@ -17,15 +17,9 @@ jobs:
       - run:
           name: install dependencies
           command: |
-            curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
-            sudo apt-get install -y git-lfs patchelf
-            git lfs install
-            curl -o cmake-3.12.4-Linux-x86_64.sh https://cmake.org/files/v3.12/cmake-3.12.4-Linux-x86_64.sh
-            mkdir /home/circleci/project/cmake-bin
-            sh cmake-3.12.4-Linux-x86_64.sh --prefix=/home/circleci/project/cmake-bin --skip-license
-            sudo ln -s /home/circleci/project/cmake-bin/bin/cmake /usr/local/bin/cmake
-            echo `cmake --version`
-            rm -rf /home/circleci/project/deps
+            ./deps/readies/bin/getpy2
+            ./system-setup.py
+            make deps
             bash get_deps.sh cpu
             git clone git://github.com/antirez/redis.git --branch 5.0.3
             (cd redis && make malloc=libc -j4 && sudo make install)
@@ -38,22 +32,12 @@ jobs:
       - run:
           name: Build
           command: |
-            rm -rf build
-            mkdir build
-            cd build
-            cmake -DDEPS_PATH=../deps/install ..
-            make
+            make all
+            make pack
 
       - run:
           name: Test
-          command: |
-            python3 -m venv venv
-            . venv/bin/activate
-            pip -q uninstall -y RLTest || true
-            pip -q install git+https://github.com/RedisLabsModules/RLTest@master
-            pip -q install -r test/test_requirements.txt
-            git lfs pull
-            python3 -m RLTest --test test/basic_tests.py --module install/redisai.so
+          command: make test
 
       - run:
           name: Persist Artifacts

From 01fa83f5289f5249a812d321dbd59abf2eaf100f Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Wed, 31 Jul 2019 21:55:19 +0300
Subject: [PATCH 05/31] CircleCI: refactoring #2

---
 .circleci/config.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 3fdac959c..e8f2fc957 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -2,7 +2,7 @@ version: 2.1
 jobs:
   build:
     docker:
-      - image: circleci/buster
+      - image: circleci/redis:5.0.5-buster
 
     steps:
       - checkout
@@ -21,8 +21,8 @@ jobs:
             ./system-setup.py
             make deps
             bash get_deps.sh cpu
-            git clone git://github.com/antirez/redis.git --branch 5.0.3
-            (cd redis && make malloc=libc -j4 && sudo make install)
+            # git clone git://github.com/antirez/redis.git --branch 5.0.3
+            # (cd redis && make malloc=libc -j4 && sudo make install)
 
       - save_cache:
           paths:

From f019e7cc7e94e5388870818d383652e6c4adb9dc Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Wed, 31 Jul 2019 22:24:31 +0300
Subject: [PATCH 06/31] CircleCI: github ssh keys for git lfs

---
 .circleci/config.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index e8f2fc957..b30f42bf3 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -23,6 +23,9 @@ jobs:
             bash get_deps.sh cpu
             # git clone git://github.com/antirez/redis.git --branch 5.0.3
             # (cd redis && make malloc=libc -j4 && sudo make install)
+            mkdir -p ~/.ssh
+            ssh-keyscan -H github.com >> ~/.ssh/known_hosts
+            ssh git@github.com git-lfs-authenticate "${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}" download
 
       - save_cache:
           paths:

From d8a3f04f4df82a150ce603950a21b17d4c9b26e2 Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Wed, 31 Jul 2019 21:13:56 +0000
Subject: [PATCH 07/31] CircleCI: revert to old image

---
 .circleci/config.yml | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index b30f42bf3..6097f2767 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -2,7 +2,7 @@ version: 2.1
 jobs:
   build:
     docker:
-      - image: circleci/redis:5.0.5-buster
+      - image: circleci/python:3.6.8
 
     steps:
       - checkout
@@ -20,12 +20,8 @@ jobs:
             ./deps/readies/bin/getpy2
             ./system-setup.py
             make deps
-            bash get_deps.sh cpu
-            # git clone git://github.com/antirez/redis.git --branch 5.0.3
-            # (cd redis && make malloc=libc -j4 && sudo make install)
-            mkdir -p ~/.ssh
-            ssh-keyscan -H github.com >> ~/.ssh/known_hosts
-            ssh git@github.com git-lfs-authenticate "${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}" download
+            git clone git://github.com/antirez/redis.git --branch 5.0.3
+            (cd redis && make malloc=libc -j4 && sudo make install)
 
       - save_cache:
           paths:

From 9d8e77566e2ed569527b96b35d6b2b116c91702a Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Wed, 31 Jul 2019 21:18:47 +0000
Subject: [PATCH 08/31] CircleCI: disabled cache restore

---
 .circleci/config.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 6097f2767..d9efe7f65 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -8,11 +8,11 @@ jobs:
       - checkout
 
       # Download and cache dependencies
-      - restore_cache:
-          keys:
-          - v1-dependencies-{{ checksum "get_deps.sh" }}
-          # fallback to using the latest cache if no exact match is found
-          - v1-dependencies-
+      # - restore_cache:
+      #    keys:
+      #    - v1-dependencies-{{ checksum "get_deps.sh" }}
+      #    # fallback to using the latest cache if no exact match is found
+      #    - v1-dependencies-
 
       - run:
           name: install dependencies

From aa3f03a8e3831f1dcf31f08c7fbd5c3c1a430833 Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Wed, 31 Jul 2019 21:24:24 +0000
Subject: [PATCH 09/31] CircleCI: added sudo

---
 .circleci/config.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index d9efe7f65..3c03817ed 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -8,17 +8,17 @@ jobs:
       - checkout
 
       # Download and cache dependencies
-      # - restore_cache:
-      #    keys:
-      #    - v1-dependencies-{{ checksum "get_deps.sh" }}
-      #    # fallback to using the latest cache if no exact match is found
-      #    - v1-dependencies-
+      - restore_cache:
+          keys:
+          - v1-dependencies-{{ checksum "get_deps.sh" }}
+          # fallback to using the latest cache if no exact match is found
+          - v1-dependencies-
 
       - run:
           name: install dependencies
           command: |
-            ./deps/readies/bin/getpy2
-            ./system-setup.py
+            sudo ./deps/readies/bin/getpy2
+            sudo ./system-setup.py
             make deps
             git clone git://github.com/antirez/redis.git --branch 5.0.3
             (cd redis && make malloc=libc -j4 && sudo make install)

From 602d990ceba42ce152cd578d15f511279beb3d04 Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Thu, 1 Aug 2019 06:33:40 +0000
Subject: [PATCH 10/31] CircleCI: fixed git lfs handling

---
 Makefile        | 3 ++-
 system-setup.py | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index efa06faf1..bc8bf8d87 100755
--- a/Makefile
+++ b/Makefile
@@ -60,10 +60,11 @@ pack:
 	echo "Done."
 
 test:
+	@git lfs install
 	@git lfs pull
 	@set -e ;\
 	cd test ;\
 	python3 -m RLTest --test basic_tests.py --module $(BINDIR)/redisai.so \
 		--module-args "TF $(BINDIR)/backends/redisai_tensorflow/redisai_tensorflow.so ONNX $(BINDIR)/backends/redisai_onnxruntime/redisai_onnxruntime.so TORCH $(BINDIR)/backends/redisai_torch/redisai_torch.so"
 
-# LD_LIBRARY_PATH=$(PWD)/install
\ No newline at end of file
+# LD_LIBRARY_PATH=$(PWD)/install
diff --git a/system-setup.py b/system-setup.py
index bfe6effd1..afe86502f 100755
--- a/system-setup.py
+++ b/system-setup.py
@@ -25,7 +25,7 @@ def common_first(self):
     def debian_compat(self):
         self.run("curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash")
         self.install("git-lfs")
-        self.run("git lfs install")
+        # self.run("git lfs install")
 
         self.install("build-essential")
         # self.install("python-psutil")

From 7b367164513c9291919dd1e8e67e518ae9d4a85f Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Thu, 1 Aug 2019 06:40:06 +0000
Subject: [PATCH 11/31] CircleCI: try redis:5.0.5-buster image once again

---
 .circleci/config.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 3c03817ed..82ec1e7cd 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -2,7 +2,7 @@ version: 2.1
 jobs:
   build:
     docker:
-      - image: circleci/python:3.6.8
+      - image: circleci/redis:5.0.5-buster
 
     steps:
       - checkout
@@ -20,8 +20,8 @@ jobs:
             sudo ./deps/readies/bin/getpy2
             sudo ./system-setup.py
             make deps
-            git clone git://github.com/antirez/redis.git --branch 5.0.3
-            (cd redis && make malloc=libc -j4 && sudo make install)
+            # git clone git://github.com/antirez/redis.git --branch 5.0.3
+            # (cd redis && make malloc=libc -j4 && sudo make install)
 
       - save_cache:
           paths:

From bd95c700d7706e480f68f77013e9d5e3c1d3e300 Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Thu, 1 Aug 2019 06:44:54 +0000
Subject: [PATCH 12/31] CircleCI: so maybe python:3.7.4-buster

---
 .circleci/config.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 82ec1e7cd..0b2c528a5 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -2,7 +2,7 @@ version: 2.1
 jobs:
   build:
     docker:
-      - image: circleci/redis:5.0.5-buster
+      - image: circleci/python:3.7.4-buster
 
     steps:
       - checkout
@@ -20,8 +20,8 @@ jobs:
             sudo ./deps/readies/bin/getpy2
             sudo ./system-setup.py
             make deps
-            # git clone git://github.com/antirez/redis.git --branch 5.0.3
-            # (cd redis && make malloc=libc -j4 && sudo make install)
+            git clone git://github.com/antirez/redis.git --branch 5.0.3
+            (cd redis && make malloc=libc -j4 && sudo make install)
 
       - save_cache:
           paths:

From 1ad534c9a3751c7c02fd8c10c96de8bd4a7a125e Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Mon, 5 Aug 2019 13:18:01 +0300
Subject: [PATCH 13/31] Moved Paella to Python 3, bumped version to 0.3.2

---
 .venv/bin/activate           | 84 ++++++++++++++++++++++++++++++++++++
 Dockerfile                   | 13 ++++--
 Makefile                     | 16 ++++---
 deps/readies/bin/getredis5   |  1 -
 deps/readies/bin/platform    |  2 +-
 deps/readies/paella/setup.py |  7 ++-
 ramp.yml                     |  2 +-
 src/config.h                 |  2 +-
 system-setup.py              | 21 +++++----
 9 files changed, 121 insertions(+), 27 deletions(-)
 create mode 100644 .venv/bin/activate

diff --git a/.venv/bin/activate b/.venv/bin/activate
new file mode 100644
index 000000000..9dc0d3531
--- /dev/null
+++ b/.venv/bin/activate
@@ -0,0 +1,84 @@
+# This file must be used with "source bin/activate" *from bash*
+# you cannot run it directly
+
+
+if [ "${BASH_SOURCE-}" = "$0" ]; then
+    echo "You must source this script: \$ source $0" >&2
+    exit 33
+fi
+
+deactivate () {
+    unset -f pydoc >/dev/null 2>&1
+
+    # reset old environment variables
+    # ! [ -z ${VAR+_} ] returns true if VAR is declared at all
+    if ! [ -z "${_OLD_VIRTUAL_PATH:+_}" ] ; then
+        PATH="$_OLD_VIRTUAL_PATH"
+        export PATH
+        unset _OLD_VIRTUAL_PATH
+    fi
+    if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
+        PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
+        export PYTHONHOME
+        unset _OLD_VIRTUAL_PYTHONHOME
+    fi
+
+    # This should detect bash and zsh, which have a hash command that must
+    # be called to get it to forget past commands.  Without forgetting
+    # past commands the $PATH changes we made may not be respected
+    if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
+        hash -r 2>/dev/null
+    fi
+
+    if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
+        PS1="$_OLD_VIRTUAL_PS1"
+        export PS1
+        unset _OLD_VIRTUAL_PS1
+    fi
+
+    unset VIRTUAL_ENV
+    if [ ! "${1-}" = "nondestructive" ] ; then
+    # Self destruct!
+        unset -f deactivate
+    fi
+}
+
+# unset irrelevant variables
+deactivate nondestructive
+
+VIRTUAL_ENV="/w/rafi_1/RedisAI/.venv"
+export VIRTUAL_ENV
+
+_OLD_VIRTUAL_PATH="$PATH"
+PATH="$VIRTUAL_ENV/bin:$PATH"
+export PATH
+
+# unset PYTHONHOME if set
+if ! [ -z "${PYTHONHOME+_}" ] ; then
+    _OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
+    unset PYTHONHOME
+fi
+
+if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
+    _OLD_VIRTUAL_PS1="${PS1-}"
+    if [ "x(RedisAI) " != x ] ; then
+        PS1="(RedisAI) ${PS1-}"
+    else
+        PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
+    fi
+    export PS1
+fi
+
+# Make sure to unalias pydoc if it's already there
+alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
+
+pydoc () {
+    python -m pydoc "$@"
+}
+
+# This should detect bash and zsh, which have a hash command that must
+# be called to get it to forget past commands.  Without forgetting
+# past commands the $PATH changes we made may not be respected
+if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
+    hash -r 2>/dev/null
+fi
diff --git a/Dockerfile b/Dockerfile
index 6f4742c3c..a26bef0e0 100755
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,6 +1,11 @@
+ARG OS=debian:buster
+
 #----------------------------------------------------------------------------------------------
 FROM redis AS redis
-FROM debian:buster AS builder
+FROM ${OS} AS builder
+
+ARG PACK=0
+ARG TEST=0
 
 WORKDIR /redisai
 COPY --from=redis /usr/local/ /usr/local/
@@ -9,7 +14,7 @@ COPY ./deps/readies/ deps/readies/
 COPY ./system-setup.py .
 COPY ./test/test_requirements.txt test/
 
-RUN ./deps/readies/bin/getpy2
+RUN ./deps/readies/bin/getpy
 RUN ./system-setup.py
 
 COPY ./get_deps.sh .
@@ -17,8 +22,8 @@ RUN ./get_deps.sh cpu
 
 ADD ./ /redisai
 RUN make all
-# RUN make pack
-# RUN make test
+RUN [[ $PACK == 1 ]] && make pack
+RUN [[ $TEST == 1 ]] && make test
 
 #----------------------------------------------------------------------------------------------
 FROM redis
diff --git a/Makefile b/Makefile
index efa06faf1..309ef2f1a 100755
--- a/Makefile
+++ b/Makefile
@@ -17,7 +17,9 @@ endif
 
 BINDIR=$(PWD)/install
 
-.PHONY: all build clean deps pack test
+BACKENDS_PATH ?= $(BINDIR)/backends
+
+.PHONY: all build clean deps pack pack_ramp pack_deps test
 
 all: build
 
@@ -41,7 +43,9 @@ deps:
 	@echo Fetching dependencies...
 	@./get_deps.sh $(DEPS_FLAGS)
 
-pack:
+pack: pack_ramp pack_deps
+
+pack_ramp:
 	@[ ! -z `command -v redis-server` ] || { echo "Cannot find redis-server - aborting."; exit 1; }
 	@echo "Building RAMP file ..."
 	@set -e ;\
@@ -51,6 +55,8 @@ pack:
 	tail -1 $$RAMPOUT > $(BINDIR)/PACKAGE ;\
 	rm -f $RAMPOUT ;\
 	echo "Done."
+
+pack_deps: pack_ramp
 	@echo "Building dependencies file ..."
 	@set -e ;\
 	PACK_FNAME=$$(basename `cat $(BINDIR)/PACKAGE`) ;\
@@ -63,7 +69,5 @@ test:
 	@git lfs pull
 	@set -e ;\
 	cd test ;\
-	python3 -m RLTest --test basic_tests.py --module $(BINDIR)/redisai.so \
-		--module-args "TF $(BINDIR)/backends/redisai_tensorflow/redisai_tensorflow.so ONNX $(BINDIR)/backends/redisai_onnxruntime/redisai_onnxruntime.so TORCH $(BINDIR)/backends/redisai_torch/redisai_torch.so"
-
-# LD_LIBRARY_PATH=$(PWD)/install
\ No newline at end of file
+	python3 -m RLTest $(TEST_ARGS) --test basic_tests.py --module $(BINDIR)/redisai.so \
+		--module-args "TF $(BACKENDS_PATH)/redisai_tensorflow/redisai_tensorflow.so ONNX $(BACKENDS_PATH)/redisai_onnxruntime/redisai_onnxruntime.so TORCH $(BACKENDS_PATH)/redisai_torch/redisai_torch.so"
diff --git a/deps/readies/bin/getredis5 b/deps/readies/bin/getredis5
index 3a0cd973b..ce58ccb22 100755
--- a/deps/readies/bin/getredis5
+++ b/deps/readies/bin/getredis5
@@ -2,7 +2,6 @@
 
 import sys
 import os
-import popen2
 import argparse
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "deps/readies"))
diff --git a/deps/readies/bin/platform b/deps/readies/bin/platform
index b07a605b0..5a355337b 100755
--- a/deps/readies/bin/platform
+++ b/deps/readies/bin/platform
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import sys
 import os
diff --git a/deps/readies/paella/setup.py b/deps/readies/paella/setup.py
index 7fa976d14..daf92f37b 100755
--- a/deps/readies/paella/setup.py
+++ b/deps/readies/paella/setup.py
@@ -60,6 +60,8 @@ def __init__(self, nop=False):
             self.python = "python"
         elif self.has_command("python2"):
             self.python = "python2"
+        elif self.has_command("python3"):
+            self.python = "python3"
 
         if self.os == 'macosx':
             # this is required because osx pip installed are done with --user
@@ -146,8 +148,9 @@ def pip3_install(self, cmd):
     def setup_pip(self):
         get_pip = "set -e; cd /tmp; curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py"
         if not self.has_command("pip"):
-            self.install("curl")
-            self.run(get_pip + "; " + self.python + " get-pip.py", output_on_error=True)
+            self.install("python3-pip")
+            # self.install("curl")
+            # self.run(get_pip + "; " + self.python + " get-pip.py", output_on_error=True)
         ## fails on ubuntu 18:
         # if not has_command("pip3") and has_command("python3"):
         #     run(get_pip + "; python3 get-pip.py")
diff --git a/ramp.yml b/ramp.yml
index a31e3fb4f..895b34470 100644
--- a/ramp.yml
+++ b/ramp.yml
@@ -6,7 +6,7 @@ homepage: https://oss.redislabs.com/redisai/
 license: GNU Affero General Public License v3.0
 command_line_args: ""
 min_redis_version: "5.0"
-min_redis_pack_version: "5.5"
+min_redis_pack_version: "5.4"
 capabilities:
     - types
     - no_multi_key
diff --git a/src/config.h b/src/config.h
index 09551a5cd..d4a395813 100644
--- a/src/config.h
+++ b/src/config.h
@@ -13,7 +13,7 @@ typedef enum {
   RAI_DEVICE_GPU,
 } RAI_Device;
 
-#define RAI_ENC_VER 300
+#define RAI_ENC_VER 302
 
 //#define RAI_COPY_RUN_INPUT
 #define RAI_COPY_RUN_OUTPUT
diff --git a/system-setup.py b/system-setup.py
index bfe6effd1..4afcb9577 100755
--- a/system-setup.py
+++ b/system-setup.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 import sys
 import os
-import popen2
+from subprocess import Popen, PIPE
 import argparse
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "deps/readies"))
@@ -16,11 +16,10 @@ def __init__(self, nop=False):
 
     def common_first(self):
         self.setup_pip()
-        self.pip_install("wheel")
-        self.pip_install("setuptools --upgrade")
+        self.pip3_install("wheel")
+        self.pip3_install("setuptools --upgrade")
         
         self.install("git cmake ca-certificates curl unzip wget patchelf awscli")
-        self.install("python3 python3-pip python3-venv python3-psutil python3-networkx")
 
     def debian_compat(self):
         self.run("curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash")
@@ -28,22 +27,22 @@ def debian_compat(self):
         self.run("git lfs install")
 
         self.install("build-essential")
-        # self.install("python-psutil")
+        self.install("python3 python3-pip python3-venv python3-psutil python3-networkx")
 
     def redhat_compat(self):
         self.group_install("'Development Tools'")
         self.install("redhat-lsb-core")
 
-        # uninstall and install psutil (order is important), otherwise RLTest fails
-        self.run("pip uninstall -y psutil")
-        self.install("python2-psutil")
+        self.install("epel-release")
+        self.install("python36 python36-pip")
+        self.install("python36-psutil")
 
     def fedora(self):
         self.group_install("'Development Tools'")
 
     def macosx(self):
-        r, w, e = popen2.popen3('xcode-select -p')
-        if r.readlines() == []:
+        p = Popen('xcode-select -p', stdout=PIPE, close_fds=True, shell=True)
+        if p.stdout.readlines() == []:
             fatal("Xcode tools are not installed. Please run xcode-select --install.")
 
     def common_last(self):

From 34e788ae67e6b05f706dddc68c4942079393a075 Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Mon, 5 Aug 2019 13:24:16 +0300
Subject: [PATCH 14/31] Added getpy to readies/paella

---
 deps/readies/bin/getpy | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100755 deps/readies/bin/getpy

diff --git a/deps/readies/bin/getpy b/deps/readies/bin/getpy
new file mode 100755
index 000000000..e2129478e
--- /dev/null
+++ b/deps/readies/bin/getpy
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+show_if_error() {
+	{ "${@:1}"; } > /tmp/py.log  2>&1
+	[ $? != 0 ] && cat /tmp/py.log
+	rm -f /tmp/py.log
+}
+
+if [ ! -z $(command -v python) ]; then
+	[ "$(python --version 2>&1 | cut -d" " -f2 | cut -d. -f1)" = "3" ] && exit 0
+fi
+
+[ ! -z $(command -v python3) ] && exit 0
+
+if [ ! -z $(command -v apt-get) ]; then
+	show_if_error apt-get -qq update
+	show_if_error apt-get -qq install -y python3
+elif [ ! -z $(command -v dnf) ]; then
+	show_if_error dnf install -y python3
+elif [ ! -z $(command -v yum) ]; then
+	show_if_error yum install -y epel-release
+	show_if_error yum install -y python36
+elif [ ! -z $(command -v apk) ]; then
+	show_if_error apk update
+	show_if_error apk add python3
+elif [ ! -z $(command -v brew) ]; then
+	show_if_error brew install python3
+fi
+
+if [ -z $(command -v python3) ]; then
+	>&2 echo "Cannot install Python3. Aborting."
+	exit 1
+fi

From 34713c93ddbba5904fcd513f6253d7af4c68a911 Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Mon, 5 Aug 2019 13:28:34 +0300
Subject: [PATCH 15/31] Build fix #1

---
 .venv/bin/activate | 84 ----------------------------------------------
 1 file changed, 84 deletions(-)
 delete mode 100644 .venv/bin/activate

diff --git a/.venv/bin/activate b/.venv/bin/activate
deleted file mode 100644
index 9dc0d3531..000000000
--- a/.venv/bin/activate
+++ /dev/null
@@ -1,84 +0,0 @@
-# This file must be used with "source bin/activate" *from bash*
-# you cannot run it directly
-
-
-if [ "${BASH_SOURCE-}" = "$0" ]; then
-    echo "You must source this script: \$ source $0" >&2
-    exit 33
-fi
-
-deactivate () {
-    unset -f pydoc >/dev/null 2>&1
-
-    # reset old environment variables
-    # ! [ -z ${VAR+_} ] returns true if VAR is declared at all
-    if ! [ -z "${_OLD_VIRTUAL_PATH:+_}" ] ; then
-        PATH="$_OLD_VIRTUAL_PATH"
-        export PATH
-        unset _OLD_VIRTUAL_PATH
-    fi
-    if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
-        PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
-        export PYTHONHOME
-        unset _OLD_VIRTUAL_PYTHONHOME
-    fi
-
-    # This should detect bash and zsh, which have a hash command that must
-    # be called to get it to forget past commands.  Without forgetting
-    # past commands the $PATH changes we made may not be respected
-    if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
-        hash -r 2>/dev/null
-    fi
-
-    if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
-        PS1="$_OLD_VIRTUAL_PS1"
-        export PS1
-        unset _OLD_VIRTUAL_PS1
-    fi
-
-    unset VIRTUAL_ENV
-    if [ ! "${1-}" = "nondestructive" ] ; then
-    # Self destruct!
-        unset -f deactivate
-    fi
-}
-
-# unset irrelevant variables
-deactivate nondestructive
-
-VIRTUAL_ENV="/w/rafi_1/RedisAI/.venv"
-export VIRTUAL_ENV
-
-_OLD_VIRTUAL_PATH="$PATH"
-PATH="$VIRTUAL_ENV/bin:$PATH"
-export PATH
-
-# unset PYTHONHOME if set
-if ! [ -z "${PYTHONHOME+_}" ] ; then
-    _OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
-    unset PYTHONHOME
-fi
-
-if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
-    _OLD_VIRTUAL_PS1="${PS1-}"
-    if [ "x(RedisAI) " != x ] ; then
-        PS1="(RedisAI) ${PS1-}"
-    else
-        PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
-    fi
-    export PS1
-fi
-
-# Make sure to unalias pydoc if it's already there
-alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
-
-pydoc () {
-    python -m pydoc "$@"
-}
-
-# This should detect bash and zsh, which have a hash command that must
-# be called to get it to forget past commands.  Without forgetting
-# past commands the $PATH changes we made may not be respected
-if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
-    hash -r 2>/dev/null
-fi

From 9022159a26f0a00101d5b737905e62d8979ca3de Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Mon, 5 Aug 2019 19:22:30 +0300
Subject: [PATCH 16/31] Build fix #2

---
 .circleci/config.yml | 2 +-
 .dockerignore        | 2 ++
 .gitignore           | 2 ++
 Dockerfile           | 2 ++
 Makefile             | 8 ++++----
 get_deps.sh          | 4 ++--
 ramp.yml             | 5 +++++
 7 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 0b2c528a5..61ecc398e 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -17,7 +17,7 @@ jobs:
       - run:
           name: install dependencies
           command: |
-            sudo ./deps/readies/bin/getpy2
+            sudo ./deps/readies/bin/getpy
             sudo ./system-setup.py
             make deps
             git clone git://github.com/antirez/redis.git --branch 5.0.3
diff --git a/.dockerignore b/.dockerignore
index 8a0cea68e..1ac5c1b41 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -5,3 +5,5 @@
 /install/
 **/*.o
 **/*.so
+/.venv/
+/venv/
diff --git a/.gitignore b/.gitignore
index 6730af9f2..0e88c19cf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,6 +17,8 @@ examples/js/package-lock.json
 /test/venv/
 /test/logs/
 /1/
+/venv/
+/.venv/
 
 # Eclipse (if that's your jam...)
 .project
diff --git a/Dockerfile b/Dockerfile
index a26bef0e0..7b191a26e 100755
--- a/Dockerfile
+++ b/Dockerfile
@@ -7,6 +7,8 @@ FROM ${OS} AS builder
 ARG PACK=0
 ARG TEST=0
 
+SHELL ["/bin/bash", "-c"]
+
 WORKDIR /redisai
 COPY --from=redis /usr/local/ /usr/local/
 
diff --git a/Makefile b/Makefile
index cfbe483f9..ff8223673 100755
--- a/Makefile
+++ b/Makefile
@@ -34,12 +34,13 @@ endif
 
 clean:
 ifeq ($(ALL),1)
-	rm -rf build deps/install install
+	rm -rf build install deps/dlpack deps/install deps/*.tar.gz deps/*.zip deps/*.tgz
+	
 else
 	$(MAKE) -C build clean
 endif
 
-deps:
+deps fetch:
 	@echo Fetching dependencies...
 	@./get_deps.sh $(DEPS_FLAGS)
 
@@ -70,5 +71,4 @@ test:
 	@git lfs pull
 	@set -e ;\
 	cd test ;\
-	python3 -m RLTest $(TEST_ARGS) --test basic_tests.py --module $(BINDIR)/redisai.so \
-		--module-args "TF $(BACKENDS_PATH)/redisai_tensorflow/redisai_tensorflow.so ONNX $(BACKENDS_PATH)/redisai_onnxruntime/redisai_onnxruntime.so TORCH $(BACKENDS_PATH)/redisai_torch/redisai_torch.so"
+	python3 -m RLTest $(TEST_ARGS) --test basic_tests.py --module $(BINDIR)/redisai.so
diff --git a/get_deps.sh b/get_deps.sh
index 892986952..73fa7bb4d 100755
--- a/get_deps.sh
+++ b/get_deps.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 set -e
-set -x
+[[ $VERBOSE == 1 ]] && set -x
 
 BASE_DIRECTORY=`pwd`
 
@@ -101,7 +101,7 @@ if [ ! -e "${LIBTORCH_ARCHIVE}" ]; then
   curl -s -L ${LIBTORCH_URL} > ${LIBTORCH_ARCHIVE}
 fi
 
-unzip -o ${LIBTORCH_ARCHIVE}
+unzip -q -o ${LIBTORCH_ARCHIVE}
 tar cf - libtorch | tar xf -  --no-same-owner --strip-components=1 -C ${TORCH_PREFIX}
 rm -rf libtorch
 
diff --git a/ramp.yml b/ramp.yml
index 895b34470..f15c58b28 100644
--- a/ramp.yml
+++ b/ramp.yml
@@ -7,6 +7,11 @@ license: GNU Affero General Public License v3.0
 command_line_args: ""
 min_redis_version: "5.0"
 min_redis_pack_version: "5.4"
+os_list:
+    - ubuntu18.04
+    - rhel7
+    - ubuntu16.04
+    - ubuntu14.04
 capabilities:
     - types
     - no_multi_key

From 70afcd677ece66a3c256298e3f5aa49a49f3f5a0 Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Mon, 5 Aug 2019 20:17:22 +0300
Subject: [PATCH 17/31] Dockerfile sh syntax, DEBUG flag in Makefile

---
 Dockerfile | 6 ++----
 Makefile   | 6 +++++-
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 7b191a26e..46ff1f503 100755
--- a/Dockerfile
+++ b/Dockerfile
@@ -7,8 +7,6 @@ FROM ${OS} AS builder
 ARG PACK=0
 ARG TEST=0
 
-SHELL ["/bin/bash", "-c"]
-
 WORKDIR /redisai
 COPY --from=redis /usr/local/ /usr/local/
 
@@ -24,8 +22,8 @@ RUN ./get_deps.sh cpu
 
 ADD ./ /redisai
 RUN make all
-RUN [[ $PACK == 1 ]] && make pack
-RUN [[ $TEST == 1 ]] && make test
+RUN if [ "$PACK" = "1" ]; then make pack; fi
+RUN if [ "$TEST" = "1" ]; then make test; fi
 
 #----------------------------------------------------------------------------------------------
 FROM redis
diff --git a/Makefile b/Makefile
index ff8223673..0c1f8c0c0 100755
--- a/Makefile
+++ b/Makefile
@@ -23,11 +23,15 @@ BACKENDS_PATH ?= $(BINDIR)/backends
 
 all: build
 
+ifeq ($(DEBUG),1)
+CMAKE_FLAGS += -DCMAKE_BUILD_TYPE=Debug
+endif
+
 build:
 ifeq ($(wildcard build/.),)
 	mkdir -p build
 	cd build; \
-	cmake -DDEPS_PATH=../deps/install ..
+	cmake -DDEPS_PATH=../deps/install $(CMAKE_FLAGS) ..
 endif
 	$(MAKE) -C build
 	$(MAKE) -C build install

From ea8d2bccc2bc8f5807ef4308a4cf70f65947591a Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Wed, 7 Aug 2019 15:59:54 +0300
Subject: [PATCH 18/31] system-setup: added popen.communicate()

---
 system-setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/system-setup.py b/system-setup.py
index a47ccb84b..ac9676373 100755
--- a/system-setup.py
+++ b/system-setup.py
@@ -42,6 +42,7 @@ def fedora(self):
 
     def macosx(self):
         p = Popen('xcode-select -p', stdout=PIPE, close_fds=True, shell=True)
+        p.communicate()
         if p.stdout.readlines() == []:
             fatal("Xcode tools are not installed. Please run xcode-select --install.")
 

From 04e6cff55e51b4adcade247390ea82d5480292ff Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Sun, 11 Aug 2019 10:22:09 +0300
Subject: [PATCH 19/31] Build: better cpu/gpu separation

---
 CMakeLists.txt               |   4 +-
 Makefile                     |  31 +++--
 deps/readies/paella/setup.py |   6 +-
 get_deps.sh                  | 226 +++++++++++++++++++++--------------
 4 files changed, 161 insertions(+), 106 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 183b84481..976c53652 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -18,6 +18,8 @@ GET_FILENAME_COMPONENT(depsAbs
     "${DEPS_PATH}" REALPATH
     BASE_DIR "${CMAKE_BINARY_DIR}")
 
+# SET(CUDA_TOOLKIT_ROOT_DIR /usr/local/cuda-9.0)
+
 INCLUDE_DIRECTORIES(${depsAbs}/dlpack/include)
 INCLUDE_DIRECTORIES(${depsAbs}/libtensorflow/include)
 INCLUDE_DIRECTORIES(${depsAbs}/libtorch/include)
@@ -68,7 +70,7 @@ IF (APPLE)
         LINK_FLAGS "-undefined dynamic_lookup")
 ENDIF()
 
-SET(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/install)
+SET(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/install-${ENGINE})
 
 INSTALL(TARGETS redisai LIBRARY DESTINATION .)
 
diff --git a/Makefile b/Makefile
index 0c1f8c0c0..25f910f7c 100755
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,14 @@
-ifeq ($(CUDA),1)
-DEPS_FLAGS=
+override GPU:=$(or $(findstring $(CUDA),1),$(findstring $(GPU),1))
+
+ifeq ($(GPU),1)
+ifeq ($(CPU),1)
+$(error CPU=1 and GPU=1 (or CUDA=1) are conflicting)
+endif
+DEPS_FLAGS=gpu
+ENGINE=gpu
 else
 DEPS_FLAGS=cpu
+ENGINE=cpu
 endif
 
 export REDIS_ENT_LIB_PATH=/opt/redislabs/lib
@@ -15,11 +22,18 @@ else
 PACK_VER:=$(VERSION)
 endif
 
-BINDIR=$(PWD)/install
+BINDIR=$(PWD)/install-$(ENGINE)
 
 BACKENDS_PATH ?= $(BINDIR)/backends
 
-.PHONY: all build clean deps pack pack_ramp pack_deps test
+#----------------------------------------------------------------------------------------------
+
+setup:
+	@echo Setting up system...
+	$(SHOW)./deps/readies/bin/getpy
+	$(SHOW)./system-setup.py
+
+.PHONY: all setup build clean deps pack pack_ramp pack_deps test
 
 all: build
 
@@ -28,18 +42,17 @@ CMAKE_FLAGS += -DCMAKE_BUILD_TYPE=Debug
 endif
 
 build:
-ifeq ($(wildcard build/.),)
+ifeq ($(wildcard build/Makefile),)
 	mkdir -p build
 	cd build; \
-	cmake -DDEPS_PATH=../deps/install $(CMAKE_FLAGS) ..
+	cmake -DENGINE=$(ENGINE) -DDEPS_PATH=../deps/install-$(ENGINE) $(CMAKE_FLAGS) ..
 endif
 	$(MAKE) -C build
 	$(MAKE) -C build install
 
 clean:
 ifeq ($(ALL),1)
-	rm -rf build install deps/dlpack deps/install deps/*.tar.gz deps/*.zip deps/*.tgz
-	
+	rm -rf build install deps/dlpack deps/install-$(ENGINE) deps/*.tar.gz deps/*.zip deps/*.tgz
 else
 	$(MAKE) -C build clean
 endif
@@ -67,7 +80,7 @@ pack_deps: pack_ramp
 	PACK_FNAME=$$(basename `cat $(BINDIR)/PACKAGE`) ;\
 	ARCHOSVER=$$(echo "$$PACK_FNAME" | sed -e "s/^redisai\.\([^.]*\..*\)\.zip/\1/") ;\
 	cd install ;\
-	find backends -name "*.so*" | xargs tar pczf redisai-dependencies.$$ARCHOSVER.tgz ;\
+	find backends -name "*.so*" | xargs tar pczf redisai-dependencies.$$ARCHOSVER-$(ENGINE).tgz ;\
 	echo "Done."
 
 test:
diff --git a/deps/readies/paella/setup.py b/deps/readies/paella/setup.py
index daf92f37b..8c63a13d6 100755
--- a/deps/readies/paella/setup.py
+++ b/deps/readies/paella/setup.py
@@ -148,9 +148,9 @@ def pip3_install(self, cmd):
     def setup_pip(self):
         get_pip = "set -e; cd /tmp; curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py"
         if not self.has_command("pip"):
-            self.install("python3-pip")
-            # self.install("curl")
-            # self.run(get_pip + "; " + self.python + " get-pip.py", output_on_error=True)
+            # self.install("python3-pip")
+            self.install("curl ca-certificates")
+            self.run(get_pip + "; " + self.python + " get-pip.py", output_on_error=True)
         ## fails on ubuntu 18:
         # if not has_command("pip3") and has_command("python3"):
         #     run(get_pip + "; python3 get-pip.py")
diff --git a/get_deps.sh b/get_deps.sh
index 73fa7bb4d..520824fda 100755
--- a/get_deps.sh
+++ b/get_deps.sh
@@ -1,142 +1,182 @@
 #!/usr/bin/env bash
 
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+
 set -e
 [[ $VERBOSE == 1 ]] && set -x
 
-BASE_DIRECTORY=`pwd`
-
-# Allow different deps for different platforms:
-if [ -z "$DEPS_DIRECTORY" ]; then
-  DEPS_DIRECTORY=${BASE_DIRECTORY}/deps
+if [[ "$1" == "cpu" ]]; then
+	GPU=no
+	VARIANT=cpu
+elif [[ "$1" == "gpu" ]]; then
+	GPU=yes
+	VARIANT=gpu
+else
+	GPU=${GPU:-no}
+	if [[ $GPU == 1 ]]; then
+		VARIANT=gpu
+	else
+		VARIANT=cpu
+	fi
 fi
 
-mkdir -p ${DEPS_DIRECTORY}
-cd ${DEPS_DIRECTORY}
-DEPS_DIRECTORY=$PWD # -- to avoid relative/absolute confusion
+DEPS_DIR=$HERE/deps
+mkdir -p ${DEPS_DIR}
+cd ${DEPS_DIR}
 
-PREFIX=${DEPS_DIRECTORY}/install
+PREFIX=${DEPS_DIR}/install-$VARIANT
 mkdir -p ${PREFIX}
 
-DLPACK_PREFIX=${PREFIX}/dlpack
-rm -rf ${DLPACK_PREFIX}
-mkdir -p ${DLPACK_PREFIX}
+## DLPACK
 
-TF_PREFIX=${PREFIX}/libtensorflow
-rm -rf ${TF_PREFIX}
-mkdir -p ${TF_PREFIX}
+DLPACK_PREFIX=${PREFIX}/dlpack
 
-TORCH_PREFIX=${PREFIX}/libtorch
-rm -rf ${TORCH_PREFIX}
-mkdir -p ${TORCH_PREFIX}
+[[ $FORCE == 1 ]] && rm -rf ${DLPACK_PREFIX}
 
-ORT_PREFIX=${PREFIX}/onnxruntime
-rm -rf ${ORT_PREFIX}
-mkdir -p ${ORT_PREFIX}
-
-if [ ! -d dlpack ]; then
-    echo "Cloning dlpack"
+if [[ ! -d dlpack ]]; then
+    echo "Cloning dlpack ..."
     git clone --depth 1 https://github.com/dmlc/dlpack.git
+	echo "Done."
+else
+  echo "dlpack is in place."
 fi
 
-if [[ "$OSTYPE" == "linux-gnu" ]]; then
-  cp -d -r --no-preserve=ownership dlpack/include ${DLPACK_PREFIX}
-elif [[ "$OSTYPE" == "darwin"* ]]; then
-  cp -r dlpack/include ${DLPACK_PREFIX}
-fi
+[[ ! -d ${DLPACK_PREFIX}/include ]] && ln -s dlpack/include ${DLPACK_PREFIX}/include
 
 ## TENSORFLOW
+
 TF_VERSION="1.12.0"
-if [[ "$OSTYPE" == "linux-gnu" ]]; then
-  TF_OS="linux"
-  if [[ "$1" == "cpu" ]]; then
-    TF_BUILD="cpu"
-  else
-    TF_BUILD="gpu"
-  fi
-elif [[ "$OSTYPE" == "darwin"* ]]; then
-  TF_OS="darwin"
-  TF_BUILD="cpu"
-fi
+TF_PREFIX=${PREFIX}/libtensorflow
 
-LIBTF_ARCHIVE=libtensorflow-${TF_BUILD}-${TF_OS}-x86_64-${TF_VERSION}.tar.gz
+[[ $FORCE == 1 ]] && rm -rf ${TF_PREFIX}
 
-if [ ! -e ${LIBTF_ARCHIVE} ]; then
-  echo "Downloading libtensorflow ${TF_VERSION} ${TF_BUILD}"
-  wget -q https://storage.googleapis.com/tensorflow/libtensorflow/${LIBTF_ARCHIVE}
-fi
+if [[ ! -d ${TF_PREFIX} ]]; then
+  echo "Installing TensorFlow ..."
+
+  mkdir -p ${TF_PREFIX}
 
-tar xf ${LIBTF_ARCHIVE} --no-same-owner --strip-components=1 -C ${TF_PREFIX}
+  if [[ "$OSTYPE" == "linux-gnu" ]]; then
+    TF_OS="linux"
+    if [[ "$1" == "cpu" ]]; then
+      TF_BUILD="cpu"
+    else
+      TF_BUILD="gpu"
+    fi
+  elif [[ "$OSTYPE" == "darwin"* ]]; then
+    TF_OS="darwin"
+    TF_BUILD="cpu"
+  fi
+  
+  LIBTF_ARCHIVE=libtensorflow-${TF_BUILD}-${TF_OS}-x86_64-${TF_VERSION}.tar.gz
+  
+  if [ ! -e ${LIBTF_ARCHIVE} ]; then
+    echo "Downloading libtensorflow ${TF_VERSION} ${TF_BUILD}"
+    wget -q https://storage.googleapis.com/tensorflow/libtensorflow/${LIBTF_ARCHIVE}
+  fi
+  
+  tar xf ${LIBTF_ARCHIVE} --no-same-owner --strip-components=1 -C ${TF_PREFIX}
+  
+  echo "Done."
+else
+  echo "TensorFlow is in place."
+fi
 
 ## PYTORCH
 
 PT_VERSION="1.1.0"
 #PT_VERSION="latest"
 
-if [[ "$OSTYPE" == "linux-gnu" ]]; then
-  PT_OS="shared-with-deps"
-  if [[ "$1" == "cpu" ]]; then
-    PT_BUILD="cpu"
-  else
-    PT_BUILD="cu90"
-  fi
-elif [[ "$OSTYPE" == "darwin"* ]]; then
-  PT_OS="macos"
-  PT_BUILD="cpu"
-fi
-
-if [[ "$PT_VERSION" == "latest" ]]; then
-  PT_BUILD=nightly/${PT_BUILD}
-fi
-
-# Where to get the archive
-LIBTORCH_URL=https://download.pytorch.org/libtorch/${PT_BUILD}/libtorch-${PT_OS}-${PT_VERSION}.zip
-# Directory where torch is extracted to
-LIBTORCH_DIRECTORY=libtorch-${PT_OS}-${PT_VERSION}
+TORCH_PREFIX=${PREFIX}/libtorch
 
-# Archive - specifically named
-LIBTORCH_ARCHIVE=libtorch-${PT_OS}-${PT_BUILD}-${PT_VERSION}.zip
+[[ $FORCE == 1 ]] && rm -rf ${TORCH_PREFIX}
 
-if [ ! -e "${LIBTORCH_ARCHIVE}" ]; then
-  echo "Downloading libtorch ${PT_VERSION} ${PT_BUILD}"
-  curl -s -L ${LIBTORCH_URL} > ${LIBTORCH_ARCHIVE}
-fi
+if [[ ! -d ${TORCH_PREFIX} ]]; then
+  echo "Installing libtorch ..."
 
-unzip -q -o ${LIBTORCH_ARCHIVE}
-tar cf - libtorch | tar xf -  --no-same-owner --strip-components=1 -C ${TORCH_PREFIX}
-rm -rf libtorch
+  mkdir -p ${TORCH_PREFIX}
 
-if [[ "${PT_OS}" == "macos" ]]; then
-  # also download mkl
-  MKL_BUNDLE=mklml_mac_2019.0.3.20190220
-  if [ ! -e "${MKL_BUNDLE}.tgz" ]; then
-    wget -q "https://github.com/intel/mkl-dnn/releases/download/v0.18/${MKL_BUNDLE}.tgz"
+  if [[ "$OSTYPE" == "linux-gnu" ]]; then
+    PT_OS="shared-with-deps"
+    if [[ "$1" == "cpu" ]]; then
+      PT_BUILD="cpu"
+    else
+      PT_BUILD="cu90"
+    fi
+  elif [[ "$OSTYPE" == "darwin"* ]]; then
+    PT_OS="macos"
+    PT_BUILD="cpu"
+  fi
+  
+  if [[ "$PT_VERSION" == "latest" ]]; then
+    PT_BUILD=nightly/${PT_BUILD}
+  fi
+  
+  # Where to get the archive
+  LIBTORCH_URL=https://download.pytorch.org/libtorch/${PT_BUILD}/libtorch-${PT_OS}-${PT_VERSION}.zip
+  # Directory where torch is extracted to
+  LIBTORCH_DIRECTORY=libtorch-${PT_OS}-${PT_VERSION}
+  
+  # Archive - specifically named
+  LIBTORCH_ARCHIVE=libtorch-${PT_OS}-${PT_BUILD}-${PT_VERSION}.zip
+  
+  if [ ! -e "${LIBTORCH_ARCHIVE}" ]; then
+    echo "Downloading libtorch ${PT_VERSION} ${PT_BUILD}"
+    # curl -s -L ${LIBTORCH_URL} > ${LIBTORCH_ARCHIVE}
   fi
-  tar xf ${MKL_BUNDLE}.tgz --no-same-owner --strip-components=1 -C ${TORCH_PREFIX}
-  tar xf ${MKL_BUNDLE}.tgz --no-same-owner --strip-components=1 -C ${ORT_PREFIX}
+  
+  unzip -q -o ${LIBTORCH_ARCHIVE} -d ${TORCH_PREFIX}/..
+  
+  echo "Done."
+  
+  if [[ "${PT_OS}" == "macos" ]]; then
+    # also download mkl
+    MKL_BUNDLE=mklml_mac_2019.0.3.20190220
+    if [ ! -e "${MKL_BUNDLE}.tgz" ]; then
+      wget -q "https://github.com/intel/mkl-dnn/releases/download/v0.18/${MKL_BUNDLE}.tgz"
+    fi
+    tar xf ${MKL_BUNDLE}.tgz --no-same-owner --strip-components=1 -C ${TORCH_PREFIX}
+    tar xf ${MKL_BUNDLE}.tgz --no-same-owner --strip-components=1 -C ${ORT_PREFIX}
+  fi
+else
+  echo "librotch is in place."
 fi
 
 ## ONNXRUNTIME
+
 ORT_VERSION="0.4.0"
+ORT_ARCH="x64"
 if [[ "$OSTYPE" == "linux-gnu" ]]; then
   if [[ "$1" == "cpu" ]]; then
-    ORT_OS="linux-x64"
-    ORT_BUILD="cpu"
+    ORT_OS="linux"
+    ORT_BUILD=""
   else
-    ORT_OS="linux-x64-gpu"
-    ORT_BUILD="gpu"
+    ORT_OS="linux"
+    ORT_BUILD="-gpu"
   fi
 elif [[ "$OSTYPE" == "darwin"* ]]; then
   ORT_OS="osx-x64"
   ORT_BUILD=""
 fi
 
-ORT_ARCHIVE=onnxruntime-${ORT_OS}-${ORT_VERSION}.tgz
+ORT_PREFIX=${PREFIX}/onnxruntime
+
+[[ $FORCE == 1 ]] && rm -rf ${ORT_PREFIX}
 
-if [ ! -e ${ORT_ARCHIVE} ]; then
-  echo "Downloading ONNXRuntime ${ORT_VERSION} ${ORT_BUILD}"
-  wget -q https://github.com/Microsoft/onnxruntime/releases/download/v${ORT_VERSION}/${ORT_ARCHIVE}
-fi
+ORT_ARCHIVE=onnxruntime-${ORT_OS}-${ORT_ARCH}${ORT_BUILD}-${ORT_VERSION}.tgz
+
+if [[ ! -d ${ORT_PREFIX} ]]; then
+  echo "Installing ONNXRuntime ..."
 
-tar xf ${ORT_ARCHIVE} --no-same-owner --strip-components=1 -C ${ORT_PREFIX}
+  mkdir -p ${ORT_PREFIX}
+
+  if [ ! -e ${ORT_ARCHIVE} ]; then
+    echo "Downloading ONNXRuntime ${ORT_VERSION} ${VARIANT}"
+    wget -q https://github.com/Microsoft/onnxruntime/releases/download/v${ORT_VERSION}/${ORT_ARCHIVE}
+  fi
+  
+  tar xf ${ORT_ARCHIVE} --no-same-owner --strip-components=1 -C ${ORT_PREFIX}
+else
+  echo "onnxruntime is in place."
+fi
 
-echo "Done"
+echo "Done."

From 521bcd2c28004d58f7a78da5629dc14e8f3c6073 Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Sun, 11 Aug 2019 11:40:13 +0300
Subject: [PATCH 20/31] Docker fixes

---
 Dockerfile                   | 2 +-
 deps/readies/paella/setup.py | 2 +-
 get_deps.sh                  | 9 ++++++---
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 46ff1f503..9d9228f53 100755
--- a/Dockerfile
+++ b/Dockerfile
@@ -32,7 +32,7 @@ RUN set -e; apt-get -qq update; apt-get install -y libgomp1
 
 RUN mkdir -p /usr/lib/redis/modules/
 
-COPY --from=builder /redisai/install/ /usr/lib/redis/modules/
+COPY --from=builder /redisai/install-cpu/ /usr/lib/redis/modules/
 
 WORKDIR /data
 EXPOSE 6379
diff --git a/deps/readies/paella/setup.py b/deps/readies/paella/setup.py
index 8c63a13d6..351e02ac1 100755
--- a/deps/readies/paella/setup.py
+++ b/deps/readies/paella/setup.py
@@ -148,7 +148,7 @@ def pip3_install(self, cmd):
     def setup_pip(self):
         get_pip = "set -e; cd /tmp; curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py"
         if not self.has_command("pip"):
-            # self.install("python3-pip")
+            self.install("python3-distutils")
             self.install("curl ca-certificates")
             self.run(get_pip + "; " + self.python + " get-pip.py", output_on_error=True)
         ## fails on ubuntu 18:
diff --git a/get_deps.sh b/get_deps.sh
index 520824fda..80835d53a 100755
--- a/get_deps.sh
+++ b/get_deps.sh
@@ -41,7 +41,10 @@ else
   echo "dlpack is in place."
 fi
 
-[[ ! -d ${DLPACK_PREFIX}/include ]] && ln -s dlpack/include ${DLPACK_PREFIX}/include
+if [[ ! -d ${DLPACK_PREFIX}/include ]]; then
+	mkdir -p ${DLPACK_PREFIX}
+	ln -sf ${DEPS_DIR}/dlpack/include ${DLPACK_PREFIX}/include
+fi
 
 ## TENSORFLOW
 
@@ -121,10 +124,10 @@ if [[ ! -d ${TORCH_PREFIX} ]]; then
   
   if [ ! -e "${LIBTORCH_ARCHIVE}" ]; then
     echo "Downloading libtorch ${PT_VERSION} ${PT_BUILD}"
-    # curl -s -L ${LIBTORCH_URL} > ${LIBTORCH_ARCHIVE}
+    curl -s -L ${LIBTORCH_URL} > ${LIBTORCH_ARCHIVE}
   fi
   
-  unzip -q -o ${LIBTORCH_ARCHIVE} -d ${TORCH_PREFIX}/..
+  unzip -q -o ${LIBTORCH_ARCHIVE} -d ${TORCH_PREFIX}/../
   
   echo "Done."
   

From 3e1f7576fcdf7473b743ecaff938103976e4a93d Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Sun, 11 Aug 2019 11:49:58 +0300
Subject: [PATCH 21/31] Build fix #3

---
 Makefile        | 4 ++--
 system-setup.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index 25f910f7c..d337a43ed 100755
--- a/Makefile
+++ b/Makefile
@@ -68,7 +68,7 @@ pack_ramp:
 	@echo "Building RAMP file ..."
 	@set -e ;\
 	RAMPOUT=$$(mktemp /tmp/ramp.XXXXXX) ;\
-	LD_LIBRARY_PATH=$(PWD)/deps/install/lib \
+	LD_LIBRARY_PATH=$(PWD)/deps/install-$(ENGINE)/lib \
 	ramp pack -m $(PWD)/ramp.yml -o "build/redisai.{os}-{architecture}.{semantic_version}.zip" $(BINDIR)/redisai.so 2> /dev/null | grep '.zip' > $$RAMPOUT ;\
 	tail -1 $$RAMPOUT > $(BINDIR)/PACKAGE ;\
 	rm -f $RAMPOUT ;\
@@ -79,7 +79,7 @@ pack_deps: pack_ramp
 	@set -e ;\
 	PACK_FNAME=$$(basename `cat $(BINDIR)/PACKAGE`) ;\
 	ARCHOSVER=$$(echo "$$PACK_FNAME" | sed -e "s/^redisai\.\([^.]*\..*\)\.zip/\1/") ;\
-	cd install ;\
+	cd install-$(ENGINE) ;\
 	find backends -name "*.so*" | xargs tar pczf redisai-dependencies.$$ARCHOSVER-$(ENGINE).tgz ;\
 	echo "Done."
 
diff --git a/system-setup.py b/system-setup.py
index ac9676373..aa1ca2337 100755
--- a/system-setup.py
+++ b/system-setup.py
@@ -27,7 +27,7 @@ def debian_compat(self):
         # self.run("git lfs install")
 
         self.install("build-essential")
-        self.install("python3 python3-pip python3-venv python3-psutil python3-networkx")
+        self.install("python3-venv python3-psutil python3-networkx")
 
     def redhat_compat(self):
         self.group_install("'Development Tools'")

From 0f8d2eecd5adcc2435bed5ff6eaaf40151e3b802 Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Sun, 11 Aug 2019 11:57:01 +0300
Subject: [PATCH 22/31] Build fix #4

---
 .circleci/config.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 61ecc398e..03cd6de9f 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -42,9 +42,8 @@ jobs:
           name: Persist Artifacts
           command: |
             mkdir -p ~/workspace/build
-            cp install/$MODULE_ARTIFACT ~/workspace/
-            # cp deps/install/lib/*.so*  ~/workspace/
-            cp -r install/backends ~/workspace/
+            cp install-cpu/$MODULE_ARTIFACT ~/workspace/
+            cp -r install-cpu/backends ~/workspace/
             cp ramp.yml ~/workspace/
       - persist_to_workspace:
           root: ~/workspace

From 923bb7614eab0f1e72d827dc9b8a206fad915c03 Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Thu, 15 Aug 2019 15:43:52 +0300
Subject: [PATCH 23/31] Fixes from review, updated README

---
 .dockerignore                |  2 +-
 .gitignore                   |  2 +-
 CMakeLists.txt               |  2 +-
 Dockerfile                   | 12 ++++-----
 Makefile                     | 50 ++++++++++++++++++++++++------------
 README.md                    | 31 +++++++++++++++-------
 deps/readies/bin/getpy       | 28 +++++++++++---------
 deps/readies/bin/getpy2      |  2 ++
 deps/readies/paella/setup.py |  7 ++++-
 get_deps.sh                  | 12 ++++-----
 ramp.yml                     |  5 ----
 11 files changed, 93 insertions(+), 60 deletions(-)

diff --git a/.dockerignore b/.dockerignore
index 1ac5c1b41..2905b0714 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -2,7 +2,7 @@
 !/deps/readies/
 !/deps/*.py
 /build/
-/install/
+/install*/
 **/*.o
 **/*.so
 /.venv/
diff --git a/.gitignore b/.gitignore
index 0e88c19cf..55ee895f2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,7 +7,7 @@ __pycache__
 *.pyc
 
 # Dirs
-/install/
+/install*/
 /deps/*
 !/deps/readies/
 !/deps/*.py
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 976c53652..388177454 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -70,7 +70,7 @@ IF (APPLE)
         LINK_FLAGS "-undefined dynamic_lookup")
 ENDIF()
 
-SET(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/install-${ENGINE})
+SET(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/install-${DEVICE})
 
 INSTALL(TARGETS redisai LIBRARY DESTINATION .)
 
diff --git a/Dockerfile b/Dockerfile
index 9d9228f53..dc13ca8dc 100755
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,9 +4,6 @@ ARG OS=debian:buster
 FROM redis AS redis
 FROM ${OS} AS builder
 
-ARG PACK=0
-ARG TEST=0
-
 WORKDIR /redisai
 COPY --from=redis /usr/local/ /usr/local/
 
@@ -22,6 +19,10 @@ RUN ./get_deps.sh cpu
 
 ADD ./ /redisai
 RUN make all
+
+ARG PACK=0
+ARG TEST=0
+
 RUN if [ "$PACK" = "1" ]; then make pack; fi
 RUN if [ "$TEST" = "1" ]; then make test; fi
 
@@ -36,7 +37,4 @@ COPY --from=builder /redisai/install-cpu/ /usr/lib/redis/modules/
 
 WORKDIR /data
 EXPOSE 6379
-CMD ["--loadmodule", "/usr/lib/redis/modules/redisai.so", \
-        "TF", "/usr/lib/redis/modules//backends/redisai_tensorflow/redisai_tensorflow.so", \
-        "TORCH", "/usr/lib/redis/modules//backends/redisai_torch/redisai_torch.so", \
-        "ONNX", "/usr/lib/redis/modules//backends/redisai_onnxruntime/redisai_onnxruntime.so"]
+CMD ["--loadmodule", "/usr/lib/redis/modules/redisai.so"]
diff --git a/Makefile b/Makefile
index d337a43ed..c42ef73b0 100755
--- a/Makefile
+++ b/Makefile
@@ -5,10 +5,10 @@ ifeq ($(CPU),1)
 $(error CPU=1 and GPU=1 (or CUDA=1) are conflicting)
 endif
 DEPS_FLAGS=gpu
-ENGINE=gpu
+DEVICE=gpu
 else
 DEPS_FLAGS=cpu
-ENGINE=cpu
+DEVICE=cpu
 endif
 
 export REDIS_ENT_LIB_PATH=/opt/redislabs/lib
@@ -22,21 +22,27 @@ else
 PACK_VER:=$(VERSION)
 endif
 
-BINDIR=$(PWD)/install-$(ENGINE)
+BINDIR=$(PWD)/build
+INSTALL_DIR=$(PWD)/install-$(DEVICE)
 
-BACKENDS_PATH ?= $(BINDIR)/backends
+BACKENDS_PATH ?= $(INSTALL_DIR)/backends
 
-#----------------------------------------------------------------------------------------------
+RAMP:=ramp
 
-setup:
-	@echo Setting up system...
-	$(SHOW)./deps/readies/bin/getpy
-	$(SHOW)./system-setup.py
+#----------------------------------------------------------------------------------------------
 
 .PHONY: all setup build clean deps pack pack_ramp pack_deps test
 
 all: build
 
+setup:
+	@echo Setting up system...
+	./deps/readies/bin/getpy
+	./system-setup.py
+
+show-setup:
+	@./system-setup.py --nop
+
 ifeq ($(DEBUG),1)
 CMAKE_FLAGS += -DCMAKE_BUILD_TYPE=Debug
 endif
@@ -45,14 +51,15 @@ build:
 ifeq ($(wildcard build/Makefile),)
 	mkdir -p build
 	cd build; \
-	cmake -DENGINE=$(ENGINE) -DDEPS_PATH=../deps/install-$(ENGINE) $(CMAKE_FLAGS) ..
+	cmake -DDEVICE=$(DEVICE) -DDEPS_PATH=../deps/install-$(DEVICE) $(CMAKE_FLAGS) ..
 endif
 	$(MAKE) -C build
 	$(MAKE) -C build install
+	ln -sf install-$(DEVICE) install
 
 clean:
 ifeq ($(ALL),1)
-	rm -rf build install deps/dlpack deps/install-$(ENGINE) deps/*.tar.gz deps/*.zip deps/*.tgz
+	rm -rf build install deps/dlpack deps/install-$(DEVICE) deps/*.tar.gz deps/*.zip deps/*.tgz
 else
 	$(MAKE) -C build clean
 endif
@@ -68,10 +75,14 @@ pack_ramp:
 	@echo "Building RAMP file ..."
 	@set -e ;\
 	RAMPOUT=$$(mktemp /tmp/ramp.XXXXXX) ;\
-	LD_LIBRARY_PATH=$(PWD)/deps/install-$(ENGINE)/lib \
-	ramp pack -m $(PWD)/ramp.yml -o "build/redisai.{os}-{architecture}.{semantic_version}.zip" $(BINDIR)/redisai.so 2> /dev/null | grep '.zip' > $$RAMPOUT ;\
+	LD_LIBRARY_PATH=$(PWD)/deps/install-$(DEVICE)/lib \
+	$(RAMP) pack -m $(PWD)/ramp.yml -o "build/redisai.{os}-{architecture}.{semantic_version}.zip" $(INSTALL_DIR)/redisai.so 2> /dev/null | grep '.zip' > $$RAMPOUT ;\
 	tail -1 $$RAMPOUT > $(BINDIR)/PACKAGE ;\
-	rm -f $RAMPOUT ;\
+	cat $(BINDIR)/PACKAGE | sed -e "s/[^.]*\.[^.]*\.\(.*\)\.zip/\1/" > $(BINDIR)/VERSION ;\
+	VERSION=$$(cat $(BINDIR)/VERSION) ;\
+	$(RAMP) pack -m $(PWD)/ramp.yml -o "build/redisai.{os}-{architecture}.{semantic_version}.zip" \
+		-c "BACKENDSPATH /opt/redislabs/lib/redisai-cpu-$$VERSION/backends" $(INSTALL_DIR)/redisai.so 2> /dev/null | grep '.zip' > $$RAMPOUT ;\
+	rm -f $$RAMPOUT ;\
 	echo "Done."
 
 pack_deps: pack_ramp
@@ -79,8 +90,13 @@ pack_deps: pack_ramp
 	@set -e ;\
 	PACK_FNAME=$$(basename `cat $(BINDIR)/PACKAGE`) ;\
 	ARCHOSVER=$$(echo "$$PACK_FNAME" | sed -e "s/^redisai\.\([^.]*\..*\)\.zip/\1/") ;\
-	cd install-$(ENGINE) ;\
-	find backends -name "*.so*" | xargs tar pczf redisai-dependencies.$$ARCHOSVER-$(ENGINE).tgz ;\
+	VERSION=$$(cat $(BINDIR)/VERSION) ;\
+	cd install-$(DEVICE) ;\
+	rm -rf redisai-$(DEVICE)-$$VERSION ;\
+	mkdir redisai-$(DEVICE)-$$VERSION ;\
+	mv backends redisai-$(DEVICE)-$$VERSION ;\
+	ln -s redisai-$(DEVICE)-$$VERSION/backends backends ;\
+	find redisai-$(DEVICE)-$$VERSION -name "*.so*" | xargs tar pczf redisai-$(DEVICE)-dependencies.$$ARCHOSVER.tgz ;\
 	echo "Done."
 
 test:
@@ -88,4 +104,4 @@ test:
 	@git lfs pull
 	@set -e ;\
 	cd test ;\
-	python3 -m RLTest $(TEST_ARGS) --test basic_tests.py --module $(BINDIR)/redisai.so
+	python3 -m RLTest $(TEST_ARGS) --test basic_tests.py --module $(INSTALL_DIR)/redisai.so
diff --git a/README.md b/README.md
index 3cc79724d..aca9eb0e3 100644
--- a/README.md
+++ b/README.md
@@ -28,7 +28,7 @@ docker run -p 6379:6379 -it --rm redisai/redisai
 
 ### Give it a try
 
-On the client, load a backend (TF, TORCH or ONNX), and set the model
+On the client, load a backend (TF, TORCH, or ONNX), and set the model
 ```sh
 redis-cli AI.CONFIG LOADBACKEND TF install/backends/redisai_tensorflow/redisai_tensorflow.so
 redis-cli -x AI.MODELSET foo TF CPU INPUTS a b OUTPUTS c < test/test_data/graph.pb
@@ -50,25 +50,38 @@ redis-cli
 ```
 
 ## Building
-This will checkout and build and download the libraries for the backends (TensorFlow, PyTorch, ONNXRuntime) for your platform. Note that this requires CUDA to be installed.
+In order to build RedisAI, some artifacts need to be installed, notably Python 3. Those can be installed using:
+
+```
+make setup
+```
+
+To examine dependencies in detail, prior to installing them, it is possible to invoke:
+
+```
+./deps/readies/bin/getpy
+make show-setup
+```
+
+(The first command will install Python 3, and the second one will show list of installation commands for the rest of the artifacts.)
+
+----
+
+The following will checkout and build and download the libraries for the backends (TensorFlow, PyTorch, ONNXRuntime) for your platform. Note that this requires CUDA to be installed.
 
 ```sh
-bash get_deps.sh
+make deps GPU=1
 ```
 Alternatively, run the following to only fetch the CPU-only backends.
 ```sh
-bash get_deps.sh cpu
+make deps CPU=1
 ```
 
 Once the dependencies are downloaded, build the module itself. Note that
 CMake 3.0 or higher is required.
 
 ```sh
-mkdir build
-cd build
-cmake -DDEPS_PATH=../deps/install ..
-make && make install
-cd ..
+make build
 ```
 
 Note: in order to use the PyTorch backend on Linux, at least `gcc 4.9.2` is required.
diff --git a/deps/readies/bin/getpy b/deps/readies/bin/getpy
index e2129478e..edf875647 100755
--- a/deps/readies/bin/getpy
+++ b/deps/readies/bin/getpy
@@ -1,9 +1,13 @@
 #!/bin/bash
 
-show_if_error() {
-	{ "${@:1}"; } > /tmp/py.log  2>&1
-	[ $? != 0 ] && cat /tmp/py.log
-	rm -f /tmp/py.log
+[ "$VERBOSE" = "1" ] && set -x
+
+runn() {
+	[[ $NOP == 1 ]] && { echo "${@:1}"; return; }
+	__runn_log=$(mktemp /tmp/run.XXXXX)
+	{ "${@:1}"; } > $__runn_log  2>&1
+	[ $? != 0 ] && cat $__runn_log
+	rm -f $__runn_log
 }
 
 if [ ! -z $(command -v python) ]; then
@@ -13,18 +17,18 @@ fi
 [ ! -z $(command -v python3) ] && exit 0
 
 if [ ! -z $(command -v apt-get) ]; then
-	show_if_error apt-get -qq update
-	show_if_error apt-get -qq install -y python3
+	runn apt-get -qq update
+	runn apt-get -qq install -y python3
 elif [ ! -z $(command -v dnf) ]; then
-	show_if_error dnf install -y python3
+	runn dnf install -y python3
 elif [ ! -z $(command -v yum) ]; then
-	show_if_error yum install -y epel-release
-	show_if_error yum install -y python36
+	runn yum install -y epel-release
+	runn yum install -y python36
 elif [ ! -z $(command -v apk) ]; then
-	show_if_error apk update
-	show_if_error apk add python3
+	runn apk update
+	runn apk add python3
 elif [ ! -z $(command -v brew) ]; then
-	show_if_error brew install python3
+	runn brew install python3
 fi
 
 if [ -z $(command -v python3) ]; then
diff --git a/deps/readies/bin/getpy2 b/deps/readies/bin/getpy2
index aa1908e64..f957042ca 100755
--- a/deps/readies/bin/getpy2
+++ b/deps/readies/bin/getpy2
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+[ "$VERBOSE" = "1" ] && set -x
+
 show_if_error() {
 	{ "${@:1}"; } > /tmp/py2.log  2>&1
 	[ $? != 0 ] && cat /tmp/py2.log
diff --git a/deps/readies/paella/setup.py b/deps/readies/paella/setup.py
index 351e02ac1..643042c11 100755
--- a/deps/readies/paella/setup.py
+++ b/deps/readies/paella/setup.py
@@ -44,6 +44,9 @@ def redhat_compat(self):
     def debian_compat(self):
         self.runner.run("apt-get -qq update -y")
 
+    def macosx(self):
+        self.runner.run("brew update || true")
+
 #----------------------------------------------------------------------------------------------
 
 class Setup(OnPlatform):
@@ -107,7 +110,9 @@ def pacman_install(self, packs, group=False):
         self.run("pacman --noconfirm -S " + packs, output_on_error=True)
 
     def brew_install(self, packs, group=False):
-        self.run('brew install ' + packs, output_on_error=True)
+        # brew will fail if package is already installed
+        for pack in packs.split():
+            self.run("brew list {} &>/dev/null || brew install {}".format(pack, pack), output_on_error=True)
 
     def install(self, packs, group=False):
         if self.os == 'linux':
diff --git a/get_deps.sh b/get_deps.sh
index 80835d53a..3f799e7ae 100755
--- a/get_deps.sh
+++ b/get_deps.sh
@@ -7,16 +7,16 @@ set -e
 
 if [[ "$1" == "cpu" ]]; then
 	GPU=no
-	VARIANT=cpu
+	DEVICE=cpu
 elif [[ "$1" == "gpu" ]]; then
 	GPU=yes
-	VARIANT=gpu
+	DEVICE=gpu
 else
 	GPU=${GPU:-no}
 	if [[ $GPU == 1 ]]; then
-		VARIANT=gpu
+		DEVICE=gpu
 	else
-		VARIANT=cpu
+		DEVICE=cpu
 	fi
 fi
 
@@ -24,7 +24,7 @@ DEPS_DIR=$HERE/deps
 mkdir -p ${DEPS_DIR}
 cd ${DEPS_DIR}
 
-PREFIX=${DEPS_DIR}/install-$VARIANT
+PREFIX=${DEPS_DIR}/install-$DEVICE
 mkdir -p ${PREFIX}
 
 ## DLPACK
@@ -173,7 +173,7 @@ if [[ ! -d ${ORT_PREFIX} ]]; then
   mkdir -p ${ORT_PREFIX}
 
   if [ ! -e ${ORT_ARCHIVE} ]; then
-    echo "Downloading ONNXRuntime ${ORT_VERSION} ${VARIANT}"
+    echo "Downloading ONNXRuntime ${ORT_VERSION} ${DEVICE}"
     wget -q https://github.com/Microsoft/onnxruntime/releases/download/v${ORT_VERSION}/${ORT_ARCHIVE}
   fi
   
diff --git a/ramp.yml b/ramp.yml
index f15c58b28..895b34470 100644
--- a/ramp.yml
+++ b/ramp.yml
@@ -7,11 +7,6 @@ license: GNU Affero General Public License v3.0
 command_line_args: ""
 min_redis_version: "5.0"
 min_redis_pack_version: "5.4"
-os_list:
-    - ubuntu18.04
-    - rhel7
-    - ubuntu16.04
-    - ubuntu14.04
 capabilities:
     - types
     - no_multi_key

From a2fc02d6caa84543e50cfca725835e35ba9e3ab2 Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Fri, 16 Aug 2019 14:33:15 +0300
Subject: [PATCH 24/31] Fixes from review #2

- Restored README.md
- Moved Makefile and system-setup.py to automation/
- Removed setup and show-setup from Makefile
- Removed LD_LIBRARY_PATH from Makefile
- Moved deps/readies into automation/
- Will provice automation/README.md in a subsequent PR
- Added default value to DEVICE in CMakeLists.txt
---
 .circleci/config.yml                          | 10 ++--
 .dockerignore                                 |  2 -
 .gitignore                                    |  2 -
 CMakeLists.txt                                |  4 ++
 Dockerfile                                    | 13 +++---
 README.md                                     | 31 ++++---------
 Makefile => automation/Makefile               | 46 +++++++++----------
 {deps => automation}/readies/.gitignore       |  0
 {deps => automation}/readies/LICENSE          |  0
 {deps => automation}/readies/README.md        |  0
 {deps => automation}/readies/bin/getpy        |  0
 {deps => automation}/readies/bin/getpy2       |  0
 {deps => automation}/readies/bin/getredis5    |  2 +-
 {deps => automation}/readies/bin/platform     |  0
 automation/readies/bin/python                 |  7 +++
 {deps => automation}/readies/bin/python2      |  0
 .../readies/cetara/diag/gdb.c                 |  0
 .../readies/cetara/diag/gdb.h                 |  0
 {deps => automation}/readies/mk/bindirs.defs  |  0
 {deps => automation}/readies/mk/bindirs.rules |  0
 {deps => automation}/readies/mk/common.defs   |  0
 {deps => automation}/readies/mk/common.rules  |  0
 .../readies/mk/configure.defs                 |  0
 .../readies/mk/configure.rules                |  0
 {deps => automation}/readies/mk/defs          |  2 +-
 {deps => automation}/readies/mk/help.defs     |  0
 {deps => automation}/readies/mk/help.rules    |  0
 {deps => automation}/readies/mk/rules         |  0
 {deps => automation}/readies/mk/variant.defs  |  6 +--
 {deps => automation}/readies/mk/variant.rules |  0
 .../readies/paella/__init__.py                |  0
 {deps => automation}/readies/paella/debug.py  |  0
 .../readies/paella/docopt1.py                 |  0
 {deps => automation}/readies/paella/files.py  |  0
 {deps => automation}/readies/paella/log.py    |  0
 .../readies/paella/platform.py                |  0
 {deps => automation}/readies/paella/setup.py  |  0
 {deps => automation}/readies/paella/utils.py  |  0
 {deps => automation}/readies/paella/utils2.py |  0
 {deps => automation}/readies/paella/utils3.py |  0
 .../readies/shibumi/functions                 |  0
 {deps => automation}/readies/shibumi/here     |  0
 system-setup.py => automation/system-setup.py | 19 +++++---
 get_deps.sh                                   | 25 +++++-----
 44 files changed, 82 insertions(+), 87 deletions(-)
 rename Makefile => automation/Makefile (82%)
 rename {deps => automation}/readies/.gitignore (100%)
 rename {deps => automation}/readies/LICENSE (100%)
 rename {deps => automation}/readies/README.md (100%)
 rename {deps => automation}/readies/bin/getpy (100%)
 rename {deps => automation}/readies/bin/getpy2 (100%)
 rename {deps => automation}/readies/bin/getredis5 (96%)
 rename {deps => automation}/readies/bin/platform (100%)
 create mode 100755 automation/readies/bin/python
 rename {deps => automation}/readies/bin/python2 (100%)
 rename {deps => automation}/readies/cetara/diag/gdb.c (100%)
 rename {deps => automation}/readies/cetara/diag/gdb.h (100%)
 rename {deps => automation}/readies/mk/bindirs.defs (100%)
 rename {deps => automation}/readies/mk/bindirs.rules (100%)
 rename {deps => automation}/readies/mk/common.defs (100%)
 rename {deps => automation}/readies/mk/common.rules (100%)
 rename {deps => automation}/readies/mk/configure.defs (100%)
 rename {deps => automation}/readies/mk/configure.rules (100%)
 rename {deps => automation}/readies/mk/defs (71%)
 rename {deps => automation}/readies/mk/help.defs (100%)
 rename {deps => automation}/readies/mk/help.rules (100%)
 rename {deps => automation}/readies/mk/rules (100%)
 rename {deps => automation}/readies/mk/variant.defs (84%)
 rename {deps => automation}/readies/mk/variant.rules (100%)
 rename {deps => automation}/readies/paella/__init__.py (100%)
 rename {deps => automation}/readies/paella/debug.py (100%)
 rename {deps => automation}/readies/paella/docopt1.py (100%)
 rename {deps => automation}/readies/paella/files.py (100%)
 rename {deps => automation}/readies/paella/log.py (100%)
 rename {deps => automation}/readies/paella/platform.py (100%)
 rename {deps => automation}/readies/paella/setup.py (100%)
 rename {deps => automation}/readies/paella/utils.py (100%)
 rename {deps => automation}/readies/paella/utils2.py (100%)
 rename {deps => automation}/readies/paella/utils3.py (100%)
 rename {deps => automation}/readies/shibumi/functions (100%)
 rename {deps => automation}/readies/shibumi/here (100%)
 rename system-setup.py => automation/system-setup.py (85%)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 03cd6de9f..8e2cc466e 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -17,8 +17,8 @@ jobs:
       - run:
           name: install dependencies
           command: |
-            sudo ./deps/readies/bin/getpy
-            sudo ./system-setup.py
+            sudo ./automation/readies/bin/getpy
+            sudo ./automation/system-setup.py
             make deps
             git clone git://github.com/antirez/redis.git --branch 5.0.3
             (cd redis && make malloc=libc -j4 && sudo make install)
@@ -31,12 +31,12 @@ jobs:
       - run:
           name: Build
           command: |
-            make all
-            make pack
+            make -C automation all
+            make -C automation pack
 
       - run:
           name: Test
-          command: make test
+          command: make -C automation test
 
       - run:
           name: Persist Artifacts
diff --git a/.dockerignore b/.dockerignore
index 2905b0714..9c4ba7f87 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,6 +1,4 @@
 /deps/*
-!/deps/readies/
-!/deps/*.py
 /build/
 /install*/
 **/*.o
diff --git a/.gitignore b/.gitignore
index 55ee895f2..0bb1f7455 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,8 +9,6 @@ __pycache__
 # Dirs
 /install*/
 /deps/*
-!/deps/readies/
-!/deps/*.py
 /build/
 examples/js/node_modules/
 examples/js/package-lock.json
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 388177454..68b3cb4a5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -6,6 +6,10 @@ IF (NOT DEPS_PATH)
     MESSAGE(FATAL_ERROR "DEPS PATH Missing!")
 ENDIF()
 
+IF (NOT DEVICE)
+    SET(DEVICE cpu)
+ENDIF()
+
 FUNCTION(ADD_LDFLAGS _TARGET NEW_FLAGS)
     GET_TARGET_PROPERTY(LD_FLAGS ${_TARGET} LINK_FLAGS)
     IF(LD_FLAGS)
diff --git a/Dockerfile b/Dockerfile
index dc13ca8dc..e01c43072 100755
--- a/Dockerfile
+++ b/Dockerfile
@@ -7,24 +7,23 @@ FROM ${OS} AS builder
 WORKDIR /redisai
 COPY --from=redis /usr/local/ /usr/local/
 
-COPY ./deps/readies/ deps/readies/
-COPY ./system-setup.py .
+COPY ./automation/ automation/
 COPY ./test/test_requirements.txt test/
 
-RUN ./deps/readies/bin/getpy
-RUN ./system-setup.py
+RUN ./automation/readies/bin/getpy
+RUN ./automation/system-setup.py
 
 COPY ./get_deps.sh .
 RUN ./get_deps.sh cpu
 
 ADD ./ /redisai
-RUN make all
+RUN make -C automation all
 
 ARG PACK=0
 ARG TEST=0
 
-RUN if [ "$PACK" = "1" ]; then make pack; fi
-RUN if [ "$TEST" = "1" ]; then make test; fi
+RUN if [ "$PACK" = "1" ]; then make -C automation pack; fi
+RUN if [ "$TEST" = "1" ]; then make -C automation test; fi
 
 #----------------------------------------------------------------------------------------------
 FROM redis
diff --git a/README.md b/README.md
index aca9eb0e3..3cc79724d 100644
--- a/README.md
+++ b/README.md
@@ -28,7 +28,7 @@ docker run -p 6379:6379 -it --rm redisai/redisai
 
 ### Give it a try
 
-On the client, load a backend (TF, TORCH, or ONNX), and set the model
+On the client, load a backend (TF, TORCH or ONNX), and set the model
 ```sh
 redis-cli AI.CONFIG LOADBACKEND TF install/backends/redisai_tensorflow/redisai_tensorflow.so
 redis-cli -x AI.MODELSET foo TF CPU INPUTS a b OUTPUTS c < test/test_data/graph.pb
@@ -50,38 +50,25 @@ redis-cli
 ```
 
 ## Building
-In order to build RedisAI, some artifacts need to be installed, notably Python 3. Those can be installed using:
-
-```
-make setup
-```
-
-To examine dependencies in detail, prior to installing them, it is possible to invoke:
-
-```
-./deps/readies/bin/getpy
-make show-setup
-```
-
-(The first command will install Python 3, and the second one will show list of installation commands for the rest of the artifacts.)
-
-----
-
-The following will checkout and build and download the libraries for the backends (TensorFlow, PyTorch, ONNXRuntime) for your platform. Note that this requires CUDA to be installed.
+This will checkout and build and download the libraries for the backends (TensorFlow, PyTorch, ONNXRuntime) for your platform. Note that this requires CUDA to be installed.
 
 ```sh
-make deps GPU=1
+bash get_deps.sh
 ```
 Alternatively, run the following to only fetch the CPU-only backends.
 ```sh
-make deps CPU=1
+bash get_deps.sh cpu
 ```
 
 Once the dependencies are downloaded, build the module itself. Note that
 CMake 3.0 or higher is required.
 
 ```sh
-make build
+mkdir build
+cd build
+cmake -DDEPS_PATH=../deps/install ..
+make && make install
+cd ..
 ```
 
 Note: in order to use the PyTorch backend on Linux, at least `gcc 4.9.2` is required.
diff --git a/Makefile b/automation/Makefile
similarity index 82%
rename from Makefile
rename to automation/Makefile
index c42ef73b0..09a079527 100755
--- a/Makefile
+++ b/automation/Makefile
@@ -1,3 +1,6 @@
+
+ROOT:=..
+
 override GPU:=$(or $(findstring $(CUDA),1),$(findstring $(GPU),1))
 
 ifeq ($(GPU),1)
@@ -11,8 +14,6 @@ DEPS_FLAGS=cpu
 DEVICE=cpu
 endif
 
-export REDIS_ENT_LIB_PATH=/opt/redislabs/lib
-
 GIT_BRANCH:=$(shell git rev-parse --abbrev-ref HEAD)
 GIT_COMMIT:=$(shell git describe --always --abbrev=7 --dirty="+")
 
@@ -29,44 +30,38 @@ BACKENDS_PATH ?= $(INSTALL_DIR)/backends
 
 RAMP:=ramp
 
+ifeq ($(DEBUG),1)
+CMAKE_FLAGS += -DCMAKE_BUILD_TYPE=Debug
+endif
+
 #----------------------------------------------------------------------------------------------
 
 .PHONY: all setup build clean deps pack pack_ramp pack_deps test
 
 all: build
 
-setup:
-	@echo Setting up system...
-	./deps/readies/bin/getpy
-	./system-setup.py
-
-show-setup:
-	@./system-setup.py --nop
-
-ifeq ($(DEBUG),1)
-CMAKE_FLAGS += -DCMAKE_BUILD_TYPE=Debug
-endif
-
 build:
 ifeq ($(wildcard build/Makefile),)
-	mkdir -p build
-	cd build; \
-	cmake -DDEVICE=$(DEVICE) -DDEPS_PATH=../deps/install-$(DEVICE) $(CMAKE_FLAGS) ..
+	mkdir -p $(ROOT)/build
+	cd $(ROOT)/build; \
+	cmake -DDEVICE=$(DEVICE) -DDEPS_PATH=$(ROOT)/deps/install-$(DEVICE) $(CMAKE_FLAGS) $(ROOT)
 endif
-	$(MAKE) -C build
-	$(MAKE) -C build install
-	ln -sf install-$(DEVICE) install
+	$(MAKE) -C $(ROOT)/build
+	$(MAKE) -C $(ROOT)/build install
+	cd $(ROOT) ;\
+	[ ! -e install ] && ln -sf install-$(DEVICE) install
 
 clean:
 ifeq ($(ALL),1)
+	cd $(ROOT) ;\
 	rm -rf build install deps/dlpack deps/install-$(DEVICE) deps/*.tar.gz deps/*.zip deps/*.tgz
 else
-	$(MAKE) -C build clean
+	$(MAKE) -C $(ROOT)/build clean
 endif
 
 deps fetch:
 	@echo Fetching dependencies...
-	@./get_deps.sh $(DEPS_FLAGS)
+	@$(ROOT)/get_deps.sh $(DEPS_FLAGS)
 
 pack: pack_ramp pack_deps
 
@@ -74,8 +69,8 @@ pack_ramp:
 	@[ ! -z `command -v redis-server` ] || { echo "Cannot find redis-server - aborting."; exit 1; }
 	@echo "Building RAMP file ..."
 	@set -e ;\
+	cd $(ROOT) ;\
 	RAMPOUT=$$(mktemp /tmp/ramp.XXXXXX) ;\
-	LD_LIBRARY_PATH=$(PWD)/deps/install-$(DEVICE)/lib \
 	$(RAMP) pack -m $(PWD)/ramp.yml -o "build/redisai.{os}-{architecture}.{semantic_version}.zip" $(INSTALL_DIR)/redisai.so 2> /dev/null | grep '.zip' > $$RAMPOUT ;\
 	tail -1 $$RAMPOUT > $(BINDIR)/PACKAGE ;\
 	cat $(BINDIR)/PACKAGE | sed -e "s/[^.]*\.[^.]*\.\(.*\)\.zip/\1/" > $(BINDIR)/VERSION ;\
@@ -88,6 +83,7 @@ pack_ramp:
 pack_deps: pack_ramp
 	@echo "Building dependencies file ..."
 	@set -e ;\
+	cd $(ROOT) ;\
 	PACK_FNAME=$$(basename `cat $(BINDIR)/PACKAGE`) ;\
 	ARCHOSVER=$$(echo "$$PACK_FNAME" | sed -e "s/^redisai\.\([^.]*\..*\)\.zip/\1/") ;\
 	VERSION=$$(cat $(BINDIR)/VERSION) ;\
@@ -95,7 +91,7 @@ pack_deps: pack_ramp
 	rm -rf redisai-$(DEVICE)-$$VERSION ;\
 	mkdir redisai-$(DEVICE)-$$VERSION ;\
 	mv backends redisai-$(DEVICE)-$$VERSION ;\
-	ln -s redisai-$(DEVICE)-$$VERSION/backends backends ;\
+	ln -sf redisai-$(DEVICE)-$$VERSION/backends backends ;\
 	find redisai-$(DEVICE)-$$VERSION -name "*.so*" | xargs tar pczf redisai-$(DEVICE)-dependencies.$$ARCHOSVER.tgz ;\
 	echo "Done."
 
@@ -103,5 +99,5 @@ test:
 	@git lfs install
 	@git lfs pull
 	@set -e ;\
-	cd test ;\
+	cd $(ROOT)/test ;\
 	python3 -m RLTest $(TEST_ARGS) --test basic_tests.py --module $(INSTALL_DIR)/redisai.so
diff --git a/deps/readies/.gitignore b/automation/readies/.gitignore
similarity index 100%
rename from deps/readies/.gitignore
rename to automation/readies/.gitignore
diff --git a/deps/readies/LICENSE b/automation/readies/LICENSE
similarity index 100%
rename from deps/readies/LICENSE
rename to automation/readies/LICENSE
diff --git a/deps/readies/README.md b/automation/readies/README.md
similarity index 100%
rename from deps/readies/README.md
rename to automation/readies/README.md
diff --git a/deps/readies/bin/getpy b/automation/readies/bin/getpy
similarity index 100%
rename from deps/readies/bin/getpy
rename to automation/readies/bin/getpy
diff --git a/deps/readies/bin/getpy2 b/automation/readies/bin/getpy2
similarity index 100%
rename from deps/readies/bin/getpy2
rename to automation/readies/bin/getpy2
diff --git a/deps/readies/bin/getredis5 b/automation/readies/bin/getredis5
similarity index 96%
rename from deps/readies/bin/getredis5
rename to automation/readies/bin/getredis5
index ce58ccb22..caabb8ddf 100755
--- a/deps/readies/bin/getredis5
+++ b/automation/readies/bin/getredis5
@@ -4,7 +4,7 @@ import sys
 import os
 import argparse
 
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), "deps/readies"))
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
 import paella
 
 os.environ["PYTHONWARNINGS"] = 'ignore:DEPRECATION::pip._internal.cli.base_command'
diff --git a/deps/readies/bin/platform b/automation/readies/bin/platform
similarity index 100%
rename from deps/readies/bin/platform
rename to automation/readies/bin/platform
diff --git a/automation/readies/bin/python b/automation/readies/bin/python
new file mode 100755
index 000000000..814ff07ce
--- /dev/null
+++ b/automation/readies/bin/python
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+if [ ! -z $(command -v python3) ]; then
+	python3 "$@"
+else
+	python "$@"
+fi
diff --git a/deps/readies/bin/python2 b/automation/readies/bin/python2
similarity index 100%
rename from deps/readies/bin/python2
rename to automation/readies/bin/python2
diff --git a/deps/readies/cetara/diag/gdb.c b/automation/readies/cetara/diag/gdb.c
similarity index 100%
rename from deps/readies/cetara/diag/gdb.c
rename to automation/readies/cetara/diag/gdb.c
diff --git a/deps/readies/cetara/diag/gdb.h b/automation/readies/cetara/diag/gdb.h
similarity index 100%
rename from deps/readies/cetara/diag/gdb.h
rename to automation/readies/cetara/diag/gdb.h
diff --git a/deps/readies/mk/bindirs.defs b/automation/readies/mk/bindirs.defs
similarity index 100%
rename from deps/readies/mk/bindirs.defs
rename to automation/readies/mk/bindirs.defs
diff --git a/deps/readies/mk/bindirs.rules b/automation/readies/mk/bindirs.rules
similarity index 100%
rename from deps/readies/mk/bindirs.rules
rename to automation/readies/mk/bindirs.rules
diff --git a/deps/readies/mk/common.defs b/automation/readies/mk/common.defs
similarity index 100%
rename from deps/readies/mk/common.defs
rename to automation/readies/mk/common.defs
diff --git a/deps/readies/mk/common.rules b/automation/readies/mk/common.rules
similarity index 100%
rename from deps/readies/mk/common.rules
rename to automation/readies/mk/common.rules
diff --git a/deps/readies/mk/configure.defs b/automation/readies/mk/configure.defs
similarity index 100%
rename from deps/readies/mk/configure.defs
rename to automation/readies/mk/configure.defs
diff --git a/deps/readies/mk/configure.rules b/automation/readies/mk/configure.rules
similarity index 100%
rename from deps/readies/mk/configure.rules
rename to automation/readies/mk/configure.rules
diff --git a/deps/readies/mk/defs b/automation/readies/mk/defs
similarity index 71%
rename from deps/readies/mk/defs
rename to automation/readies/mk/defs
index 27059268b..819d0d30c 100755
--- a/deps/readies/mk/defs
+++ b/automation/readies/mk/defs
@@ -1,5 +1,5 @@
 
-MK=$(ROOT)/deps/readies/mk
+MK=$(ROOT)/automation/readies/mk
 
 include $(MK)/common.defs
 include $(MK)/variant.defs
diff --git a/deps/readies/mk/help.defs b/automation/readies/mk/help.defs
similarity index 100%
rename from deps/readies/mk/help.defs
rename to automation/readies/mk/help.defs
diff --git a/deps/readies/mk/help.rules b/automation/readies/mk/help.rules
similarity index 100%
rename from deps/readies/mk/help.rules
rename to automation/readies/mk/help.rules
diff --git a/deps/readies/mk/rules b/automation/readies/mk/rules
similarity index 100%
rename from deps/readies/mk/rules
rename to automation/readies/mk/rules
diff --git a/deps/readies/mk/variant.defs b/automation/readies/mk/variant.defs
similarity index 84%
rename from deps/readies/mk/variant.defs
rename to automation/readies/mk/variant.defs
index a1c242702..aaccb3849 100755
--- a/deps/readies/mk/variant.defs
+++ b/automation/readies/mk/variant.defs
@@ -1,10 +1,10 @@
 
-OS:=$(shell $(ROOT)/deps/readies/bin/platform --os)
+OS:=$(shell $(ROOT)/automation/readies/bin/platform --os)
 # ifeq ($(OS),linux)
-# OS:=$(shell $(ROOT)/deps/readies/bin/platform --dist)
+# OS:=$(shell $(ROOT)/automation/readies/bin/platform --dist)
 # endif
 
-ARCH=$(shell $(ROOT)/deps/readies/bin/platform --arch)
+ARCH=$(shell $(ROOT)/automation/readies/bin/platform --arch)
 
 #----------------------------------------------------------------------------------------------
 
diff --git a/deps/readies/mk/variant.rules b/automation/readies/mk/variant.rules
similarity index 100%
rename from deps/readies/mk/variant.rules
rename to automation/readies/mk/variant.rules
diff --git a/deps/readies/paella/__init__.py b/automation/readies/paella/__init__.py
similarity index 100%
rename from deps/readies/paella/__init__.py
rename to automation/readies/paella/__init__.py
diff --git a/deps/readies/paella/debug.py b/automation/readies/paella/debug.py
similarity index 100%
rename from deps/readies/paella/debug.py
rename to automation/readies/paella/debug.py
diff --git a/deps/readies/paella/docopt1.py b/automation/readies/paella/docopt1.py
similarity index 100%
rename from deps/readies/paella/docopt1.py
rename to automation/readies/paella/docopt1.py
diff --git a/deps/readies/paella/files.py b/automation/readies/paella/files.py
similarity index 100%
rename from deps/readies/paella/files.py
rename to automation/readies/paella/files.py
diff --git a/deps/readies/paella/log.py b/automation/readies/paella/log.py
similarity index 100%
rename from deps/readies/paella/log.py
rename to automation/readies/paella/log.py
diff --git a/deps/readies/paella/platform.py b/automation/readies/paella/platform.py
similarity index 100%
rename from deps/readies/paella/platform.py
rename to automation/readies/paella/platform.py
diff --git a/deps/readies/paella/setup.py b/automation/readies/paella/setup.py
similarity index 100%
rename from deps/readies/paella/setup.py
rename to automation/readies/paella/setup.py
diff --git a/deps/readies/paella/utils.py b/automation/readies/paella/utils.py
similarity index 100%
rename from deps/readies/paella/utils.py
rename to automation/readies/paella/utils.py
diff --git a/deps/readies/paella/utils2.py b/automation/readies/paella/utils2.py
similarity index 100%
rename from deps/readies/paella/utils2.py
rename to automation/readies/paella/utils2.py
diff --git a/deps/readies/paella/utils3.py b/automation/readies/paella/utils3.py
similarity index 100%
rename from deps/readies/paella/utils3.py
rename to automation/readies/paella/utils3.py
diff --git a/deps/readies/shibumi/functions b/automation/readies/shibumi/functions
similarity index 100%
rename from deps/readies/shibumi/functions
rename to automation/readies/shibumi/functions
diff --git a/deps/readies/shibumi/here b/automation/readies/shibumi/here
similarity index 100%
rename from deps/readies/shibumi/here
rename to automation/readies/shibumi/here
diff --git a/system-setup.py b/automation/system-setup.py
similarity index 85%
rename from system-setup.py
rename to automation/system-setup.py
index aa1ca2337..03fbb8705 100755
--- a/system-setup.py
+++ b/automation/system-setup.py
@@ -5,7 +5,7 @@
 from subprocess import Popen, PIPE
 import argparse
 
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), "deps/readies"))
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "readies"))
 import paella
 
 #----------------------------------------------------------------------------------------------
@@ -22,12 +22,9 @@ def common_first(self):
         self.install("git cmake ca-certificates curl unzip wget patchelf awscli")
 
     def debian_compat(self):
-        self.run("curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash")
-        self.install("git-lfs")
-        # self.run("git lfs install")
-
         self.install("build-essential")
         self.install("python3-venv python3-psutil python3-networkx")
+        self.install_git_lfs_on_linux()
 
     def redhat_compat(self):
         self.group_install("'Development Tools'")
@@ -37,15 +34,23 @@ def redhat_compat(self):
         self.install("python36 python36-pip")
         self.install("python36-psutil")
 
+        self.install_git_lfs_on_linux()
+        
     def fedora(self):
         self.group_install("'Development Tools'")
+        self.install_git_lfs_on_linux()
 
     def macosx(self):
         p = Popen('xcode-select -p', stdout=PIPE, close_fds=True, shell=True)
-        p.communicate()
-        if p.stdout.readlines() == []:
+        out, _ = p.communicate()
+        if out.splitlines() == []:
             fatal("Xcode tools are not installed. Please run xcode-select --install.")
+        self.install("git-lfs")
 
+    def install_git_lfs_on_linux(self):
+        self.run("curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash")
+        self.install("git-lfs")
+    
     def common_last(self):
         if not self.has_command("ramp"):
             self.pip3_install("git+https://github.com/RedisLabs/RAMP --upgrade")
diff --git a/get_deps.sh b/get_deps.sh
index 3f799e7ae..6212ea4f6 100755
--- a/get_deps.sh
+++ b/get_deps.sh
@@ -24,12 +24,15 @@ DEPS_DIR=$HERE/deps
 mkdir -p ${DEPS_DIR}
 cd ${DEPS_DIR}
 
-PREFIX=${DEPS_DIR}/install-$DEVICE
+PREFIX=${DEPS_DIR}/install
 mkdir -p ${PREFIX}
 
-## DLPACK
-
 DLPACK_PREFIX=${PREFIX}/dlpack
+TF_PREFIX=${PREFIX}/libtensorflow
+TORCH_PREFIX=${PREFIX}/libtorch
+ORT_PREFIX=${PREFIX}/onnxruntime
+
+## DLPACK
 
 [[ $FORCE == 1 ]] && rm -rf ${DLPACK_PREFIX}
 
@@ -49,7 +52,6 @@ fi
 ## TENSORFLOW
 
 TF_VERSION="1.12.0"
-TF_PREFIX=${PREFIX}/libtensorflow
 
 [[ $FORCE == 1 ]] && rm -rf ${TF_PREFIX}
 
@@ -89,8 +91,6 @@ fi
 PT_VERSION="1.1.0"
 #PT_VERSION="latest"
 
-TORCH_PREFIX=${PREFIX}/libtorch
-
 [[ $FORCE == 1 ]] && rm -rf ${TORCH_PREFIX}
 
 if [[ ! -d ${TORCH_PREFIX} ]]; then
@@ -132,13 +132,16 @@ if [[ ! -d ${TORCH_PREFIX} ]]; then
   echo "Done."
   
   if [[ "${PT_OS}" == "macos" ]]; then
+	echo "Installing MKL ..."
     # also download mkl
     MKL_BUNDLE=mklml_mac_2019.0.3.20190220
     if [ ! -e "${MKL_BUNDLE}.tgz" ]; then
       wget -q "https://github.com/intel/mkl-dnn/releases/download/v0.18/${MKL_BUNDLE}.tgz"
     fi
-    tar xf ${MKL_BUNDLE}.tgz --no-same-owner --strip-components=1 -C ${TORCH_PREFIX}
-    tar xf ${MKL_BUNDLE}.tgz --no-same-owner --strip-components=1 -C ${ORT_PREFIX}
+    tar xzf ${MKL_BUNDLE}.tgz --no-same-owner --strip-components=1 -C ${TORCH_PREFIX}
+	mkdir -p ${ORT_PREFIX}
+    tar xzf ${MKL_BUNDLE}.tgz --no-same-owner --strip-components=1 -C ${ORT_PREFIX}
+	echo "Done."
   fi
 else
   echo "librotch is in place."
@@ -157,12 +160,10 @@ if [[ "$OSTYPE" == "linux-gnu" ]]; then
     ORT_BUILD="-gpu"
   fi
 elif [[ "$OSTYPE" == "darwin"* ]]; then
-  ORT_OS="osx-x64"
+  ORT_OS="osx"
   ORT_BUILD=""
 fi
 
-ORT_PREFIX=${PREFIX}/onnxruntime
-
 [[ $FORCE == 1 ]] && rm -rf ${ORT_PREFIX}
 
 ORT_ARCHIVE=onnxruntime-${ORT_OS}-${ORT_ARCH}${ORT_BUILD}-${ORT_VERSION}.tgz
@@ -174,7 +175,7 @@ if [[ ! -d ${ORT_PREFIX} ]]; then
 
   if [ ! -e ${ORT_ARCHIVE} ]; then
     echo "Downloading ONNXRuntime ${ORT_VERSION} ${DEVICE}"
-    wget -q https://github.com/Microsoft/onnxruntime/releases/download/v${ORT_VERSION}/${ORT_ARCHIVE}
+    wget -q https://github.com/microsoft/onnxruntime/releases/download/v${ORT_VERSION}/${ORT_ARCHIVE}
   fi
   
   tar xf ${ORT_ARCHIVE} --no-same-owner --strip-components=1 -C ${ORT_PREFIX}

From 0250daa1e11d3aaeff2db8428aaf82fea55d50eb Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Fri, 16 Aug 2019 15:50:59 +0300
Subject: [PATCH 25/31] Fixes from review #3

---
 .dockerignore       | 6 +++---
 automation/Makefile | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.dockerignore b/.dockerignore
index 9c4ba7f87..9418ba6fc 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,7 +1,7 @@
-/deps/*
+/deps/
 /build/
 /install*/
 **/*.o
 **/*.so
-/.venv/
-/venv/
+.venv/
+venv*/
diff --git a/automation/Makefile b/automation/Makefile
index 09a079527..f15a6cfd6 100755
--- a/automation/Makefile
+++ b/automation/Makefile
@@ -44,7 +44,7 @@ build:
 ifeq ($(wildcard build/Makefile),)
 	mkdir -p $(ROOT)/build
 	cd $(ROOT)/build; \
-	cmake -DDEVICE=$(DEVICE) -DDEPS_PATH=$(ROOT)/deps/install-$(DEVICE) $(CMAKE_FLAGS) $(ROOT)
+	cmake -DDEVICE=$(DEVICE) -DDEPS_PATH=$(ROOT)/deps/install $(CMAKE_FLAGS) $(ROOT)
 endif
 	$(MAKE) -C $(ROOT)/build
 	$(MAKE) -C $(ROOT)/build install

From 4b1f91f26c33fdf821c57271337933f24700aef1 Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Fri, 16 Aug 2019 16:00:16 +0300
Subject: [PATCH 26/31] Fixes from review #4

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 8e2cc466e..6fec9a526 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -19,7 +19,7 @@ jobs:
           command: |
             sudo ./automation/readies/bin/getpy
             sudo ./automation/system-setup.py
-            make deps
+            make -C automation deps
             git clone git://github.com/antirez/redis.git --branch 5.0.3
             (cd redis && make malloc=libc -j4 && sudo make install)
 

From ff16afaaec6b73d7d5cc9b56759a024d0e9e9100 Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Fri, 16 Aug 2019 16:11:42 +0300
Subject: [PATCH 27/31] Fixes from review #5

---
 .circleci/config.yml | 2 +-
 automation/Makefile  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 6fec9a526..a8a8df15d 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -19,7 +19,7 @@ jobs:
           command: |
             sudo ./automation/readies/bin/getpy
             sudo ./automation/system-setup.py
-            make -C automation deps
+            ./get_deps.sh cpu
             git clone git://github.com/antirez/redis.git --branch 5.0.3
             (cd redis && make malloc=libc -j4 && sudo make install)
 
diff --git a/automation/Makefile b/automation/Makefile
index f15a6cfd6..f4398ce9c 100755
--- a/automation/Makefile
+++ b/automation/Makefile
@@ -61,7 +61,7 @@ endif
 
 deps fetch:
 	@echo Fetching dependencies...
-	@$(ROOT)/get_deps.sh $(DEPS_FLAGS)
+	$(ROOT)/get_deps.sh $(DEPS_FLAGS)
 
 pack: pack_ramp pack_deps
 

From 4b0377565c68cb97c16777e2bd9a236f925219af Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Fri, 16 Aug 2019 16:17:42 +0300
Subject: [PATCH 28/31] Fixes from review #6

---
 .circleci/config.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index a8a8df15d..82d255c47 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -8,11 +8,11 @@ jobs:
       - checkout
 
       # Download and cache dependencies
-      - restore_cache:
-          keys:
-          - v1-dependencies-{{ checksum "get_deps.sh" }}
-          # fallback to using the latest cache if no exact match is found
-          - v1-dependencies-
+#      - restore_cache:
+#          keys:
+#          - v1-dependencies-{{ checksum "get_deps.sh" }}
+#          # fallback to using the latest cache if no exact match is found
+#          - v1-dependencies-
 
       - run:
           name: install dependencies

From ba62dfd3ce939012e3a17f3489f04cf5ab7c4444 Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Fri, 16 Aug 2019 16:23:16 +0300
Subject: [PATCH 29/31] Fixes from review #7

---
 .circleci/config.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 82d255c47..a8a8df15d 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -8,11 +8,11 @@ jobs:
       - checkout
 
       # Download and cache dependencies
-#      - restore_cache:
-#          keys:
-#          - v1-dependencies-{{ checksum "get_deps.sh" }}
-#          # fallback to using the latest cache if no exact match is found
-#          - v1-dependencies-
+      - restore_cache:
+          keys:
+          - v1-dependencies-{{ checksum "get_deps.sh" }}
+          # fallback to using the latest cache if no exact match is found
+          - v1-dependencies-
 
       - run:
           name: install dependencies

From 50e6770ce27269a45753b84fb17368036a62fc1b Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Fri, 16 Aug 2019 16:37:19 +0300
Subject: [PATCH 30/31] CircleCI cache hacking

---
 .circleci/config.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index a8a8df15d..89ffdf0cf 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -10,9 +10,9 @@ jobs:
       # Download and cache dependencies
       - restore_cache:
           keys:
-          - v1-dependencies-{{ checksum "get_deps.sh" }}
+          - v2-dependencies-{{ checksum "get_deps.sh" }}
           # fallback to using the latest cache if no exact match is found
-          - v1-dependencies-
+          - v2-dependencies-
 
       - run:
           name: install dependencies
@@ -26,7 +26,7 @@ jobs:
       - save_cache:
           paths:
             - deps
-          key: v1-dependencies-{{ checksum "get_deps.sh" }}
+          key: v2-dependencies-{{ checksum "get_deps.sh" }}
 
       - run:
           name: Build

From d24b7d30f009cf647ffad0a6d69a49ce4720ad96 Mon Sep 17 00:00:00 2001
From: rafie <rafi@redislabs.com>
Date: Fri, 16 Aug 2019 18:25:55 +0300
Subject: [PATCH 31/31] CircleCI cache: cancelled fallback

---
 .circleci/config.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 89ffdf0cf..56fe7236c 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -10,9 +10,9 @@ jobs:
       # Download and cache dependencies
       - restore_cache:
           keys:
-          - v2-dependencies-{{ checksum "get_deps.sh" }}
+          - v1-dependencies-{{ checksum "get_deps.sh" }}
           # fallback to using the latest cache if no exact match is found
-          - v2-dependencies-
+          # - v1-dependencies-
 
       - run:
           name: install dependencies