Install redis-gears from S3 as part of module deps, and test AI-Gears LLAPI.

Dockerfile.gpu-test

@@ -35,6 +35,7 @@ RUN echo export LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/cuda-11.0/lib64
 
 COPY ./opt/ opt/
 COPY ./tests/flow/test_requirements.txt tests/flow/
+COPY ./tests/flow/Install_RedisGears.sh tests/flow/
 
 RUN VENV=venv FORCE=1 ./opt/readies/bin/getpy3
 

opt/redis_valgrind.sup

@@ -68,3 +68,110 @@
    fun:RAI_LoadBackend
 }
 
+{
+   <ignore_py_error0>
+   Memcheck:Cond
+   fun:lzf_compress
+}
+{
+   <ignore_py_error0.1>
+   Memcheck:Cond
+   ...
+   fun:Py_InitializeEx
+   ...
+}
+{
+   <ignore_py_error0.2>
+   Memcheck:Cond
+   ...
+   fun:_PyEval_EvalFrameDefault
+   ...
+}
+{
+   <ignore_py_error0.3>
+   Memcheck:Leak
+   ...
+   fun:_imp_exec_builtin
+   ...
+}
+{
+   <ignore_py_error0.4>
+   Memcheck:Cond
+   ...
+   fun:PySys_SetArgvEx
+   ...
+}
+{
+   <ignore_py_error0.5>
+   Memcheck:Cond
+   ...
+   fun:_PyMethodDef_RawFastCallKeywords
+   ...
+}
+{
+   <ignore_py_error0>
+   Memcheck:Value8
+   fun:lzf_compress
+}
+{
+   <ignore_py_error1>
+   Memcheck:Addr4
+   obj:*libpython2.7*
+}
+{
+   <ignore_py_error1>
+   Memcheck:Addr8
+   obj:*libpython2.7*
+}
+{
+   <ignore_py_error2>
+   Memcheck:Value8
+   obj:*libpython2.7*
+}
+{
+   <ignore_py_error3>
+   Memcheck:Cond
+   obj:*libpython2.7*
+}
+{
+   <ignore_py_error4>
+   Memcheck:Leak
+   ...
+   obj:*libpython2.7*
+   ...
+}
+{
+   <ignore_libevent_issues>
+   Memcheck:Leak
+   ...
+   fun:redisLibeventAttach
+   ...
+}
+{
+   <ignore_compiler_visit_expr_for_now>
+   Memcheck:Leak
+   ...
+   fun:compiler_visit_expr
+   ...
+}
+{
+   <ignore_compiler_visit_expr_for_now>
+   Memcheck:Leak
+   ...
+   fun:redisConnectNonBlock
+   ...
+}
+{
+   <ignore_connection_leak_for_now>
+   Memcheck:Leak
+   ...
+   fun:_redisContextConnectTcp
+   ...
+}
+{
+   Libuv_epoll_ctl
+   Memcheck:Param
+   epoll_ctl(event)
+   fun:epoll_ctl
+   ...
+}

tests/flow/Install_RedisGears.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+ROOT=$(cd $HERE/../..; pwd)
+READIES=$ROOT/opt/readies
+. $READIES/shibumi/defs
+
+if [[ $1 == --help || $1 == help || $HELP == 1 ]]; then
+	cat <<-END
+		Obtain RedisGears module binaries
+
+		Install_RedisGears.sh [--help|help]
+
+		Argument variables:
+		GEARS_OSNICK=nick   Get binaries for give osnick
+		GEARS_PATH=dir      Get binaries from given Gears repo
+		NOP=1               No operation
+		HELP=1              Show help
+
+	END
+	exit 0
+fi
+
+OP=""
+[[ $NOP == 1 ]] && OP=echo
+
+os=$($READIES/bin/platform --os)
+arch=$($READIES/bin/platform --arch)
+
+if [[ ! -z $GEARS_PATH ]]; then
+	platform=$($READIES/bin/platform -t)
+else
+	if [[ $os != linux || $arch != x64 ]]; then
+		eprint "Cannot match binary artifacts - build RedisGears and set GEARS_PATH"
+		exit 1
+	fi
+
+	dist=$($READIES/bin/platform --dist)
+	nick=$($READIES/bin/platform --osnick)
+
+	if [[ $dist == ubuntu ]]; then
+		if [[ $nick != bionic && $nick != xenial && $nick != trusty ]]; then
+			nick=bionic
+		fi
+	elif [[ $dist == debian ]]; then
+		nick=bionic
+	elif [[ $dist == centos || $dist == redhat || $dist == fedora ]]; then
+		nick=centos7
+	elif [[ ! -z $GEARS_OSNICK ]]; then
+		nick="$GEARS_OSNICK"
+	else
+		eprint "Cannot match binary artifacts - build RedisGears and set GEARS_PATH"
+		exit 1
+	fi
+	platform="${os}-${nick}-${arch}"
+fi
+
+GEARS_S3_URL=http://redismodules.s3.amazonaws.com/redisgears/snapshots
+GEARS_MOD=redisgears.$platform.master.zip
+GEARS_DEPS=redisgears-python.$platform.master.tgz
+
+FINAL_WORK_DIR=$ROOT/bin/$($READIES/bin/platform -t)/RedisGears
+
+if [[ -d $FINAL_WORK_DIR && -f $FINAL_WORK_DIR/redisgears.so ]]; then
+	echo "RedisGears is in $FINAL_WORK_DIR"
+	exit 0
+fi
+
+$OP mkdir -p $(dirname $FINAL_WORK_DIR)
+$OP rm -rf ${FINAL_WORK_DIR}.*
+WORK_DIR=$(mktemp -d ${FINAL_WORK_DIR}.XXXXXX)
+$OP mkdir -p $WORK_DIR
+
+if [[ -z $GEARS_PATH ]]; then
+	F_GEARS_MOD=$WORK_DIR/$GEARS_MOD
+	if [[ ! -f $F_GEARS_MOD ]]; then
+		echo "Download RedisGears ..."
+		$OP wget -q -P $WORK_DIR $GEARS_S3_URL/$GEARS_MOD
+	fi
+
+	F_GEARS_DEPS=$WORK_DIR/$GEARS_DEPS
+	if [[ ! -f $F_GEARS_DEPS ]]; then
+		echo "Download RedisGears deps ..."
+		$OP wget -q -P $WORK_DIR $GEARS_S3_URL/$GEARS_DEPS
+	fi
+else
+	F_GEARS_MOD=$GEARS_PATH/artifacts/snapshot/$GEARS_MOD
+	F_GEARS_DEPS=$GEARS_PATH/artifacts/snapshot/$GEARS_DEPS
+	[[ ! -f $F_GEARS_MOD ]] && { eprint "$F_GEARS_MOD is missing"; exit 1; }
+	[[ ! -f $F_GEARS_DEPS ]] && { eprint "$F_GEARS_DEPS is missing"; exit 1; }
+fi
+
+$OP unzip -q $F_GEARS_MOD -d $WORK_DIR
+$OP tar --no-same-owner -C $WORK_DIR -xzf $F_GEARS_DEPS
+$OP mv $WORK_DIR $FINAL_WORK_DIR

tests/flow/includes.py

@@ -10,12 +10,14 @@
 from skimage.io import imread
 from skimage.transform import resize
 
-try:
-    sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../deps/readies"))
-    import paella
-except:
-    pass
 
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../opt/readies"))
+import paella
+
+ROOT = os.environ.get("ROOT", None)
+if ROOT is None:
+    sys.stderr.write("ROOT was not defined in the environment.\n")
+    sys.exit(1)
 MAX_ITERATIONS = 2 if os.environ.get("MAX_ITERATIONS") == None else os.environ.get("MAX_ITERATIONS")
 TEST_TF = os.environ.get("TEST_TF") != "0" and os.environ.get("WITH_TF") != "0"
 TEST_TFLITE = os.environ.get("TEST_TFLITE") != "0" and os.environ.get("WITH_TFLITE") != "0"

tests/flow/tests.sh

@@ -136,6 +136,7 @@ cd $ROOT/tests/flow
 
 install_git_lfs
 check_redis_server
+./Install_RedisGears.sh
 
 [[ ! -z $REDIS ]] && RL_TEST_ARGS+=" --env exiting-env --existing-env-addr $REDIS" run_tests "redis-server: $REDIS"
 [[ $GEN == 1 ]]    && run_tests