RedisAI · filipecosta90 · Sep 2, 2020 · May 19, 2020 · May 20, 2020 · May 25, 2020
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -30,7 +30,7 @@ commands:
           command: |
             ./opt/readies/bin/getpy3
             BREW_NO_UPDATE=1 ./opt/system-setup.py
-            # ./opt/readies/bin/getredis -v 6 --force
+            ./opt/readies/bin/getredis -v 6 --force
             ./get_deps.sh cpu
       - save_cache:
           paths:
@@ -46,12 +46,17 @@ commands:
           no_output_timeout: 20m
       - run:
           name: Package
-          command: make -C opt pack SHOW=1
+          command: |
+            make -C opt pack SHOW=1
+            (cd bin/artifacts; tar -cf snapshots-<<parameters.platform>>.tar snapshots/)
       - persist_to_workspace:
           root: bin/
           paths:
             - artifacts/*.zip
             - artifacts/*.tgz
+            - artifacts/*.tar
+            # - artifacts/shapshots/*.zip
+            # - artifacts/shapshots/*.tgz
       - store_artifacts:
           path: test/logs
 
@@ -71,16 +76,27 @@ commands:
           name: Build for platform
           command: |
             docker login -u redisfab -p $DOCKER_REDISFAB_PWD
-            cd opt/build/docker
+            pushd opt/build/docker
             #@@ make build publish $(./version-params) CPU=1 OSNICK=<<parameters.platform>> X64=1 ARTIFACTS=1 TEST=1 VERBOSE=1
             make build publish $(./version-params) CPU=1 OSNICK=<<parameters.platform>> X64=1 ARTIFACTS=1 VERBOSE=1
             make build publish $(./version-params) GPU=1 OSNICK=<<parameters.platform>> X64=1 ARTIFACTS=1 VERBOSE=1
+            popd > /dev/null
+            logstar=bin/artifacts/tests-logs-cpu.tgz
+            logsdir=tests/logs/cpu
+            mkdir -p $logsdir
+            if [[ -e $logstar ]]; then tar -C $logsdir -xzf $logstar; fi
+            (cd bin/artifacts; tar -cf snapshots-<<parameters.platform>>.tar snapshots/)
           no_output_timeout: 40m
       - persist_to_workspace:
           root: bin/
           paths:
             - artifacts/*.zip
             - artifacts/*.tgz
+            - artifacts/*.tar
+            # - artifacts/shapshots/*.zip
+            # - artifacts/shapshots/*.tgz
+      - store_artifacts:
+          path: test/logs
 
   deploy-steps:
     parameters:
@@ -95,7 +111,7 @@ commands:
 jobs:
   build-debian:
     docker:
-      - image: redisfab/rmbuilder:6.0.1-x64-buster
+      - image: redisfab/rmbuilder:6.0.5-x64-buster
     steps:
       - build-steps:
           platform: debian
@@ -112,7 +128,7 @@ jobs:
 
   coverage:
     docker:
-      - image: redisfab/rmbuilder:6.0.1-x64-buster
+      - image: redisfab/rmbuilder:6.0.5-x64-buster
     steps:
       - checkout
       - run:
@@ -136,7 +152,7 @@ jobs:
       - run:
           name: Test with coverage
           command: |
-            make -C opt test SHOW=1 COV=1
+            make -C opt test SHOW=1 COV=1 CLUSTER=1
             make -C opt cov-upload
           no_output_timeout: 20m
 
@@ -202,22 +218,56 @@ jobs:
             docker run --gpus all -v $HOME/tests:/build/test/logs -it --rm redisai-gpu:latest-x64-bionic-test
           no_output_timeout: 40m
       - store_artifacts:
-          path: tests
+          path: test/log
 
   deploy-artifacts:
     parameters:
-      package:
+      location:
         type: string
     docker:
-      - image: redisfab/rmbuilder:6.0.1-x64-buster
+      - image: redisfab/rmbuilder:6.0.5-x64-buster
     steps:
       - attach_workspace:
           at: workspace
       - run:
           name: Deploy to S3
           command: |
-            cd workspace
-            aws s3 cp artifacts/ s3://redismodules/$PACKAGE_NAME/ --acl public-read --recursive --exclude "*" --include "*.zip" --include "*.tgz"
+            cd "workspace/artifacts/<<parameters.location>>"
+            for f in *.zip snapshot/*.tgz; do
+              aws s3 cp $f s3://redismodules/$PACKAGE_NAME/<<parameters.location>>/ --acl public-read
+            done
+
+  deploy-snapshot:
+    docker:
+      - image: redisfab/rmbuilder:6.0.5-x64-buster
+    steps:
+      - attach_workspace:
+          at: workspace
+      - run:
+          name: Deploy Snapshots to S3
+          command: |
+            cd workspace/artifacts
+            for f in snapshots-*.tar; do
+                tar xf $f
+            done
+            cd snapshots
+            for f in *.zip *.tgz; do
+              aws s3 cp --no-progress $f s3://redismodules/$PACKAGE_NAME/snapshots/ --acl public-read
+            done
+
+  deploy-release:
+    docker:
+      - image: redisfab/rmbuilder:6.0.5-x64-buster
+    steps:
+      - attach_workspace:
+          at: workspace
+      - run:
+          name: Deploy Releases to S3
+          command: |
+            cd workspace/artifacts
+            for f in *.zip *.tgz; do
+              aws s3 cp --no-progress $f s3://redismodules/$PACKAGE_NAME/ --acl public-read
+            done
 
 
 on-any-branch: &on-any-branch
@@ -296,14 +346,10 @@ workflows:
       - build-multiarch-docker:
           <<: *never # temporarily disabled
           # <<: *on-version-tags
-      - deploy-artifacts:
-          name: deploy-branch
-          package: branch
+      - deploy-snapshot:
           <<: *after-platform-builds
-          <<: *on-any-branch
-      - deploy-artifacts:
-          name: deploy-release
-          package: release
+          <<: *on-master
+      - deploy-release:
           <<: *after-platform-builds
           <<: *on-version-tags
 

diff --git a/.github/workflows/deploy-docs.yaml b/.github/workflows/deploy-docs.yaml
@@ -21,6 +21,8 @@ jobs:
         python-version: '3.x'
     - name: Display Python version
       run: python -c "import sys; print(sys.version)"
+    - name: Install Doxygen
+      run: apt-get install -y doxygen
     - name: Install docs dependencies
       run: pip install -r docs/requirements.txt
     - name: Get entire project

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -39,8 +39,10 @@ set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O3")
 set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")
 
 # Add -fno-omit-frame-pointer to avoid seeing incomplete stack traces
-set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g -ggdb -fno-omit-frame-pointer -DVALGRIND")
-set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g -ggdb -fno-omit-frame-pointer -DVALGRIND")
+set(CMAKE_COMMON_FLAGS_DEBUG "-g -ggdb -fno-omit-frame-pointer -D_DEBUG -DVALGRIND -include \
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/common.h -I${CMAKE_CURRENT_SOURCE_DIR}/opt")
+set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${CMAKE_COMMON_FLAGS_DEBUG}")
+set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${CMAKE_COMMON_FLAGS_DEBUG}")
 
 #----------------------------------------------------------------------------------------------
 

diff --git a/Dockerfile b/Dockerfile
@@ -1,6 +1,6 @@
 # BUILD redisfab/redisai:${VERSION}-cpu-${ARCH}-${OSNICK}
 
-ARG REDIS_VER=6.0.1
+ARG REDIS_VER=6.0.5
 
 # OSNICK=bionic|stretch|buster
 ARG OSNICK=buster
@@ -31,7 +31,7 @@ COPY --from=redis /usr/local/ /usr/local/
 COPY ./opt/ opt/
 COPY ./test/test_requirements.txt test/
 
-RUN ./opt/readies/bin/getpy3
+RUN PIP=1 FORCE=1 ./opt/readies/bin/getpy3
 RUN ./opt/system-setup.py
 
 ARG DEPS_ARGS=""
@@ -40,17 +40,24 @@ RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh cpu; else env $DEPS_ARGS ./get_
 
 ARG BUILD_ARGS=""
 ADD ./ /build
-RUN set -e ;\
+RUN bash -c "set -e ;\
     . ./opt/readies/bin/sourced ./profile.d ;\
-    make -C opt build $BUILD_ARGS SHOW=1
+    make -C opt build $BUILD_ARGS SHOW=1"
 
 ARG PACK
 ARG TEST
 
 RUN mkdir -p bin/artifacts
-RUN if [ "$PACK" = "1" ]; then make -C opt pack; fi
+RUN set -e ;\
+    if [ "$PACK" = "1" ]; then make -C opt pack; fi
 
-RUN if [ "$TEST" = "1" ]; then TEST= make -C opt test $BUILD_ARGS NO_LFS=1; fi
+RUN set -e ;\
+    if [ "$TEST" = "1" ]; then \
+        TEST= make -C opt test $BUILD_ARGS NO_LFS=1 ;\
+        if [[ -d test/logs ]]; then \
+            tar -C test/logs -czf bin/artifacts/test-logs-cpu.tgz . ;\
+        fi ;\
+    fi
 
 #----------------------------------------------------------------------------------------------
 FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK}

diff --git a/Dockerfile.arm b/Dockerfile.arm
@@ -1,6 +1,6 @@
 # BUILD redisfab/redisai:${VERSION}-cpu-${ARCH}-${OSNICK}
 
-ARG REDIS_VER=6.0.1
+ARG REDIS_VER=6.0.5
 
 # OSNICK=bionic|stretch|buster
 ARG OSNICK=buster

diff --git a/Dockerfile.gpu b/Dockerfile.gpu
@@ -1,16 +1,16 @@
 # BUILD redisfab/redisai:${VERSION}-gpu-${ARCH}-${OSNICK}
 
-ARG REDIS_VER=6.0.1
+ARG REDIS_VER=6.0.5
 
 # OSNICK=bionic|centos7|centos6
 ARG OSNICK=bionic
 
-# ARCH=x64|arm64v8|arm32v7
-ARG ARCH=x64
-
 # OS=ubuntu18.04|ubuntu16.04|centos7
 ARG OS=ubuntu18.04
 
+# ARCH=x64|arm64v8|arm32v7
+ARG ARCH=x64
+
 ARG CUDA_VER=10.1-cudnn7
 
 ARG PACK=0
@@ -22,10 +22,11 @@ FROM nvidia/cuda:${CUDA_VER}-devel-${OS} AS builder
 
 ARG OSNICK
 ARG OS
+ARG ARCH
 ARG REDIS_VER
 ARG CUDA_VER
 
-RUN echo "Building for ${OSNICK} (${OS}) [with Redis ${REDIS_VER}]"
+RUN echo "Building for ${OSNICK} (${OS}) for ${ARCH} [with Redis ${REDIS_VER}]"
 
 ENV NVIDIA_VISIBLE_DEVICES all
 ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
@@ -36,7 +37,7 @@ COPY --from=redis /usr/local/ /usr/local/
 COPY ./opt/ opt/
 COPY ./test/test_requirements.txt test/
 
-RUN ./opt/readies/bin/getpy3
+RUN PIP=1 FORCE=1 ./opt/readies/bin/getpy3
 RUN ./opt/system-setup.py
 
 ARG DEPS_ARGS=""
@@ -45,17 +46,24 @@ RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh gpu; else env $DEPS_ARGS ./get_
 
 ARG BUILD_ARGS=""
 ADD ./ /build
-RUN set -e ;\
+RUN bash -c "set -e ;\
     . ./opt/readies/bin/sourced ./profile.d ;\
-    make -C opt build GPU=1 $BUILD_ARGS SHOW=1
+    make -C opt build GPU=1 $BUILD_ARGS SHOW=1"
 
 ARG PACK
 ARG TEST
 
 RUN mkdir -p bin/artifacts
-RUN if [ "$PACK" = "1" ]; then make -C opt pack GPU=1; fi
+RUN set -e ;\
+    if [ "$PACK" = "1" ]; then make -C opt pack GPU=1; fi
 
-RUN if [ "$TEST" = "1" ]; then TEST= make -C opt test GPU=1 $BUILD_ARGS NO_LFS=1; fi
+RUN set -e ;\
+    if [ "$TEST" = "1" ]; then \
+        TEST= make -C opt test GPU=1 $BUILD_ARGS NO_LFS=1 ;\
+        if [[ -d test/logs ]]; then \
+            tar -C test/logs -czf bin/artifacts/test-logs-gpu.tgz . ;\
+        fi ;\
+    fi
 
 #----------------------------------------------------------------------------------------------
 FROM nvidia/cuda:${CUDA_VER}-runtime-${OS}

diff --git a/Dockerfile.gpu-test b/Dockerfile.gpu-test
@@ -1,6 +1,6 @@
 # BUILD redisfab/redisai:${VERSION}-gpu-${ARCH}-${OSNICK}-test
 
-ARG REDIS_VER=6.0.1
+ARG REDIS_VER=6.0.5
 
 # OSNICK=bionic|centos7|centos6
 ARG OSNICK=bionic
@@ -29,11 +29,12 @@ COPY --from=redis /usr/local/ /usr/local/
 COPY ./opt/ opt/
 COPY ./test/test_requirements.txt test/
 
-RUN ./opt/readies/bin/getpy3
-RUN ./opt/system-setup.py
+RUN PIP=1 VENV=1 FORCE=1 ./opt/readies/bin/getpy3
+
 RUN set -e ;\
-	python3 -m virtualenv venv --system-site-packages;\
+	python3 -m virtualenv venv --system-site-packages ;\
 	. venv/bin/activate ;\
+	python -m pip install pip==19.3.1 ;\
 	./opt/system-setup.py
 
 ARG DEPS_ARGS=""

diff --git a/README.md b/README.md
@@ -130,6 +130,7 @@ RedisAI currently supports PyTorch (libtorch), Tensorflow (libtensorflow), Tenso
 | 0.3.1   | 1.1.0   | 1.12.0     | None   | 0.4.0         |
 | 0.4.0   | 1.2.0   | 1.14.0     | None   | 0.5.0         |
 | 0.9.0   | 1.3.1   | 1.14.0     | 2.0.0  | 1.0.0         |
+| 1.0.0   | 1.5.0   | 1.15.0     | 2.0.0  | 1.2.0         |
 | master  | 1.5.0   | 1.15.0     | 2.0.0  | 1.2.0         |
 
 Note: Keras and TensorFlow 2.x are supported through graph freezing. See [this script](https://github.com/RedisAI/RedisAI/blob/master/test/test_data/tf2-minimal.py) to see how to export a frozen graph from Keras and TensorFlow 2.x. Note that a frozen graph will be executed using the TensorFlow 1.15 backend. Should any 2.0 ops be not supported on the 1.15 after freezing, please open an Issue.