CircleCI: multiarch docker build + selective engines build

.circleci/config.yml

@@ -7,18 +7,18 @@ commands:
         type: string
     steps:
       - checkout
-      - restore_cache:
-          keys:
-          - v1-dependencies-{{ checksum "get_deps.sh" }}
-          # fallback to using the latest cache if no exact match is found
-          # - v1-dependencies-
+#      - restore_cache:
+#          keys:
+#          - v1-dependencies-{{ checksum "get_deps.sh" }}
+#          # fallback to using the latest cache if no exact match is found
+#          # - v1-dependencies-
       - run:
           name: Install dependencies
           command: |
-            sudo ./opt/readies/bin/getpy
-            sudo ./opt/system-setup.py
-            git clone git://github.com/antirez/redis.git --branch 5.0.5
-            (cd redis && make malloc=libc -j $(nproc) && sudo make install)
+            ./opt/readies/bin/getpy
+            ./opt/system-setup.py
+            # git clone git://github.com/antirez/redis.git --branch 5.0.5
+            # (cd redis && make malloc=libc -j $(nproc) && make install)
             ./get_deps.sh cpu
       - save_cache:
           paths:
@@ -27,29 +27,29 @@ commands:
       - run:
           name: Set up workspace
           command: |
-            sudo mkdir -p /workspace
-            sudo chown `whoami` /workspace
+            mkdir -p ~/workspace
+            chown `whoami` ~/workspace
       - run:
           name: Build
           command: make -C opt all SHOW=1
       - run:
           name: Test
           command: |
-            mkdir -p /workspace/tests
+            mkdir -p ~/workspace/tests
             make -C opt test SHOW=1
-            cp test/logs/* /workspace/tests
+            cp test/logs/* ~/workspace/tests
       - run:
           name: Package
-          command: make -C opt pack BRANCH="${CIRCLE_BRANCH//[^A-Za-z0-9._-]/_}" INTO=/workspace/packages SHOW=1
+          command: make -C opt pack BRANCH="${CIRCLE_BRANCH//[^A-Za-z0-9._-]/_}" INTO=~/workspace/packages SHOW=1
       - persist_to_workspace:
-          root: /workspace
+          root: ~/workspace
           paths:
             - 'packages/release/*.zip'
             - 'packages/release/*.tgz'
             - 'packages/branch/*.zip'
             - 'packages/branch/*.tgz'
       - store_test_results:
-          path: /workspace/tests
+          path: ~/workspace/tests
   deploy:
     parameters:
       from:
@@ -63,9 +63,7 @@ commands:
 jobs:
   build:
     docker:
-      - image: circleci/python:3.7.4-buster
-        environment:
-          DEPS: ""
+      - image: redisfab/rmbuilder:x64-buster
     steps:
       - ci_steps:
           platform: debian 
@@ -75,24 +73,41 @@ jobs:
     steps:
       - ci_steps:
           platform: macosx
+  build-multiarch-docker:
+    machine:
+      enabled: true
+      image: ubuntu-1604:201903-01
+    steps:
+      - checkout
+      - run:
+          name: Setup Docker client experimental features
+          command: |
+            sudo ./opt/readies/bin/getdocker --just-enable-exp
+            docker version
+      - run:
+          name: Build
+          command: |
+            sudo docker login -u redisfab -p $DOCKER_REDISFAB_PWD
+            make -C opt/build/docker build
+            sudo make -C opt/build/docker publish
 
   deploy_package:
     parameters:
       package:
         type: string
     docker:
-      - image: 'redislabsmodules/rmbuilder:latest'
+      - image: redisfab/rmbuilder:x64-buster
     steps:
       - attach_workspace:
-          at: /workspace
+          at: ~/workspace
       - deploy:
-          from: /workspace/packages/<<parameters.package>>
+          from: ~/workspace/packages/<<parameters.package>>
       - store_artifacts:
-          path: /workspace/packages/<<parameters.package>>
+          path: ~/workspace/packages/<<parameters.package>>
 
   deploy_docs:
     docker:
-      - image: 'redislabsmodules/rmbuilder:latest'
+      - image: redisfab/rmbuilder:x64-buster
     steps:
       - checkout
       - run:
@@ -111,16 +126,23 @@ workflows:
           filters:
             tags:
               only: /.*/
-#      - build-macos:
-#          filters:
-#            tags:
-#              only: /.*/
+      - build-macos:
+          filters:
+            tags:
+              only: /.*/
+      - build-multiarch-docker:
+          filters:
+            tags:
+              only: /.*/
+            branches:
+              only: master
       - deploy_package:
           name: deploy_branch
           package: branch
           requires:
             - build
           filters:
+#             # uncomment to debug
 #            tags:
 #              only: /.*/
             branches:

CMakeLists.txt

@@ -20,8 +20,8 @@ SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC")
 SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
 
 # For adding specific Release flags
-set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
-set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
+set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O3")
+set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")
 
 # Add -fno-omit-frame-pointer to avoid seeing incomplete stack traces
 set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g -ggdb -fno-omit-frame-pointer")
@@ -93,9 +93,15 @@ ADD_DEFINITIONS(-DREDISMODULE_EXPERIMENTAL_API)
 IF(BUILD_TF)
     FIND_LIBRARY(TF_LIBRARIES NAMES tensorflow
             PATHS ${depsAbs}/libtensorflow/lib)
-    MESSAGE(STATUS "Found TensorFlow Libraries: \"${TF_LIBRARIES}\")")
     IF (NOT TF_LIBRARIES)
-        MESSAGE(FATAL_ERROR "Could not find tensorflow in ${depsAbs}/libtensorflow/lib")
+        MESSAGE(WARNING "Could not find TensorFlow in ${depsAbs}/libtensorflow/lib. Trying find_package method")
+        FIND_PACKAGE(TensorFlow REQUIRED)
+        SET(TF_LIBRARIES ${TensorFlow_LIBRARY})
+        IF (NOT TF_LIBRARIES)
+            MESSAGE(FATAL_ERROR "Could not find TensorFlow")
+        ELSE()
+            MESSAGE(STATUS "Found TensorFlow Libraries: \"${TF_LIBRARIES}\")")
+        ENDIF()
     ENDIF()
 ENDIF()
 
@@ -192,7 +198,7 @@ IF(BUILD_TORCH)
     SET_TARGET_PROPERTIES(redisai_torch PROPERTIES SUFFIX ".so")
     IF (APPLE)
         SET_TARGET_PROPERTIES(redisai_torch PROPERTIES INSTALL_RPATH "@loader_path/lib")
-    ELSE ()
+    ELSE()
         ADD_LDFLAGS(redisai_torch "-Wl,--enable-new-dtags")
         SET_TARGET_PROPERTIES(redisai_torch PROPERTIES INSTALL_RPATH "\$ORIGIN/lib")
     ENDIF()

Dockerfile

@@ -22,11 +22,13 @@ COPY ./test/test_requirements.txt test/
 RUN ./opt/readies/bin/getpy
 RUN ./opt/system-setup.py
 
+ARG DEPS_ARGS=""
 COPY ./get_deps.sh .
-RUN ./get_deps.sh cpu
+RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh cpu; else env $DEPS_ARGS ./get_deps.sh cpu; fi
 
+ARG BUILD_ARGS=""
 ADD ./ /build
-RUN make -C opt build SHOW=1
+RUN make -C opt build $BUILD_ARGS SHOW=1
 
 ARG PACK=0
 ARG TEST=0

Dockerfile.arm

@@ -4,7 +4,7 @@
 ARG OSNICK=buster
 
 # OS=debian:buster-slim|debian:stretch-slim|ubuntu:bionic
-OS=debian:buster-slim
+ARG OS=debian:buster-slim
 
 # ARCH=arm64v8|arm32v7
 ARG ARCH=arm64v8
@@ -14,7 +14,6 @@ FROM redisfab/redis-${ARCH}-${OSNICK}-xbuild:5.0.5 AS builder
 
 RUN [ "cross-build-start" ]
 
-ADD ./ /build
 WORKDIR /build
 
 COPY ./opt/ opt/
@@ -23,11 +22,13 @@ COPY ./test/test_requirements.txt test/
 RUN ./opt/readies/bin/getpy
 RUN ./opt/system-setup.py
 
+ARG DEPS_ARGS=""
 COPY ./get_deps.sh .
-RUN ./get_deps.sh cpu
+RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh cpu; else env $DEPS_ARGS ./get_deps.sh cpu; fi
 
-ADD ./ /redisai
-RUN make -C opt all SHOW=1
+ARG BUILD_ARGS=""
+ADD ./ /build
+RUN make -C opt build $BUILD_ARGS SHOW=1
 
 ARG PACK=0
 ARG TEST=0