1.2 cherries (#774)

Co-authored-by: tomerhekredis <72793005+tomerhekredis@users.noreply.github.com>
Co-authored-by: Guy Korland <gkorland@gmail.com>
Co-authored-by: alonre24 <alonreshef24@gmail.com>
Co-authored-by: DvirDukhan <dvir@redislabs.com>

Author: 5 people

.circleci/config.yml

@@ -43,6 +43,19 @@ commands:
               circleci step halt
             fi
 
+  only_run_if_forked_pull_request:
+    description: >-
+      If this build is from an internal (i.e not a forked) pull request, then end early.
+      This is required when we want different behaviours for external, versus internal PRs.
+    steps:
+      - run:
+          name: Return early if not a forked pull request
+          command: |
+            if [[ ! -n "$CIRCLE_PR_NUMBER" ]]; then
+              echo "Not a forked PR so marking as successfull and exiting"
+              circleci step halt
+            fi
+
   relocate-docker-storage:
     steps:
       - run:
@@ -127,6 +140,64 @@ commands:
       - store_artifacts:
           path: tests/logs
 
+  valgrind-general-steps:
+    parameters:
+      test_args:
+        type: string
+        default: "CLUSTER=0 AOF=0"
+    steps:
+      - abort_for_docs
+      - abort_for_noci
+      - checkout-all
+      - restore_cache:
+          keys:
+          - build-dependencies-{{ checksum "get_deps.sh" }}
+          # If no exact match is found will get dependencies from source
+      - setup-build-system
+      - run:
+          name: Install dependencies
+          command: |
+            ./opt/readies/bin/getredis -v 6.0 --valgrind --force
+            ./get_deps.sh cpu
+      - run:
+          name: Build for valgrind
+          command: |
+            make -C opt all VALGRIND=1 SHOW=1
+      - run:
+          name: Test with valgrind
+          command: |
+            make -C opt test VALGRIND=1 <<parameters.test_args>>
+          no_output_timeout: 120m
+
+  build-and-test-gpu-steps:
+    steps:
+      - abort_for_docs
+      - abort_for_noci
+      - checkout-all
+      - run:
+          name: Relocate docker overlay2 dir
+          command: |
+            sudo systemctl stop docker
+            sudo mkdir -p /var2/lib/docker
+            sudo mv /var/lib/docker/overlay2 /var2/lib/docker
+            sudo mkdir /var/lib/docker/overlay2
+            sudo mount --bind /var2/lib/docker/overlay2 /var/lib/docker/overlay2
+            sudo systemctl start docker
+      - run:
+          name: Build
+          command: |
+            docker build -f Dockerfile.gpu-test --no-cache -t redisai-gpu:latest-x64-bionic-test .
+      - run:
+          name: Test
+          command: |
+            mkdir -p $HOME/tests
+            docker run --gpus all -v $HOME/tests:/build/tests/logs -it --rm redisai-gpu:latest-x64-bionic-test
+          no_output_timeout: 40m
+      - store_artifacts:
+          path: tests/logs
+
+
+
 jobs:
   lint:
     docker:
@@ -218,7 +289,7 @@ jobs:
     parameters:
       test_args:
         type: string
-        default: "CLUSTER=0 AOF=0"
+        default: "CLUSTER=0"
     docker:
       - image: redisfab/rmbuilder:6.2.1-x64-buster
     steps:
@@ -245,37 +316,18 @@ jobs:
             make -C opt test VALGRIND=1 <<parameters.test_args>>
           no_output_timeout: 120m
 
+  valgrind-general-for-forked-prs:
+    docker:
+      - image: redisfab/rmbuilder:6.2.1-x64-buster
+    steps:
+      - only_run_if_forked_pull_request
+      - valgrind-general-steps
+
   valgrind-general:
-    parameters:
-      test_args:
-        type: string
-        default: "CLUSTER=0 AOF=0"
     docker:
       - image: redisfab/rmbuilder:6.2.1-x64-buster
     steps:
-      - abort_for_docs
-      - abort_for_noci
-      - early_return_for_forked_pull_requests
-      - checkout-all
-      - restore_cache:
-          keys:
-          - build-dependencies-{{ checksum "get_deps.sh" }}
-          # If no exact match is found will get dependencies from source
-      - setup-build-system
-      - run:
-          name: Install dependencies
-          command: |
-            ./opt/readies/bin/getredis -v 6.0 --valgrind --force
-            ./get_deps.sh cpu
-      - run:
-          name: Build for valgrind
-          command: |
-            make -C opt all VALGRIND=1 SHOW=1
-      - run:
-          name: Test with valgrind
-          command: |
-            make -C opt test VALGRIND=1 <<parameters.test_args>>
-          no_output_timeout: 120m
+      - valgrind-general-steps
 
 #  build-macos:
 #    macos:
@@ -316,6 +368,8 @@ jobs:
 #            make build
 #            sudo make publish
 
+  # internal PRs execute build-and-test either in a workflow or
+  # via a github action trigger
   build-and-test-gpu:
     machine:
       enabled: true
@@ -324,30 +378,20 @@ jobs:
       image: ubuntu-1604-cuda-11.1:202012-01
 
     steps:
-      - abort_for_docs
-      - abort_for_noci
-      - checkout-all
-      - run:
-          name: Relocate docker overlay2 dir
-          command: |
-            sudo systemctl stop docker
-            sudo mkdir -p /var2/lib/docker
-            sudo mv /var/lib/docker/overlay2 /var2/lib/docker
-            sudo mkdir /var/lib/docker/overlay2
-            sudo mount --bind /var2/lib/docker/overlay2 /var/lib/docker/overlay2
-            sudo systemctl start docker
-      - run:
-          name: Build
-          command: |
-            docker build -f Dockerfile.gpu-test --no-cache -t redisai-gpu:latest-x64-bionic-test .
-      - run:
-          name: Test
-          command: |
-            mkdir -p $HOME/tests
-            docker run --gpus all -v $HOME/tests:/build/tests/logs -it --rm redisai-gpu:latest-x64-bionic-test
-          no_output_timeout: 40m
-      - store_artifacts:
-          path: tests/logs
+      - build-and-test-gpu-steps
+
+  # in the case of a forked PR, we want to run the GPU steps
+  # hence we check if we're forked, explicitly
+  build-and-test-gpu-for-forked-prs:
+    machine:
+      enabled: true
+      docker_layer_caching: true
+      resource_class: gpu.nvidia.small
+      image: ubuntu-1604-cuda-11.1:202012-01
+
+    steps:
+      - only_run_if_forked_pull_request
+      - build-and-test-gpu-steps
 
   deploy-artifacts:
     parameters:
@@ -414,16 +458,20 @@ jobs:
       - run:
           name: Run QA Automation
           command: MODULE_VERSION=$CIRCLE_TAG VERBOSE=1 TEST=release ./tests/qa/run
-
-  nightly-automation:
-    docker:
-      - image: redisfab/rmbuilder:6.2.1-x64-buster
-    steps:
-      - checkout
-      - setup-automation
       - run:
-          name: Run QA Automation
-          command: MODULE_VERSION=$CIRCLE_BRANCH VERBOSE=1 TEST=nightly QUICK=1 ./tests/qa/run
+          name: Run QA Automation (AILite)
+          command: MODULE_VERSION=$CIRCLE_TAG VERBOSE=1 TEST=release VARIANT=lite ./tests/qa/run
+
+
+#  nightly-automation:
+#    docker:
+#      - image: redisfab/rmbuilder:6.2.1-x64-buster
+#    steps:
+#      - checkout
+#      - setup-automation
+#      - run:
+#          name: Run QA Automation
+#          command: MODULE_VERSION=$CIRCLE_BRANCH VERBOSE=1 TEST=nightly QUICK=1 ./tests/qa/run
 
 
 on-any-branch: &on-any-branch
@@ -512,6 +560,12 @@ workflows:
       - build-and-test:
           <<: *on-any-branch
           <<: *after-linter
+      - build-and-test-gpu:
+          <<: *on-integ-branch
+          <<: *after-linter
+      - build-and-test-gpu-for-forked-prs:
+          <<: *on-any-branch
+          <<: *after-linter
       - platforms-build:
           <<: *after-build-and-test
           <<: *on-master-version-tags-and-dockertests
@@ -532,16 +586,11 @@ workflows:
           <<: *after-linter
       - valgrind:
           name: valgrind-cluster
-          test_args: GEN=0 AOF=0
+          test_args: GEN=0
           <<: *on-integ-branch
           <<: *after-linter
-      - valgrind:
-          name: valgrind-aof
-          test_args: GEN=0 CLUSTER=0
-          <<: *on-integ-branch
-          <<: *after-linter
-      - build-and-test-gpu:
-          <<: *on-integ-branch
+      - valgrind-general-for-forked-prs:
+          <<: *on-any-branch
           <<: *after-linter
       - deploy-snapshot:
           context: common
@@ -557,15 +606,15 @@ workflows:
           requires:
             - deploy-release
 
-  nightly:
-    triggers:
-      - schedule:
-          cron: "20 17 * * *"
-          filters:
-            branches:
-              only: master
-    jobs:
+#  nightly:
+#    triggers:
+#      - schedule:
+#          cron: "20 17 * * *"
+#          filters:
+#            branches:
+#              only: master
+#    jobs:
 #      - build-macos:
 #          <<: *never # temporarily disabled
-      - nightly-automation:
-          context: common
+#      - nightly-automation:
+#          context: common

Dockerfile

@@ -1,6 +1,6 @@
 # BUILD redisfab/redisai:${VERSION}-cpu-${ARCH}-${OSNICK}
 
-ARG REDIS_VER=6.0.9
+ARG REDIS_VER=6.2.3
 
 # OSNICK=bionic|stretch|buster
 ARG OSNICK=buster
@@ -37,6 +37,9 @@ ADD ./tests/flow/ tests/flow/
 
 RUN FORCE=1 ./opt/readies/bin/getpy3
 RUN ./opt/system-setup.py
+RUN apt-get update -qq
+RUN apt-get upgrade -yqq
+RUN rm -rf /var/cache/apt
 
 ARG DEPS_ARGS=""
 COPY ./get_deps.sh .

Dockerfile.arm

@@ -1,6 +1,6 @@
 # BUILD redisfab/redisai:${VERSION}-cpu-${ARCH}-${OSNICK}
 
-ARG REDIS_VER=6.0.9
+ARG REDIS_VER=6.2.3
 
 # OSNICK=bionic|stretch|buster
 ARG OSNICK=buster

Dockerfile.gpu

@@ -1,6 +1,6 @@
 # BUILD redisfab/redisai:${VERSION}-gpu-${ARCH}-${OSNICK}
 
-ARG REDIS_VER=6.0.9
+ARG REDIS_VER=6.2.3
 
 # OSNICK=bionic|centos7|centos6
 ARG OSNICK=bionic
@@ -42,6 +42,9 @@ ADD ./tests/flow/ tests/flow/
 
 RUN FORCE=1 ./opt/readies/bin/getpy3
 RUN ./opt/system-setup.py
+RUN apt-get update -qq
+RUN apt-get upgrade -yqq
+RUN rm -rf /var/cache/apt
 
 ARG DEPS_ARGS=""
 COPY ./get_deps.sh .

Dockerfile.gpu-test

@@ -1,6 +1,6 @@
 # BUILD redisfab/redisai:${VERSION}-gpu-${ARCH}-${OSNICK}-test
 
-ARG REDIS_VER=6.0.9
+ARG REDIS_VER=6.2.3
 
 # OSNICK=bionic|centos7|centos6
 ARG OSNICK=bionic

Dockerfile.jetson

@@ -1,6 +1,6 @@
 # BUILD redisfab/redisai:${VERSION}-jetson-${ARCH}-${OSNICK}
 
-ARG REDIS_VER=6.0.9
+ARG REDIS_VER=6.2.3
 
 # OSNICK=bionic|centos7|centos6
 ARG OSNICK=bionic

opt/Makefile

@@ -240,7 +240,6 @@ endif
 
 export GEN ?= 1
 export SLAVES ?= 1
-export AOF ?= 1
 export CLUSTER ?= 1
 
 define UNIT_TESTS
@@ -256,7 +255,7 @@ $(SHOW)\
 DEVICE=$(DEVICE) \
 MODULE=$(realpath $(INSTALLED_TARGET)) \
 CLUSTER=$(CLUSTER) \
-GEN=$(GEN) AOF=$(AOF) SLAVES=$(SLAVES) \
+GEN=$(GEN) SLAVES=$(SLAVES) \
 VALGRIND=$(VALGRIND) \
 $(ROOT)/tests/flow/tests_setup/tests.sh
 endef
@@ -272,7 +271,7 @@ flow_tests: build
 		DEVICE=$(DEVICE) \
 		MODULE=$(realpath $(INSTALLED_TARGET)) \
 		CLUSTER=$(CLUSTER) \
-		GEN=$(GEN) AOF=$(AOF) SLAVES=$(SLAVES) \
+		GEN=$(GEN) SLAVES=$(SLAVES) \
 		VALGRIND=$(VALGRIND) \
 		REDIS=$(REDIS) \
 		$(ROOT)/tests/flow/tests_setup/tests.sh
@@ -290,7 +289,7 @@ test: build
 		MODULE=$(realpath $(INSTALLED_TARGET)) \
 		TESTMOD=$(realpath $(BINDIR)/tests/module/testmod.so) \
 		CLUSTER=$(CLUSTER) \
-		GEN=$(GEN) AOF=$(AOF) SLAVES=$(SLAVES) \
+		GEN=$(GEN) SLAVES=$(SLAVES) \
 		VALGRIND=$(VALGRIND) \
 		REDIS=$(REDIS) \
 		$(ROOT)/tests/flow/tests_setup/tests.sh

opt/build/docker/Makefile

@@ -11,7 +11,7 @@ REPO=redisai
 # LATEST_BRANCH=1.2
 INT_BRANCHES=1.2
 
-REDIS_VER:=6.0.9
+REDIS_VER:=6.2.3
 REDISAI_LITE:=0
 
 export ART_DIR=$(ROOT)/bin/artifacts