From 80fa87771236abf47e60235311acfba1cd4a8f95 Mon Sep 17 00:00:00 2001 From: rafie Date: Sat, 30 Jan 2021 22:48:25 +0200 Subject: [PATCH 1/7] CircleCI: dockerhub builds --- .circleci/config.yml | 148 +++++++++++++++++++------------------- opt/build/docker/Makefile | 6 ++ opt/readies | 2 +- 3 files changed, 81 insertions(+), 75 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8e4a254a4..77dadfd74 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -106,6 +106,29 @@ commands: - store_artifacts: path: test/logs + dockerhub-build-steps: + parameters: + build_args: + type: string + steps: + - setup-executor + - checkout + - run: + name: Setup + command: | + git submodule update --init --recursive + ./opt/readies/bin/getpy3 + - run: + name: Build for platform + command: | + #@@ docker login -u redisai -p $DOCKER_REDISAI_PWD + cd opt/build/docker + #@@ make build publish $(./version-params) CPU=1 OSNICK=bionic X64=1 REPO=redisai VERBOSE=1 <> + #@@ make build publish $(./version-params) GPU=1 OSNICK=bionic X64=1 REPO=redisai VERBOSE=1 <> + make build $(./version-params) CPU=1 OSNICK=bionic X64=1 REPO=redisai VERBOSE=1 <> + make build $(./version-params) GPU=1 OSNICK=bionic X64=1 REPO=redisai VERBOSE=1 <> + no_output_timeout: 40m + deploy-steps: parameters: from: @@ -131,9 +154,9 @@ jobs: command: | make -C opt lint - build-debian: + build-and-test: docker: - - image: redisfab/rmbuilder:6.0.5-x64-buster + - image: redisfab/rmbuilder:6.0.9-x64-buster steps: - build-steps: platform: debian @@ -150,7 +173,7 @@ jobs: coverage: docker: - - image: redisfab/rmbuilder:6.0.5-x64-buster + - image: redisfab/rmbuilder:6.0.9-x64-buster steps: - checkout - run: @@ -178,8 +201,12 @@ jobs: no_output_timeout: 30m valgrind: + parameters: + test_args: + type: string + default: "CLUSTER=0 AOF=0" docker: - - image: redisfab/rmbuilder:6.0.5-x64-buster + - image: redisfab/rmbuilder:6.0.9-x64-buster steps: - checkout - run: @@ -202,63 +229,7 @@ jobs: - run: name: Test with valgrind command: | - make -C opt test VALGRIND=1 CLUSTER=0 AOF=0 - no_output_timeout: 120m - - valgrind-cluster: - docker: - - image: redisfab/rmbuilder:6.0.5-x64-buster - steps: - - checkout - - run: - name: Submodule checkout - command: git submodule update --init --recursive - - restore_cache: - keys: - - build-dependencies-{{ checksum "get_deps.sh" }} - # If no exact match is found will get dependencies from source - - setup-automation - - run: - name: Install dependencies - command: | - ./opt/readies/bin/getredis -v 6 --valgrind --force - ./get_deps.sh cpu - - run: - name: Build for valgrind with cluster - command: | - make -C opt all VALGRIND=1 SHOW=1 - - run: - name: Test with valgrind and cluster - command: | - make -C opt test VALGRIND=1 GEN=0 AOF=0 - no_output_timeout: 120m - - valgrind-AOF: - docker: - - image: redisfab/rmbuilder:6.0.5-x64-buster - steps: - - checkout - - run: - name: Submodule checkout - command: git submodule update --init --recursive - - restore_cache: - keys: - - build-dependencies-{{ checksum "get_deps.sh" }} - # If no exact match is found will get dependencies from source - - setup-automation - - run: - name: Install dependencies - command: | - ./opt/readies/bin/getredis -v 6 --valgrind --force - ./get_deps.sh cpu - - run: - name: Build for valgrind with AOF - command: | - make -C opt all VALGRIND=1 SHOW=1 - - run: - name: Test with valgrind and AOF - command: | - make -C opt test VALGRIND=1 GEN=0 CLUSTER=0 + make -C opt test VALGRIND=1 <> no_output_timeout: 120m build-macos: @@ -325,12 +296,22 @@ jobs: - store_artifacts: path: test/log + dockerhub-build: + parameters: + build_args: + type: string + docker: + - image: debian:buster + steps: + - dockerhub-build-steps: + build_args: <> + deploy-artifacts: parameters: location: type: string docker: - - image: redisfab/rmbuilder:6.0.5-x64-buster + - image: redisfab/rmbuilder:6.0.9-x64-buster steps: - attach_workspace: at: workspace @@ -344,7 +325,7 @@ jobs: deploy-snapshot: docker: - - image: redisfab/rmbuilder:6.0.5-x64-buster + - image: redisfab/rmbuilder:6.0.9-x64-buster steps: - attach_workspace: at: workspace @@ -362,7 +343,7 @@ jobs: deploy-release: docker: - - image: redisfab/rmbuilder:6.0.5-x64-buster + - image: redisfab/rmbuilder:6.0.9-x64-buster steps: - attach_workspace: at: workspace @@ -422,11 +403,13 @@ on-integ-and-version-tags: &on-integ-and-version-tags only: /^v[0-9].*/ -platform-build-defs: &platform-build-defs +after-linter: &after-linter requires: - - build-debian - <<: *on-integ-and-version-tags - # <<: *on-any-branch # debugging + - lint + +after-build-and-test: &after-build-and-test + requires: + - build-and-test after-platform-builds: &after-platform-builds requires: @@ -434,9 +417,11 @@ after-platform-builds: &after-platform-builds - build-bionic - build-xenial -after-linter: &after-linter - requires: - - lint +platform-build-defs: &platform-build-defs + <<: *after-build-and-test + <<: *on-integ-and-version-tags + # <<: *on-any-branch # debugging + workflows: version: 2 @@ -444,9 +429,14 @@ workflows: jobs: - lint: <<: *on-any-branch - - build-debian: + - build-and-test: <<: *on-any-branch <<: *after-linter + - dockerhub-build: + name: dockerhub-edge + build_args: EDGE=1 + <<: *after-build-and-test + <<: *on-any-branch - platform-build: name: build-centos7 platform: centos7 @@ -463,12 +453,17 @@ workflows: <<: *on-any-branch <<: *after-linter - valgrind: + test_args: CLUSTER=0 AOF=0 <<: *on-any-branch <<: *after-linter - - valgrind-cluster: + - valgrind: + name: valgrind-cluster + test_args: GEN=0 AOF=0 <<: *on-integ-branch <<: *after-linter - - valgrind-AOF: + - valgrind: + name: valgrind-aof + test_args: GEN=0 CLUSTER=0 <<: *on-integ-branch <<: *after-linter - build-and-test-gpu: @@ -486,6 +481,11 @@ workflows: - deploy-release: <<: *after-platform-builds <<: *on-version-tags + - dockerhub-build: + name: dockerhub-release + build_args: LATEST=1 + <<: *after-platform-builds + <<: *on-version-tags nightly: triggers: diff --git a/opt/build/docker/Makefile b/opt/build/docker/Makefile index 5833629dc..b3a76c56d 100755 --- a/opt/build/docker/Makefile +++ b/opt/build/docker/Makefile @@ -6,7 +6,13 @@ include $(ROOT)/opt/readies/mk/main # OSNICK=buster|stretch|bionic|xenial|centos7 OSNICK ?= buster +ifeq ($(REPO),redisai) +REPO=redisai +SIMPLE ?= 1 +else REPO=redisfab +endif + STEM=$(REPO)/redisai REDIS_VER:=6.0.9 diff --git a/opt/readies b/opt/readies index b300e14f4..aa215168b 160000 --- a/opt/readies +++ b/opt/readies @@ -1 +1 @@ -Subproject commit b300e14f44c80bf7ca0e74cf60be22720da49e9e +Subproject commit aa215168b8e7b462afd9b53f12666f3f6cb38480 From 96b201d66c5dc3363b0ad1cea6ebedd4611c815a Mon Sep 17 00:00:00 2001 From: rafie Date: Sun, 31 Jan 2021 00:19:46 +0200 Subject: [PATCH 2/7] fixes 1 --- .circleci/config.yml | 20 ++++++++++++-------- opt/build/docker/Makefile | 2 ++ opt/readies | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 77dadfd74..84dce049c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -432,11 +432,6 @@ workflows: - build-and-test: <<: *on-any-branch <<: *after-linter - - dockerhub-build: - name: dockerhub-edge - build_args: EDGE=1 - <<: *after-build-and-test - <<: *on-any-branch - platform-build: name: build-centos7 platform: centos7 @@ -450,11 +445,13 @@ workflows: platform: xenial <<: *platform-build-defs - coverage: - <<: *on-any-branch + #@@ <<: *on-any-branch <<: *after-linter + <<: *never #@@ remove this - valgrind: test_args: CLUSTER=0 AOF=0 - <<: *on-any-branch + #@@ <<: *on-any-branch + <<: *never #@@ remove this <<: *after-linter - valgrind: name: valgrind-cluster @@ -467,7 +464,8 @@ workflows: <<: *on-integ-branch <<: *after-linter - build-and-test-gpu: - <<: *on-any-branch + #@@<<: *on-any-branch + <<: *never #@@ remove this <<: *after-linter - build-macos: <<: *never # temporarily disabled @@ -478,6 +476,12 @@ workflows: - deploy-snapshot: <<: *after-platform-builds <<: *on-integ-branch + - dockerhub-build: + name: dockerhub-edge + build_args: EDGE=1 + <<: *after-build-and-test + #@@ <<: *on-master + <<: *on-any-branch - deploy-release: <<: *after-platform-builds <<: *on-version-tags diff --git a/opt/build/docker/Makefile b/opt/build/docker/Makefile index b3a76c56d..86ac59dce 100755 --- a/opt/build/docker/Makefile +++ b/opt/build/docker/Makefile @@ -15,6 +15,8 @@ endif STEM=$(REPO)/redisai +LATEST_BRANCH=1.2 + REDIS_VER:=6.0.9 ART_DIR=$(ROOT)/bin/artifacts diff --git a/opt/readies b/opt/readies index aa215168b..ff19955b6 160000 --- a/opt/readies +++ b/opt/readies @@ -1 +1 @@ -Subproject commit aa215168b8e7b462afd9b53f12666f3f6cb38480 +Subproject commit ff19955b643b3da2dd4609dbaea304a469b80de1 From 6a37d3ec6069aabd45e1f833cb628825bb229449 Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 2 Feb 2021 21:23:48 +0200 Subject: [PATCH 3/7] fixes 2 --- .circleci/config.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 84dce049c..fba0620a3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -121,12 +121,10 @@ commands: - run: name: Build for platform command: | - #@@ docker login -u redisai -p $DOCKER_REDISAI_PWD + docker login -u redisfab -p $DOCKER_REDISFAB_PWD cd opt/build/docker - #@@ make build publish $(./version-params) CPU=1 OSNICK=bionic X64=1 REPO=redisai VERBOSE=1 <> - #@@ make build publish $(./version-params) GPU=1 OSNICK=bionic X64=1 REPO=redisai VERBOSE=1 <> - make build $(./version-params) CPU=1 OSNICK=bionic X64=1 REPO=redisai VERBOSE=1 <> - make build $(./version-params) GPU=1 OSNICK=bionic X64=1 REPO=redisai VERBOSE=1 <> + make build publish $(./version-params) CPU=1 OSNICK=bionic X64=1 REPO=redisai VERBOSE=1 <> + make build publish $(./version-params) GPU=1 OSNICK=bionic X64=1 REPO=redisai VERBOSE=1 <> no_output_timeout: 40m deploy-steps: @@ -445,6 +443,7 @@ workflows: platform: xenial <<: *platform-build-defs - coverage: + context: common #@@ <<: *on-any-branch <<: *after-linter <<: *never #@@ remove this @@ -471,22 +470,27 @@ workflows: <<: *never # temporarily disabled # <<: *on-version-tags - build-multiarch-docker: + context: common <<: *never # temporarily disabled # <<: *on-version-tags - deploy-snapshot: + context: common <<: *after-platform-builds <<: *on-integ-branch - dockerhub-build: name: dockerhub-edge + context: common build_args: EDGE=1 <<: *after-build-and-test #@@ <<: *on-master <<: *on-any-branch - deploy-release: + context: common <<: *after-platform-builds <<: *on-version-tags - dockerhub-build: name: dockerhub-release + context: common build_args: LATEST=1 <<: *after-platform-builds <<: *on-version-tags From dbe866162c95c7eaa545441469c381bde1317530 Mon Sep 17 00:00:00 2001 From: rafie Date: Wed, 3 Feb 2021 08:41:42 +0200 Subject: [PATCH 4/7] fixes 3 --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fba0620a3..df10a6dff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -470,27 +470,27 @@ workflows: <<: *never # temporarily disabled # <<: *on-version-tags - build-multiarch-docker: - context: common + context: common <<: *never # temporarily disabled # <<: *on-version-tags - deploy-snapshot: - context: common + context: common <<: *after-platform-builds <<: *on-integ-branch - dockerhub-build: name: dockerhub-edge - context: common + context: common build_args: EDGE=1 <<: *after-build-and-test #@@ <<: *on-master <<: *on-any-branch - deploy-release: - context: common + context: common <<: *after-platform-builds <<: *on-version-tags - dockerhub-build: name: dockerhub-release - context: common + context: common build_args: LATEST=1 <<: *after-platform-builds <<: *on-version-tags From 09b07f5895d17d2bd4f36dd9d8023a2ef8570f5c Mon Sep 17 00:00:00 2001 From: rafie Date: Wed, 3 Feb 2021 08:48:05 +0200 Subject: [PATCH 5/7] fixes 4 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index df10a6dff..c9638a387 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -443,7 +443,7 @@ workflows: platform: xenial <<: *platform-build-defs - coverage: - context: common + context: common #@@ <<: *on-any-branch <<: *after-linter <<: *never #@@ remove this From a2b07cdee5e040cae33857f2139d52aff8e48186 Mon Sep 17 00:00:00 2001 From: rafie Date: Wed, 3 Feb 2021 23:14:54 +0200 Subject: [PATCH 6/7] fixes 5 --- .circleci/config.yml | 4 ++-- Dockerfile | 4 ++-- Dockerfile.arm | 2 +- Dockerfile.gpu | 4 ++-- Dockerfile.gpu-test | 6 ++---- Dockerfile.jetson | 2 +- opt/build/docker/Makefile | 3 +-- 7 files changed, 11 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c9638a387..18a90fdda 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -123,8 +123,8 @@ commands: command: | docker login -u redisfab -p $DOCKER_REDISFAB_PWD cd opt/build/docker - make build publish $(./version-params) CPU=1 OSNICK=bionic X64=1 REPO=redisai VERBOSE=1 <> - make build publish $(./version-params) GPU=1 OSNICK=bionic X64=1 REPO=redisai VERBOSE=1 <> + make -n build publish $(./version-params) CPU=1 OSNICK=bionic X64=1 REPO=redislabs VERBOSE=1 <> + make -n build publish $(./version-params) GPU=1 OSNICK=bionic X64=1 REPO=redislabs VERBOSE=1 <> no_output_timeout: 40m deploy-steps: diff --git a/Dockerfile b/Dockerfile index 14abc1b72..7e9fb6e3a 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # BUILD redisfab/redisai:${VERSION}-cpu-${ARCH}-${OSNICK} -ARG REDIS_VER=6.0.5 +ARG REDIS_VER=6.0.9 # OSNICK=bionic|stretch|buster ARG OSNICK=buster @@ -31,7 +31,7 @@ COPY --from=redis /usr/local/ /usr/local/ COPY ./opt/ opt/ COPY ./tests/flow/test_requirements.txt tests/flow/ -RUN PIP=1 FORCE=1 ./opt/readies/bin/getpy3 +RUN FORCE=1 ./opt/readies/bin/getpy3 RUN ./opt/system-setup.py ARG DEPS_ARGS="" diff --git a/Dockerfile.arm b/Dockerfile.arm index 074d52a9c..0e47fd439 100755 --- a/Dockerfile.arm +++ b/Dockerfile.arm @@ -1,6 +1,6 @@ # BUILD redisfab/redisai:${VERSION}-cpu-${ARCH}-${OSNICK} -ARG REDIS_VER=6.0.5 +ARG REDIS_VER=6.0.9 # OSNICK=bionic|stretch|buster ARG OSNICK=buster diff --git a/Dockerfile.gpu b/Dockerfile.gpu index 8b4ba8136..e5b6f1d26 100644 --- a/Dockerfile.gpu +++ b/Dockerfile.gpu @@ -1,6 +1,6 @@ # BUILD redisfab/redisai:${VERSION}-gpu-${ARCH}-${OSNICK} -ARG REDIS_VER=6.0.5 +ARG REDIS_VER=6.0.9 # OSNICK=bionic|centos7|centos6 ARG OSNICK=bionic @@ -37,7 +37,7 @@ COPY --from=redis /usr/local/ /usr/local/ COPY ./opt/ opt/ COPY ./tests/flow/test_requirements.txt tests/flow/ -RUN PIP=1 FORCE=1 ./opt/readies/bin/getpy3 +RUN FORCE=1 ./opt/readies/bin/getpy3 RUN ./opt/system-setup.py ARG DEPS_ARGS="" diff --git a/Dockerfile.gpu-test b/Dockerfile.gpu-test index 0b6c3fdf7..e9ff07162 100644 --- a/Dockerfile.gpu-test +++ b/Dockerfile.gpu-test @@ -1,6 +1,6 @@ # BUILD redisfab/redisai:${VERSION}-gpu-${ARCH}-${OSNICK}-test -ARG REDIS_VER=6.0.5 +ARG REDIS_VER=6.0.9 # OSNICK=bionic|centos7|centos6 ARG OSNICK=bionic @@ -30,12 +30,10 @@ COPY --from=redis /usr/local/ /usr/local/ COPY ./opt/ opt/ COPY ./tests/flow/test_requirements.txt tests/flow/ -RUN PIP=1 VENV=1 FORCE=1 ./opt/readies/bin/getpy3 +RUN VENV=venv FORCE=1 ./opt/readies/bin/getpy3 RUN set -e ;\ - python3 -m virtualenv venv --system-site-packages ;\ . venv/bin/activate ;\ - python -m pip install pip==19.3.1 ;\ ./opt/system-setup.py ARG DEPS_ARGS="" diff --git a/Dockerfile.jetson b/Dockerfile.jetson index c2529b680..99ea6243f 100644 --- a/Dockerfile.jetson +++ b/Dockerfile.jetson @@ -40,7 +40,7 @@ COPY --from=redis /usr/local/ /usr/local/ COPY ./opt/ opt/ COPY ./tests/flow/test_requirements.txt tests/flow/ -RUN PIP=1 FORCE=1 ./opt/readies/bin/getpy3 +RUN FORCE=1 ./opt/readies/bin/getpy3 RUN ./opt/system-setup.py ARG DEPS_ARGS="GPU=1 JETSON=1 WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0" diff --git a/opt/build/docker/Makefile b/opt/build/docker/Makefile index 86ac59dce..44ec61812 100755 --- a/opt/build/docker/Makefile +++ b/opt/build/docker/Makefile @@ -6,8 +6,7 @@ include $(ROOT)/opt/readies/mk/main # OSNICK=buster|stretch|bionic|xenial|centos7 OSNICK ?= buster -ifeq ($(REPO),redisai) -REPO=redisai +ifeq ($(REPO),redislabs) SIMPLE ?= 1 else REPO=redisfab From 79243577813bbac6a499da440bcf7748a47308cb Mon Sep 17 00:00:00 2001 From: rafie Date: Thu, 4 Feb 2021 17:10:04 +0200 Subject: [PATCH 7/7] fixes 6 --- .circleci/config.yml | 16 ++++++---------- opt/readies | 2 +- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 18a90fdda..f0025c41e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -123,8 +123,8 @@ commands: command: | docker login -u redisfab -p $DOCKER_REDISFAB_PWD cd opt/build/docker - make -n build publish $(./version-params) CPU=1 OSNICK=bionic X64=1 REPO=redislabs VERBOSE=1 <> - make -n build publish $(./version-params) GPU=1 OSNICK=bionic X64=1 REPO=redislabs VERBOSE=1 <> + make build publish $(./version-params) CPU=1 OSNICK=bionic X64=1 REPO=redislabs VERBOSE=1 <> + make build publish $(./version-params) GPU=1 OSNICK=bionic X64=1 REPO=redislabs VERBOSE=1 <> no_output_timeout: 40m deploy-steps: @@ -444,13 +444,11 @@ workflows: <<: *platform-build-defs - coverage: context: common - #@@ <<: *on-any-branch + <<: *on-any-branch <<: *after-linter - <<: *never #@@ remove this - valgrind: test_args: CLUSTER=0 AOF=0 - #@@ <<: *on-any-branch - <<: *never #@@ remove this + <<: *on-any-branch <<: *after-linter - valgrind: name: valgrind-cluster @@ -463,8 +461,7 @@ workflows: <<: *on-integ-branch <<: *after-linter - build-and-test-gpu: - #@@<<: *on-any-branch - <<: *never #@@ remove this + <<: *on-any-branch <<: *after-linter - build-macos: <<: *never # temporarily disabled @@ -482,8 +479,7 @@ workflows: context: common build_args: EDGE=1 <<: *after-build-and-test - #@@ <<: *on-master - <<: *on-any-branch + <<: *on-master - deploy-release: context: common <<: *after-platform-builds diff --git a/opt/readies b/opt/readies index ff19955b6..010b2685c 160000 --- a/opt/readies +++ b/opt/readies @@ -1 +1 @@ -Subproject commit ff19955b643b3da2dd4609dbaea304a469b80de1 +Subproject commit 010b2685ca3f2a8387e9582620374e792078a62d