From 80b72ffd4a7562e52f27ce2dcc3d49e242d13585 Mon Sep 17 00:00:00 2001 From: rafie Date: Sun, 7 Feb 2021 15:35:04 +0200 Subject: [PATCH 1/4] Refactored docker builder --- .circleci/config.yml | 92 +++++++++++++++++++-------------------- opt/build/docker/Makefile | 38 ++++------------ opt/readies | 2 +- 3 files changed, 56 insertions(+), 76 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 774eaf1ed..3013c3fdc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -87,9 +87,9 @@ commands: command: | docker login -u redisfab -p $DOCKER_REDISFAB_PWD pushd opt/build/docker - #@@ make build publish $(./version-params) CPU=1 OSNICK=<> X64=1 ARTIFACTS=1 TEST=1 VERBOSE=1 - make build publish $(./version-params) CPU=1 OSNICK=<> X64=1 ARTIFACTS=1 VERBOSE=1 - make build publish $(./version-params) GPU=1 OSNICK=<> X64=1 ARTIFACTS=1 VERBOSE=1 + #@@ make CPU=1 OSNICK=<> ARTIFACTS=1 TEST=1 VERBOSE=1 + make CPU=1 OSNICK=<> ARTIFACTS=1 VERBOSE=1 + make GPU=1 OSNICK=<> ARTIFACTS=1 VERBOSE=1 popd > /dev/null logstar=bin/artifacts/tests-logs-cpu.tgz logsdir=tests/logs/cpu @@ -106,26 +106,26 @@ commands: - store_artifacts: path: test/logs - dockerhub-build-steps: - parameters: - build_args: - type: string - steps: - - setup-executor - - checkout - - run: - name: Setup - command: | - git submodule update --init --recursive - ./opt/readies/bin/getpy3 - - run: - name: Build for platform - command: | - docker login -u redisfab -p $DOCKER_REDISFAB_PWD - cd opt/build/docker - make build publish $(./version-params) CPU=1 OSNICK=bionic X64=1 REPO=redislabs VERBOSE=1 <> - make build publish $(./version-params) GPU=1 OSNICK=bionic X64=1 REPO=redislabs VERBOSE=1 <> - no_output_timeout: 40m +# dockerhub-build-steps: +# parameters: +# build_args: +# type: string +# steps: +# - setup-executor +# - checkout +# - run: +# name: Setup +# command: | +# git submodule update --init --recursive +# ./opt/readies/bin/getpy3 +# - run: +# name: Build for platform +# command: | +# docker login -u redisfab -p $DOCKER_REDISFAB_PWD +# cd opt/build/docker +# make build publish $(./version-params) CPU=1 OSNICK=bionic X64=1 REPO=redislabs VERBOSE=1 <> +# make build publish $(./version-params) GPU=1 OSNICK=bionic X64=1 REPO=redislabs VERBOSE=1 <> +# no_output_timeout: 40m deploy-steps: parameters: @@ -294,15 +294,15 @@ jobs: - store_artifacts: path: test/log - dockerhub-build: - parameters: - build_args: - type: string - docker: - - image: debian:buster - steps: - - dockerhub-build-steps: - build_args: <> +# dockerhub-build: +# parameters: +# build_args: +# type: string +# docker: +# - image: debian:buster +# steps: +# - dockerhub-build-steps: +# build_args: <> deploy-artifacts: parameters: @@ -417,8 +417,8 @@ after-platform-builds: &after-platform-builds platform-build-defs: &platform-build-defs <<: *after-build-and-test - <<: *on-integ-and-version-tags - # <<: *on-any-branch # debugging + #@@ <<: *on-integ-and-version-tags + <<: *on-any-branch # debugging workflows: @@ -474,22 +474,22 @@ workflows: context: common <<: *after-platform-builds <<: *on-integ-branch - - dockerhub-build: - name: dockerhub-edge - context: common - build_args: EDGE=1 - <<: *after-build-and-test - <<: *on-master +# - dockerhub-build: +# name: dockerhub-edge +# context: common +# build_args: EDGE=1 +# <<: *after-build-and-test +# <<: *on-master - deploy-release: context: common <<: *after-platform-builds <<: *on-version-tags - - dockerhub-build: - name: dockerhub-release - context: common - build_args: LATEST=1 - <<: *after-platform-builds - <<: *on-version-tags +# - dockerhub-build: +# name: dockerhub-release +# context: common +# build_args: LATEST=1 +# <<: *after-platform-builds +# <<: *on-version-tags nightly: triggers: diff --git a/opt/build/docker/Makefile b/opt/build/docker/Makefile index 44ec61812..672becaf8 100755 --- a/opt/build/docker/Makefile +++ b/opt/build/docker/Makefile @@ -6,35 +6,23 @@ include $(ROOT)/opt/readies/mk/main # OSNICK=buster|stretch|bionic|xenial|centos7 OSNICK ?= buster -ifeq ($(REPO),redislabs) -SIMPLE ?= 1 -else -REPO=redisfab -endif - -STEM=$(REPO)/redisai +REPO=redisai -LATEST_BRANCH=1.2 +# LATEST_BRANCH=1.2 +INT_BRANCHES=1.2 REDIS_VER:=6.0.9 ART_DIR=$(ROOT)/bin/artifacts ART_INT_DIR=/var/opt/redislabs/artifacts -FETCH_ARGS.arm64v8=WITH_ORT=0 WITH_PT=0 -BUILD_ARGS.arm64v8=WITH_ORT=0 WITH_PT=0 - -FETCH_ARGS.arm32v7=WITH_ORT=0 WITH_PT=0 -BUILD_ARGS.arm32v7=WITH_ORT=0 WITH_PT=0 +#---------------------------------------------------------------------------------------------- ifeq ($(CPU),1) -VAR=cpu - -FETCH_ARGS=$(FETCH_ARGS.$(ARCH)) -BUILD_ARGS=$(BUILD_ARGS.$(ARCH)) +VARIANT=cpu else ifeq ($(GPU),1) -VAR=gpu +VARIANT=gpu DOCKERFILE_STEM ?= $(ROOT)/Dockerfile.gpu OS.custom:=1 @@ -43,20 +31,12 @@ OS.centos7=centos7 OS.bionic=ubuntu18.04 OS.xenial=ubuntu16.04 -FETCH_ARGS=$(FETCH_ARGS.$(ARCH)) -BUILD_ARGS=$(BUILD_ARGS.$(ARCH)) - else ifeq ($(JETSON),1) -VAR=jetson +VARIANT=jetson DOCKERFILE_STEM ?= $(ROOT)/Dockerfile.jetson -# FETCH_ARGS=WITH_TF=1 WITH_PT=1 WITH_TFLITE=0 WITH_ORT=0 -# BUILD_ARGS=WITH_TF=1 WITH_PT=1 WITH_TFLITE=0 WITH_ORT=0 - endif -# DOCKER_BUILD_ARGS += \ -# DEPS_ARGS="$(FETCH_ARGS)" \ -# BUILD_ARGS="$(BUILD_ARGS)" +#---------------------------------------------------------------------------------------------- -include $(MK)/docker +include $(MK)/docker.native diff --git a/opt/readies b/opt/readies index 010b2685c..392cce40c 160000 --- a/opt/readies +++ b/opt/readies @@ -1 +1 @@ -Subproject commit 010b2685ca3f2a8387e9582620374e792078a62d +Subproject commit 392cce40c884e3e3297f7c1b09f2bf3ae4cd9368 From b70f2e7649c0c76b9d88d7ac6bce4d51cd49974f Mon Sep 17 00:00:00 2001 From: rafie Date: Sun, 7 Feb 2021 20:01:07 +0200 Subject: [PATCH 2/4] fixes 1 --- .circleci/config.yml | 14 ++++++++++++-- opt/readies | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3013c3fdc..e3f444e56 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -281,6 +281,15 @@ jobs: - run: name: Submodule checkout command: git submodule update --init --recursive + - run: + name: Relocate docker overlay2 dir + command: | + sudo systemctl stop docker + sudo mkdir -p /var2/lib/docker + sudo mv /var/lib/docker/overlay2 /var2/lib/docker + sudo mkdir /var/lib/docker/overlay2 + sudo mount --bind /var2/lib/docker/overlay2 /var/lib/docker/overlay2 + sudo systemctl start docker - run: name: Build command: | @@ -292,7 +301,7 @@ jobs: docker run --gpus all -v $HOME/tests:/build/tests/logs -it --rm redisai-gpu:latest-x64-bionic-test no_output_timeout: 40m - store_artifacts: - path: test/log + path: tests/logs # dockerhub-build: # parameters: @@ -448,7 +457,8 @@ workflows: <<: *after-linter - valgrind: test_args: CLUSTER=0 AOF=0 - <<: *on-any-branch + #@@<<: *on-any-branch + <<: *never # remove this line <<: *after-linter - valgrind: name: valgrind-cluster diff --git a/opt/readies b/opt/readies index 392cce40c..03ba961f1 160000 --- a/opt/readies +++ b/opt/readies @@ -1 +1 @@ -Subproject commit 392cce40c884e3e3297f7c1b09f2bf3ae4cd9368 +Subproject commit 03ba961f139ac5e2a9ae395502620e4046c4987b From 018ce553bf083cd5cdf2970ee3dcf7c51f0962e5 Mon Sep 17 00:00:00 2001 From: rafie Date: Sun, 7 Feb 2021 22:09:40 +0200 Subject: [PATCH 3/4] fixes 2 --- .circleci/config.yml | 89 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 71 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e3f444e56..88cabdab8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -71,11 +71,7 @@ commands: path: tests/logs platform-build-steps: - parameters: - platform: - type: string steps: - - setup-executor - checkout - run: name: Setup @@ -106,6 +102,50 @@ commands: - store_artifacts: path: test/logs + platforms-build-steps: + steps: + - checkout + - run: + name: Relocate docker overlay2 dir + command: | + sudo systemctl stop docker + sudo mkdir -p /var2/lib/docker + sudo mv /var/lib/docker/overlay2 /var2/lib/docker + sudo mkdir /var/lib/docker/overlay2 + sudo mount --bind /var2/lib/docker/overlay2 /var/lib/docker/overlay2 + sudo systemctl start docker + - run: + name: Setup + command: | + git submodule update --init --recursive + ./opt/readies/bin/getpy3 + - run: + name: Build for platform + command: | + docker login -u redisfab -p $DOCKER_REDISFAB_PWD + pushd opt/build/docker + for osnick in bionic xenial; do + make CPU=1 OSNICK=$osnick ARTIFACTS=1 VERBOSE=1 + make GPU=1 OSNICK=$osnick ARTIFACTS=1 VERBOSE=1 + make CPU=1 OSNICK=$osnick ARTIFACTS=1 VERBOSE=1 + make GPU=1 OSNICK=$osnick ARTIFACTS=1 VERBOSE=1 + done + popd > /dev/null + logstar=bin/artifacts/tests-logs-cpu.tgz + logsdir=tests/logs/cpu + mkdir -p $logsdir + if [[ -e $logstar ]]; then tar -C $logsdir -xzf $logstar; fi + (cd bin/artifacts; tar -cf snapshots.tar snapshots/) + no_output_timeout: 40m + - persist_to_workspace: + root: bin/ + paths: + - artifacts/*.zip + - artifacts/*.tgz + - artifacts/*.tar + - store_artifacts: + path: test/logs + # dockerhub-build-steps: # parameters: # build_args: @@ -169,6 +209,15 @@ jobs: - platform-build-steps: platform: <> + platforms-build: + machine: + enabled: true + docker_layer_caching: true + resource_class: gpu.nvidia.small + image: ubuntu-1604-cuda-11.1:202012-01 + steps: + - platforms-build-steps + coverage: docker: - image: redisfab/rmbuilder:6.0.9-x64-buster @@ -420,9 +469,10 @@ after-build-and-test: &after-build-and-test after-platform-builds: &after-platform-builds requires: + - platforms-build # - build-centos7 - - build-bionic - - build-xenial +# - build-bionic +# - build-xenial platform-build-defs: &platform-build-defs <<: *after-build-and-test @@ -439,21 +489,24 @@ workflows: - build-and-test: <<: *on-any-branch <<: *after-linter - - platform-build: - name: build-centos7 - platform: centos7 - <<: *never # temporarily disabled - - platform-build: - name: build-bionic - platform: bionic - <<: *platform-build-defs - - platform-build: - name: build-xenial - platform: xenial +# - platform-build: +# name: build-centos7 +# platform: centos7 +# <<: *never # temporarily disabled + - platforms-build: <<: *platform-build-defs +# - platform-build: +# name: build-bionic +# platform: bionic +# <<: *platform-build-defs +# - platform-build: +# name: build-xenial +# platform: xenial +# <<: *platform-build-defs - coverage: context: common - <<: *on-any-branch + #@@ <<: *on-any-branch + <<: *never # remove this line <<: *after-linter - valgrind: test_args: CLUSTER=0 AOF=0 From e45571c1d5ac7e39fe88ef92c1ff3ce80f1aa6d0 Mon Sep 17 00:00:00 2001 From: rafie Date: Mon, 8 Feb 2021 01:52:20 +0200 Subject: [PATCH 4/4] fixes 3 --- .circleci/config.yml | 112 ++----------------------------------------- 1 file changed, 4 insertions(+), 108 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 88cabdab8..ef2673482 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,38 +70,6 @@ commands: - store_artifacts: path: tests/logs - platform-build-steps: - steps: - - checkout - - run: - name: Setup - command: | - git submodule update --init --recursive - ./opt/readies/bin/getpy3 - - run: - name: Build for platform - command: | - docker login -u redisfab -p $DOCKER_REDISFAB_PWD - pushd opt/build/docker - #@@ make CPU=1 OSNICK=<> ARTIFACTS=1 TEST=1 VERBOSE=1 - make CPU=1 OSNICK=<> ARTIFACTS=1 VERBOSE=1 - make GPU=1 OSNICK=<> ARTIFACTS=1 VERBOSE=1 - popd > /dev/null - logstar=bin/artifacts/tests-logs-cpu.tgz - logsdir=tests/logs/cpu - mkdir -p $logsdir - if [[ -e $logstar ]]; then tar -C $logsdir -xzf $logstar; fi - (cd bin/artifacts; tar -cf snapshots-<>.tar snapshots/) - no_output_timeout: 40m - - persist_to_workspace: - root: bin/ - paths: - - artifacts/*.zip - - artifacts/*.tgz - - artifacts/*.tar - - store_artifacts: - path: test/logs - platforms-build-steps: steps: - checkout @@ -127,8 +95,6 @@ commands: for osnick in bionic xenial; do make CPU=1 OSNICK=$osnick ARTIFACTS=1 VERBOSE=1 make GPU=1 OSNICK=$osnick ARTIFACTS=1 VERBOSE=1 - make CPU=1 OSNICK=$osnick ARTIFACTS=1 VERBOSE=1 - make GPU=1 OSNICK=$osnick ARTIFACTS=1 VERBOSE=1 done popd > /dev/null logstar=bin/artifacts/tests-logs-cpu.tgz @@ -146,27 +112,6 @@ commands: - store_artifacts: path: test/logs -# dockerhub-build-steps: -# parameters: -# build_args: -# type: string -# steps: -# - setup-executor -# - checkout -# - run: -# name: Setup -# command: | -# git submodule update --init --recursive -# ./opt/readies/bin/getpy3 -# - run: -# name: Build for platform -# command: | -# docker login -u redisfab -p $DOCKER_REDISFAB_PWD -# cd opt/build/docker -# make build publish $(./version-params) CPU=1 OSNICK=bionic X64=1 REPO=redislabs VERBOSE=1 <> -# make build publish $(./version-params) GPU=1 OSNICK=bionic X64=1 REPO=redislabs VERBOSE=1 <> -# no_output_timeout: 40m - deploy-steps: parameters: from: @@ -199,16 +144,6 @@ jobs: - build-steps: platform: debian - platform-build: - parameters: - platform: - type: string - docker: - - image: debian:buster - steps: - - platform-build-steps: - platform: <> - platforms-build: machine: enabled: true @@ -352,16 +287,6 @@ jobs: - store_artifacts: path: tests/logs -# dockerhub-build: -# parameters: -# build_args: -# type: string -# docker: -# - image: debian:buster -# steps: -# - dockerhub-build-steps: -# build_args: <> - deploy-artifacts: parameters: location: @@ -470,14 +395,11 @@ after-build-and-test: &after-build-and-test after-platform-builds: &after-platform-builds requires: - platforms-build - # - build-centos7 -# - build-bionic -# - build-xenial platform-build-defs: &platform-build-defs <<: *after-build-and-test - #@@ <<: *on-integ-and-version-tags - <<: *on-any-branch # debugging + <<: *on-integ-and-version-tags + # <<: *on-any-branch # debugging workflows: @@ -489,29 +411,15 @@ workflows: - build-and-test: <<: *on-any-branch <<: *after-linter -# - platform-build: -# name: build-centos7 -# platform: centos7 -# <<: *never # temporarily disabled - platforms-build: <<: *platform-build-defs -# - platform-build: -# name: build-bionic -# platform: bionic -# <<: *platform-build-defs -# - platform-build: -# name: build-xenial -# platform: xenial -# <<: *platform-build-defs - coverage: context: common - #@@ <<: *on-any-branch - <<: *never # remove this line + <<: *on-any-branch <<: *after-linter - valgrind: test_args: CLUSTER=0 AOF=0 - #@@<<: *on-any-branch - <<: *never # remove this line + <<: *on-any-branch <<: *after-linter - valgrind: name: valgrind-cluster @@ -537,22 +445,10 @@ workflows: context: common <<: *after-platform-builds <<: *on-integ-branch -# - dockerhub-build: -# name: dockerhub-edge -# context: common -# build_args: EDGE=1 -# <<: *after-build-and-test -# <<: *on-master - deploy-release: context: common <<: *after-platform-builds <<: *on-version-tags -# - dockerhub-build: -# name: dockerhub-release -# context: common -# build_args: LATEST=1 -# <<: *after-platform-builds -# <<: *on-version-tags nightly: triggers: