Skip to content

Commit 507e101

Browse files
rafielantiga
rafie
authored and
lantiga
committed
CircleCI: multiarch docker build + selective engines build (#211)
* Readies sync * CircleCI: multiarch docker build * Readies sync * CircleCI: multiarch docker build #2 * CircleCI: multiarch docker build #3 * Readies sync * CircleCI: multiarch docker build * Readies sync * CircleCI: multiarch docker build #2 * CircleCI: multiarch docker build #3 * Support selective build (i.e. excluding engines) * CircleCI: multiarch docker build #4 * CircleCI: multiarch docker build #5 * CircleCI: multiarch docker build #6 * CircleCI: multiarch docker build #7 * CircleCI: multiarch docker build #8 * CircleCI: multiarch docker build #9 * Disabled CircleCI restore from cache * Reverted python3 dependency installation * CircleCI: multiarch docker build #10 * CircleCI: multiarch docker build #11 * system-setup: fixed Python libs installations * Fixed tensorflow collect script * Enabled macOS in CircleCI + Fixed basic_tests.py for decoding * CircleCI: moved to rmbuilder:x64-build * CircleCI fixes #2 * CircleCI fixes #3 * CircleCI fixes #4 * Reverted RLTest decoding-related change * CircleCI fixes #5 * Tests: double-panda.py to diagnose macOS issue * get_deps: download libtorch from original url - download libtorch from original url via rapack.sh - paella/platform: fixed problem with RHEL identification * paella: fixed urllib3 issue
1 parent 998dde9 commit 507e101

39 files changed

+1827
-255
lines changed

‎.circleci/config.yml

Lines changed: 50 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,18 @@ commands:
77
type: string
88
steps:
99
- checkout
10-
- restore_cache:
11-
keys:
12-
- v1-dependencies-{{ checksum "get_deps.sh" }}
13-
# fallback to using the latest cache if no exact match is found
14-
# - v1-dependencies-
10+
# - restore_cache:
11+
# keys:
12+
# - v1-dependencies-{{ checksum "get_deps.sh" }}
13+
# # fallback to using the latest cache if no exact match is found
14+
# # - v1-dependencies-
1515
- run:
1616
name: Install dependencies
1717
command: |
18-
sudo ./opt/readies/bin/getpy
19-
sudo ./opt/system-setup.py
20-
git clone git://github.com/antirez/redis.git --branch 5.0.5
21-
(cd redis && make malloc=libc -j $(nproc) && sudo make install)
18+
./opt/readies/bin/getpy
19+
./opt/system-setup.py
20+
# git clone git://github.com/antirez/redis.git --branch 5.0.5
21+
# (cd redis && make malloc=libc -j $(nproc) && make install)
2222
./get_deps.sh cpu
2323
- save_cache:
2424
paths:
@@ -27,29 +27,29 @@ commands:
2727
- run:
2828
name: Set up workspace
2929
command: |
30-
sudo mkdir -p /workspace
31-
sudo chown `whoami` /workspace
30+
mkdir -p ~/workspace
31+
chown `whoami` ~/workspace
3232
- run:
3333
name: Build
3434
command: make -C opt all SHOW=1
3535
- run:
3636
name: Test
3737
command: |
38-
mkdir -p /workspace/tests
38+
mkdir -p ~/workspace/tests
3939
make -C opt test SHOW=1
40-
cp test/logs/* /workspace/tests
40+
cp test/logs/* ~/workspace/tests
4141
- run:
4242
name: Package
43-
command: make -C opt pack BRANCH="${CIRCLE_BRANCH//[^A-Za-z0-9._-]/_}" INTO=/workspace/packages SHOW=1
43+
command: make -C opt pack BRANCH="${CIRCLE_BRANCH//[^A-Za-z0-9._-]/_}" INTO=~/workspace/packages SHOW=1
4444
- persist_to_workspace:
45-
root: /workspace
45+
root: ~/workspace
4646
paths:
4747
- 'packages/release/*.zip'
4848
- 'packages/release/*.tgz'
4949
- 'packages/branch/*.zip'
5050
- 'packages/branch/*.tgz'
5151
- store_test_results:
52-
path: /workspace/tests
52+
path: ~/workspace/tests
5353
deploy:
5454
parameters:
5555
from:
@@ -63,9 +63,7 @@ commands:
6363
jobs:
6464
build:
6565
docker:
66-
- image: circleci/python:3.7.4-buster
67-
environment:
68-
DEPS: ""
66+
- image: redisfab/rmbuilder:x64-buster
6967
steps:
7068
- ci_steps:
7169
platform: debian
@@ -75,24 +73,41 @@ jobs:
7573
steps:
7674
- ci_steps:
7775
platform: macosx
76+
build-multiarch-docker:
77+
machine:
78+
enabled: true
79+
image: ubuntu-1604:201903-01
80+
steps:
81+
- checkout
82+
- run:
83+
name: Setup Docker client experimental features
84+
command: |
85+
sudo ./opt/readies/bin/getdocker --just-enable-exp
86+
docker version
87+
- run:
88+
name: Build
89+
command: |
90+
sudo docker login -u redisfab -p $DOCKER_REDISFAB_PWD
91+
make -C opt/build/docker build
92+
sudo make -C opt/build/docker publish
7893
7994
deploy_package:
8095
parameters:
8196
package:
8297
type: string
8398
docker:
84-
- image: 'redislabsmodules/rmbuilder:latest'
99+
- image: redisfab/rmbuilder:x64-buster
85100
steps:
86101
- attach_workspace:
87-
at: /workspace
102+
at: ~/workspace
88103
- deploy:
89-
from: /workspace/packages/<<parameters.package>>
104+
from: ~/workspace/packages/<<parameters.package>>
90105
- store_artifacts:
91-
path: /workspace/packages/<<parameters.package>>
106+
path: ~/workspace/packages/<<parameters.package>>
92107

93108
deploy_docs:
94109
docker:
95-
- image: 'redislabsmodules/rmbuilder:latest'
110+
- image: redisfab/rmbuilder:x64-buster
96111
steps:
97112
- checkout
98113
- run:
@@ -111,16 +126,23 @@ workflows:
111126
filters:
112127
tags:
113128
only: /.*/
114-
# - build-macos:
115-
# filters:
116-
# tags:
117-
# only: /.*/
129+
- build-macos:
130+
filters:
131+
tags:
132+
only: /.*/
133+
- build-multiarch-docker:
134+
filters:
135+
tags:
136+
only: /.*/
137+
branches:
138+
only: master
118139
- deploy_package:
119140
name: deploy_branch
120141
package: branch
121142
requires:
122143
- build
123144
filters:
145+
# # uncomment to debug
124146
# tags:
125147
# only: /.*/
126148
branches:

‎CMakeLists.txt

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@ SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC")
2020
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
2121

2222
# For adding specific Release flags
23-
set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
24-
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
23+
set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O3")
24+
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")
2525

2626
# Add -fno-omit-frame-pointer to avoid seeing incomplete stack traces
2727
set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g -ggdb -fno-omit-frame-pointer")
@@ -93,9 +93,15 @@ ADD_DEFINITIONS(-DREDISMODULE_EXPERIMENTAL_API)
9393
IF(BUILD_TF)
9494
FIND_LIBRARY(TF_LIBRARIES NAMES tensorflow
9595
PATHS ${depsAbs}/libtensorflow/lib)
96-
MESSAGE(STATUS "Found TensorFlow Libraries: \"${TF_LIBRARIES}\")")
9796
IF (NOT TF_LIBRARIES)
98-
MESSAGE(FATAL_ERROR "Could not find tensorflow in ${depsAbs}/libtensorflow/lib")
97+
MESSAGE(WARNING "Could not find TensorFlow in ${depsAbs}/libtensorflow/lib. Trying find_package method")
98+
FIND_PACKAGE(TensorFlow REQUIRED)
99+
SET(TF_LIBRARIES ${TensorFlow_LIBRARY})
100+
IF (NOT TF_LIBRARIES)
101+
MESSAGE(FATAL_ERROR "Could not find TensorFlow")
102+
ELSE()
103+
MESSAGE(STATUS "Found TensorFlow Libraries: \"${TF_LIBRARIES}\")")
104+
ENDIF()
99105
ENDIF()
100106
ENDIF()
101107

@@ -195,7 +201,7 @@ IF(BUILD_TORCH)
195201
SET_TARGET_PROPERTIES(redisai_torch PROPERTIES SUFFIX ".so")
196202
IF (APPLE)
197203
SET_TARGET_PROPERTIES(redisai_torch PROPERTIES INSTALL_RPATH "@loader_path/lib")
198-
ELSE ()
204+
ELSE()
199205
ADD_LDFLAGS(redisai_torch "-Wl,--enable-new-dtags")
200206
SET_TARGET_PROPERTIES(redisai_torch PROPERTIES INSTALL_RPATH "\$ORIGIN/lib")
201207
ENDIF()

‎Dockerfile

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,13 @@ COPY ./test/test_requirements.txt test/
2222
RUN ./opt/readies/bin/getpy
2323
RUN ./opt/system-setup.py
2424

25+
ARG DEPS_ARGS=""
2526
COPY ./get_deps.sh .
26-
RUN ./get_deps.sh cpu
27+
RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh cpu; else env $DEPS_ARGS ./get_deps.sh cpu; fi
2728

29+
ARG BUILD_ARGS=""
2830
ADD ./ /build
29-
RUN make -C opt build SHOW=1
31+
RUN make -C opt build $BUILD_ARGS SHOW=1
3032

3133
ARG PACK=0
3234
ARG TEST=0

‎Dockerfile.arm

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
ARG OSNICK=buster
55

66
# OS=debian:buster-slim|debian:stretch-slim|ubuntu:bionic
7-
OS=debian:buster-slim
7+
ARG OS=debian:buster-slim
88

99
# ARCH=arm64v8|arm32v7
1010
ARG ARCH=arm64v8
@@ -14,7 +14,6 @@ FROM redisfab/redis-${ARCH}-${OSNICK}-xbuild:5.0.5 AS builder
1414

1515
RUN [ "cross-build-start" ]
1616

17-
ADD ./ /build
1817
WORKDIR /build
1918

2019
COPY ./opt/ opt/
@@ -23,11 +22,13 @@ COPY ./test/test_requirements.txt test/
2322
RUN ./opt/readies/bin/getpy
2423
RUN ./opt/system-setup.py
2524

25+
ARG DEPS_ARGS=""
2626
COPY ./get_deps.sh .
27-
RUN ./get_deps.sh cpu
27+
RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh cpu; else env $DEPS_ARGS ./get_deps.sh cpu; fi
2828

29-
ADD ./ /redisai
30-
RUN make -C opt all SHOW=1
29+
ARG BUILD_ARGS=""
30+
ADD ./ /build
31+
RUN make -C opt build $BUILD_ARGS SHOW=1
3132

3233
ARG PACK=0
3334
ARG TEST=0

0 commit comments

Comments
 (0)