Skip to content

Commit ea172c2

Browse files
rafierafie
committed
ARM (aarch64) support #8 
1 parent 18e765a commit ea172c2

File tree

16 files changed

+278
-17
lines changed

16 files changed

+278
-17
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ bin/*
22
VARIANT
33
BINDIR
44
deps/*
5+
!deps/bazel/
56
!deps/readies/
67
!deps/*.py
78
examples/js/node_modules/

‎Dockerfile

Lines changed: 13 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,27 @@
1-
#----------------------------------------------------------------------------------------------
2-
# FROM redislabs/redis-arm:arm64-bionic AS builder
3-
FROM raffapen/redis:arm64-bionic AS builder
1+
# TAG redisai-cpu-${ARCH}-${OSNICK}:latest
2+
3+
ARG OSNICK=bionic
44

5-
RUN set -ex;\
6-
apt-get update;\
7-
apt-get install -y python python3
5+
#----------------------------------------------------------------------------------------------
6+
FROM raffapen/redis-${OSNICK}-xbuild:5.0.5 AS builder
87

9-
ADD ./ /redisai
10-
WORKDIR /redisai
8+
ADD ./ /build
9+
WORKDIR /build
1110

11+
RUN ./deps/readies/bin/getpy2
1212
RUN ./system-setup.py
1313
RUN make deps
14-
RUN make
14+
RUN make -j`nproc`
1515

1616
#----------------------------------------------------------------------------------------------
17-
# FROM redislabs/redis-arm:arm64-bionic
18-
FROM raffapen/redis-arm:arm64-bionic
17+
FROM raffapen/redis-${OSNICK}-xbuild:5.0.5
1918

2019
ENV LD_LIBRARY_PATH /usr/lib/redis/modules/
2120

22-
RUN set -ex;\
23-
mkdir -p "$LD_LIBRARY_PATH";
21+
RUN mkdir -p "$LD_LIBRARY_PATH"
2422

25-
COPY --from=builder /redisai/bin/redisai.so "$LD_LIBRARY_PATH"
26-
COPY --from=builder /redisai/deps/install/*.so* "$LD_LIBRARY_PATH"
23+
COPY --from=builder /build/bin/redisai.so "$LD_LIBRARY_PATH"
24+
COPY --from=builder /build/deps/install/*.so* "$LD_LIBRARY_PATH"
2725

2826
WORKDIR /data
2927
EXPOSE 6379

‎Dockerfile.arm64-xbuild

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# TAG redisai-cpu-${ARCH}-${OSNICK}:latest
2+
3+
ARG OSNICK=bionic
4+
5+
#----------------------------------------------------------------------------------------------
6+
FROM raffapen/redis-${OSNICK}-xbuild:5.0.5 AS builder
7+
8+
RUN [ "cross-build-start" ]
9+
10+
ADD ./ /build
11+
WORKDIR /build
12+
13+
RUN ./deps/readies/bin/getpy2
14+
RUN ./system-setup.py
15+
RUN make deps
16+
RUN make -j`nproc`
17+
18+
RUN [ "cross-build-end" ]
19+
20+
#----------------------------------------------------------------------------------------------
21+
FROM raffapen/redis-${OSNICK}-xbuild:5.0.5
22+
23+
RUN [ "cross-build-start" ]
24+
25+
ENV LD_LIBRARY_PATH /usr/lib/redis/modules/
26+
27+
RUN mkdir -p "$LD_LIBRARY_PATH";
28+
29+
COPY --from=builder /build/bin/redisai.so "$LD_LIBRARY_PATH"
30+
COPY --from=builder /build/deps/install/*.so* "$LD_LIBRARY_PATH"
31+
32+
WORKDIR /data
33+
EXPOSE 6379
34+
CMD ["--loadmodule", "/usr/lib/redis/modules/redisai.so"]
35+
36+
RUN [ "cross-build-end" ]

‎build/bazel/Dockerfile

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
FROM insready/bazel:latest as baz
2+
FROM ubuntu:latest
3+
4+
COPY --from=baz /usr/bin/bazel* /usr/local/bin/

‎build/libtorch-arm/Dockerfile

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
ARG OS=ubuntu:bionic
2+
3+
#----------------------------------------------------------------------------------------------
4+
FROM ${OS}
5+
6+
# ARG ARCH=x64
7+
ARG PT_VER=v1.0.1
8+
9+
RUN set -ex; apt-get update; apt-get install -y git
10+
11+
WORKDIR /build
12+
13+
RUN apt-get install -y build-essential
14+
RUN apt-get install -y python3-pip python3-cffi
15+
RUN apt-get install -y ninja-build cmake
16+
RUN pip3 install setuptools pyyaml typing
17+
RUN pip3 install numpy
18+
# mkl mkl-include
19+
20+
RUN git clone https://github.com/pytorch/pytorch.git ;\
21+
cd pytorch ;\
22+
git checkout ${PT_VER} ;\
23+
git submodule update --init --recursive
24+
25+
RUN cd pytorch ;\
26+
BUILD_PYTHON=0 \
27+
USE_GLOO=1 \
28+
USE_OPENCV=0 \
29+
BUILD_TORCH=ON \
30+
BUILD_BINARY=ON \
31+
BUILD_CAFFE2_OPS=ON \
32+
NO_CUDA=1 \
33+
NO_DISTRIBUTED=1 \
34+
NO_MKLDNN=1 \
35+
NO_NNPACK=1 \
36+
NO_QNNPACK=1 \
37+
python3 setup.py install

‎build/libtorch-x64/Dockerfile

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
ARG OS=ubuntu:bionic
2+
3+
#----------------------------------------------------------------------------------------------
4+
FROM ${OS}
5+
6+
ARG PT_VER=v1.0.1
7+
8+
RUN set -ex; apt-get update; apt-get install -y git
9+
10+
WORKDIR /build
11+
12+
RUN apt-get install -y build-essential
13+
RUN apt-get install -y python3-pip python3-cffi
14+
RUN apt-get install -y ninja-build cmake
15+
RUN pip3 install setuptools pyyaml typing
16+
RUN pip3 install numpy mkl mkl-include
17+
18+
RUN git clone https://github.com/pytorch/pytorch.git ;\
19+
cd pytorch ;\
20+
git checkout ${PT_VER}
21+
22+
RUN cd pytorch ;\
23+
BUILD_PYTHON=0 \
24+
USE_GLOO=1 \
25+
USE_OPENCV=0 \
26+
BUILD_TORCH=ON \
27+
BUILD_BINARY=ON \
28+
BUILD_CAFFE2_OPS=ON \
29+
NO_CUDA=1 \
30+
NO_DISTRIBUTED=1 \
31+
NO_MKLDNN=1 \
32+
NO_NNPACK=1 \
33+
NO_QNNPACK=1 \
34+
python3 setup.py install

‎build/tensorflow-arm/Dockerfile

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
ARG ARCH=x64
2+
3+
#----------------------------------------------------------------------------------------------
4+
FROM insready/bazel:latest as bazel
5+
FROM ${OS}
6+
7+
ARG OS=ubuntu:bionic
8+
ARG TF_VER=v1.13.1
9+
10+
RUN set -ex; apt-get update; apt-get install -y git
11+
12+
WORKDIR /build
13+
14+
COPY --from=bazel /usr/bin/bazel* /usr/local/bin/
15+
16+
RUN apt-get install -y build-essential
17+
RUN apt-get install -y python3-dev python3-pip
18+
19+
RUN pip install six numpy wheel setuptools mock
20+
RUN pip install keras_applications==1.0.6 --no-deps
21+
RUN pip install keras_preprocessing==1.0.5 --no-deps
22+
23+
RUN git clone https://github.com/tensorflow/tensorflow.git ;\
24+
cd tensorflow ;\
25+
git checkout ${TF_VER}
26+
27+
RUN apt-get install -y python3 python3-dev
28+
29+
RUN cd tensorflow ;\
30+
./configure ;\
31+
bazel build --jobs `nproc` --config=opt //tensorflow:libtensorflow.so

‎build/tensorflow-x64/Dockerfile

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
ARG OS=ubuntu:bionic
2+
3+
#----------------------------------------------------------------------------------------------
4+
# FROM insready/bazel:latest as bazel
5+
FROM ${OS}
6+
7+
ARG TF_VER=v1.13.1
8+
9+
RUN set -ex; apt-get update; apt-get install -y git
10+
11+
WORKDIR /build
12+
13+
# COPY --from=bazel /usr/bin/bazel* /usr/local/bin/
14+
COPY ../../deps/bazel/bazel-0.24.1.x64 /usr/local/bin/bazel
15+
16+
RUN apt-get install -y build-essential
17+
RUN apt-get install -y python3-dev python3-pip
18+
RUN ln -s /usr/bin/python3 /usr/bin/python
19+
20+
RUN pip3 install wheel setuptools
21+
RUN pip3 install six numpy mock
22+
RUN pip3 install keras_applications==1.0.6 --no-deps
23+
RUN pip3 install keras_preprocessing==1.0.5 --no-deps
24+
25+
RUN git clone https://github.com/tensorflow/tensorflow.git ;\
26+
cd tensorflow ;\
27+
git checkout ${TF_VER}
28+
29+
# from https://gist.github.com/PatWie/0c915d5be59a518f934392219ca65c3d
30+
# ENV PYTHON_BIN_PATH=$(which ${python_version})
31+
# ENV PYTHON_LIB_PATH="$($PYTHON_BIN_PATH -c 'import site; print(site.getsitepackages()[0])')"
32+
# ENV PYTHONPATH=${TF_ROOT}/lib
33+
# ENV PYTHON_ARG=${TF_ROOT}/lib
34+
35+
# ENV CUDA_TOOLKIT_PATH=${opt}/cuda/toolkit_8.0/cuda
36+
# ENV CUDNN_INSTALL_PATH=${opt}/cuda/cudnn/6/cuda
37+
38+
ENV TF_NEED_GCP=0
39+
ENV TF_NEED_CUDA=0
40+
# ENV TF_CUDA_COMPUTE_CAPABILITIES=6.1,5.2,3.5
41+
ENV TF_NEED_HDFS=0
42+
ENV TF_NEED_OPENCL=0
43+
ENV TF_NEED_JEMALLOC=1
44+
ENV TF_ENABLE_XLA=0
45+
ENV TF_NEED_VERBS=0
46+
ENV TF_CUDA_CLANG=0
47+
ENV TF_NEED_MKL=0
48+
ENV TF_DOWNLOAD_MKL=0
49+
ENV TF_NEED_AWS=0
50+
ENV TF_NEED_MPI=0
51+
ENV TF_NEED_GDR=0
52+
ENV TF_NEED_S3=0
53+
ENV TF_NEED_OPENCL_SYCL=0
54+
ENV TF_SET_ANDROID_WORKSPACE=0
55+
ENV TF_NEED_COMPUTECPP=0
56+
ENV TF_NEED_KAFKA=0
57+
ENV TF_NEED_TENSORRT=0
58+
59+
RUN cd tensorflow ;\
60+
./configure ;\
61+
bazel build --jobs `nproc` --config=opt //tensorflow:libtensorflow.so

‎deps/bazel/bazel-0.24.1.arm64

62.3 MB
Binary file not shown.

‎deps/bazel/bazel-0.24.1.x64

93.4 MB
Binary file not shown.

‎deps/readies/bin/getpy2

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/bin/sh
2+
3+
[ ! -z $(command -v python) ] && exit 0
4+
[ ! -z $(command -v python2) ] && exit 0
5+
6+
if [ ! -z $(command -v apt-get) ]; then
7+
apt-get update
8+
apt-get install -y python
9+
elif [ ! -z $(command -v dnf) ]; then
10+
dnf install -y python2
11+
elif [ ! -z $(command -v yum) ]; then
12+
yum install -y python2
13+
elif [ ! -z $(command -v apk) ]; then
14+
apk update
15+
apk add python2
16+
elif [ ! -z $(command -v brew) ]; then
17+
brew install python2
18+
fi
19+
20+
if [ -z $(command -v python) ]; then
21+
>&2 echo "Cannot install Python2. Aborting."
22+
exit 1
23+
fi

‎deps/readies/paella/files.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@ def fread(fname, mode = 'rb'):
66
with open(fname, mode) as file:
77
return file.read()
88

9+
def flines(fname, mode = 'rb'):
10+
return [line.rstrip() for line in open(fname)]
11+
912
@contextmanager
1013
def cwd(path):
1114
d0 = os.getcwd()

‎deps/readies/paella/platform.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66

77
class Platform:
88
def __init__(self):
9+
self.os = self.dist = self.os_ver = self.full_os_ver = self.os_nick = self.arch = '?'
10+
911
self.os = platform.system().lower()
1012
dist = platform.linux_distribution()
1113
distname = dist[0].lower()
@@ -33,12 +35,18 @@ def __init__(self):
3335
self.dist = self.os
3436
self.os_ver = platform.release()
3537
self.full_os_ver = os.version()
38+
elif self.os == 'sunos':
39+
self.os = 'solaris'
40+
self.os_ver = ''
41+
self.dist = ''
3642
else:
3743
Assert(False), "Cannot determine OS"
3844

3945
self.arch = platform.machine().lower()
4046
if self.arch == 'amd64' or self.arch == 'x86_64':
4147
self.arch = 'x64'
48+
elif self.arch == 'i386' or self.arch == 'i686' or self.arch == 'i86pc':
49+
self.arch = 'x86'
4250
elif self.arch == 'aarch64':
4351
self.arch = 'arm64v8'
4452

‎deps/readies/paella/setup.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,11 @@ def __init__(self, nop=False):
5353
# this is required because osx pip installed are done with --user
5454
if self.os == 'macosx':
5555
os.environ["PATH"] = os.environ["PATH"] + ':' + '$HOME/Library/Python/2.7/bin'
56+
57+
if self.platform.is_debian_compat():
58+
# prevents apt-get from interactively prompting
59+
os.environ["DEBIAN_FRONTEND"] = 'noninteractive'
60+
5661
os.environ["PYTHONWARNINGS"] = 'ignore:DEPRECATION::pip._internal.cli.base_command'
5762

5863
def setup(self):
@@ -68,7 +73,7 @@ def has_command(self, cmd):
6873
#------------------------------------------------------------------------------------------
6974

7075
def apt_install(self, packs, group=False):
71-
self.run("apt-get install -q -y " + packs)
76+
self.run("apt-get -qq install -y " + packs)
7277

7378
def yum_install(self, packs, group=False):
7479
if not group:

‎deps/readies/shibumi/functions

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,21 @@
11

2+
platform_os() {
3+
case "$OSTYPE" in
4+
linux*) echo "linux" ;;
5+
solaris*) echo "solaris" ;;
6+
darwin*) echo "macosx" ;;
7+
bsd*) echo "bsd" ;;
8+
msys*) echo "windows" ;;
9+
*) echo "?" ;;
10+
esac
11+
}
12+
13+
#----------------------------------------------------------------------------------------------
14+
15+
if [[ $(platform_os) == mac ]]; then
16+
17+
realpath() {
18+
[[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}"
19+
}
20+
21+
fi

‎system-setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ def common_first(self):
2121
self.pip_install("wheel")
2222
self.pip_install("setuptools --upgrade")
2323

24-
self.install("git cmake ca-certificates curl unzip wget")
24+
self.install("git python3 cmake ca-certificates curl unzip wget")
2525

2626
def debian_compat(self):
2727
self.install("build-essential")

0 commit comments

Comments
 (0)