Skip to content

Commit 5683649

Browse files
rafiefilipecosta90
rafie
authored and
filipecosta90
committed
Dependencies in ramp.yml (#444)
1 parent 864d618 commit 5683649

File tree

6 files changed

+198
-106
lines changed

6 files changed

+198
-106
lines changed

‎opt/Makefile

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ endif
2525

2626
define HELP
2727
make setup # install prerequisited (CAUTION: THIS WILL MODIFY YOUR SYSTEM)
28-
make fetch # download and prepare dependant modules
28+
make fetch # download and prepare dependant artifacts
2929

3030
make build # compile and link
3131
DEBUG=1 # build for debugging
@@ -179,9 +179,9 @@ pack: $(INSTALLED_TARGET)
179179
$(SHOW)find $(INSTALL_DIR) -name "*.so" -exec chmod +x {} \;
180180
$(SHOW)mkdir -p $(ROOT)/bin/artifacts
181181
ifneq ($(PACK_DEPS),0)
182-
$(SHOW)DEVICE=$(DEVICE) BINDIR=$(ROOT)/bin/artifacts INSTALL_DIR=$(INSTALL_DIR) BRANCH=$(BRANCH) DEPS=1 ./pack.sh
182+
$(SHOW)DEVICE=$(DEVICE) BINDIR=$(ROOT)/bin/artifacts INSTALL_DIR=$(INSTALL_DIR) BRANCH=$(BRANCH) RAMP=1 DEPS=1 ./pack.sh
183183
else
184-
$(SHOW)DEVICE=$(DEVICE) BINDIR=$(ROOT)/bin/artifacts INSTALL_DIR=$(INSTALL_DIR) BRANCH=$(BRANCH) DEPS=0 ./pack.sh
184+
$(SHOW)DEVICE=$(DEVICE) BINDIR=$(ROOT)/bin/artifacts INSTALL_DIR=$(INSTALL_DIR) BRANCH=$(BRANCH) RAMP=1 DEPS=0 ./pack.sh
185185
endif
186186

187187
#----------------------------------------------------------------------------------------------

‎opt/getver

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/bin/bash
2+
3+
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
4+
ROOT=$(cd $HERE/..; pwd)
5+
6+
getver_c=$(mktemp "${TMPDIR:-/tmp}/getver-XXXXXXX.c")
7+
if [[ $NUMERIC != 1 ]]; then
8+
cat <<- EOF > $getver_c
9+
#include <stdio.h>
10+
11+
#include "src/version.h"
12+
13+
int main(int argc, char *argv[]) {
14+
printf("%d.%d.%d\n", RAI_ENC_VER / 10000, (RAI_ENC_VER / 100) % 100, RAI_ENC_VER % 100);
15+
return 0;
16+
}
17+
EOF
18+
else
19+
cat <<- EOF > $getver_c
20+
#include <stdio.h>
21+
22+
#include "src/version.h"
23+
24+
int main(int argc, char *argv[]) {
25+
printf("%d\n", RAI_ENC_VER);
26+
return 0;
27+
}
28+
EOF
29+
fi
30+
prog=$(mktemp "${TMPDIR:-/tmp}/getver.XXXXXXX")
31+
gcc -I$ROOT -o $prog $getver_c
32+
ver=`$prog`
33+
rm -f $prog $getver_c
34+
echo $ver
35+
exit 0

‎opt/pack.sh

Lines changed: 140 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,38 @@
11
#!/bin/bash
22

3+
error() {
4+
>&2 echo "$0: There are errors."
5+
exit 1
6+
}
7+
8+
if [[ -z $_Dbg_DEBUGGER_LEVEL ]]; then
9+
trap error ERR
10+
fi
11+
12+
#----------------------------------------------------------------------------------------------
13+
14+
if [[ $1 == --help || $1 == help ]]; then
15+
cat <<-END
16+
pack.sh [cpu|gpu] [--help|help]
17+
18+
Argument variables:
19+
DEVICE=cpu|gpu CPU or GPU variants
20+
RAMP=1 Build RAMP file
21+
DEPS=1 Build dependencies file
22+
23+
VARIANT=name Build variant (empty for standard packages)
24+
BRANCH=branch Branch names to serve as an exta package tag
25+
GITSHA=1 Append Git SHA to shapshot package names
26+
27+
BINDIR=dir Directory in which packages are created
28+
INSTALL_DIR=dir Directory in which artifacts are found
29+
30+
END
31+
exit 0
32+
fi
33+
34+
#----------------------------------------------------------------------------------------------
35+
336
[[ $IGNERR == 1 ]] || set -e
437
[[ $VERBOSE == 1 ]] && set -x
538

@@ -10,142 +43,148 @@
1043
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
1144
. $HERE/readies/shibumi/functions
1245
ROOT=$(realpath $HERE/..)
46+
READIES=$ROOT/opt/readies/bin
1347

1448
RAMP_PROG="python3 -m RAMP.ramp"
15-
REDIS_ENT_LIB_PATH=/opt/redislabs/lib
1649

1750
BINDIR=$(realpath $BINDIR)
1851
INSTALL_DIR=$(realpath $INSTALL_DIR)
1952

20-
. $ROOT/opt/readies/bin/enable-utf8
53+
. $READIES/enable-utf8
54+
55+
export ARCH=$($READIES/platform --arch)
56+
export OS=$($READIES/platform --os)
57+
export OSNICK=$($READIES/platform --osnick)
58+
59+
export PRODUCT=redisai
60+
export PRODUCT_LIB=$PRODUCT.so
2161

22-
export ARCH=$($ROOT/opt/readies/bin/platform --arch)
23-
export OS=$($ROOT/opt/readies/bin/platform --os)
24-
export OSNICK=$($ROOT/opt/readies/bin/platform --osnick)
62+
export PACKAGE_NAME=${PACKAGE_NAME:-${PRODUCT}}
63+
64+
# BACKENDS="all torch tensorflow onnxruntime tflite"
65+
BACKENDS="torch tensorflow onnxruntime tflite"
66+
67+
#----------------------------------------------------------------------------------------------
2568

2669
pack_ramp() {
27-
echo "Building RAMP file ..."
2870
cd $ROOT
29-
30-
local STEM=$PRODUCT.$OS-$OSNICK-$ARCH
31-
local FQ_PACKAGE
32-
if [[ -z $BRANCH ]]; then
33-
FQ_PACKAGE=$STEM.$VERSION
71+
72+
local platform="$OS-$OSNICK-$ARCH"
73+
local stem=${PACKAGE_NAME}-${DEVICE}.${platform}
74+
75+
if [[ $SNAPSHOT == 0 ]]; then
76+
local verspec=${SEMVER}${VARIANT}
77+
local packdir=.
78+
local s3base=""
3479
else
35-
FQ_PACKAGE=$STEM.$BRANCH
80+
local verspec=${BRANCH}${VARIANT}
81+
local packdir=snapshots
82+
local s3base=snapshots/
3683
fi
84+
85+
local fq_package=$stem.${verspec}.zip
3786

38-
# this is only to extract {semantic_version} into VERSION
39-
RAMPOUT=$(mktemp /tmp/ramp.XXXXXX)
40-
$RAMP_PROG pack -m $ROOT/ramp.yml --packname-file $RAMPOUT -o $BINDIR/$PRODUCT.{os}-{architecture}.{semantic_version}.zip $INSTALL_DIR/$PRODUCT.so
41-
local rampfile=`realpath $(tail -1 $RAMPOUT)`
42-
rm -f $rampfile $RAMPOUT
43-
echo `basename $rampfile` | sed -e "s/[^.]*\.[^.]*\.\(.*\)\.zip/\1/" > $BINDIR/VERSION
44-
export SEMVER=$(cat $BINDIR/VERSION)
87+
[[ ! -d $BINDIR/$packdir ]] && mkdir -p $BINDIR/$packdir
4588

46-
$RAMP_PROG pack -m $ROOT/ramp.yml -o $BINDIR/$FQ_PACKAGE.zip \
47-
-c "BACKENDSPATH $REDIS_ENT_LIB_PATH/$PRODUCT-$DEVICE-$SEMVER/backends" $INSTALL_DIR/$PRODUCT.so > /dev/null 2>&1
89+
local packdir="$BINDIR/$packdir"
90+
local packfile="$packdir/$fq_package"
91+
local product_so="$INSTALL_DIR/$PRODUCT.so"
4892

49-
cd "$BINDIR"
50-
if [[ -z $BRANCH ]]; then
51-
ln -sf $FQ_PACKAGE.zip $STEM.latest.zip
93+
local xtx_vars=""
94+
for dep in $BACKENDS; do
95+
eval "export NAME_${dep}=${PACKAGE_NAME}_${dep}"
96+
local dep_fname=${PACKAGE_NAME}-${DEVICE}-${dep}.${platform}.${verspec}.tgz
97+
eval "export PATH_${dep}=${s3base}${dep_fname}"
98+
local dep_sha256="$packdir/${dep_fname}.sha256"
99+
eval "export SHA256_${dep}=$(cat $dep_sha256)"
100+
101+
xtx_vars+=" -e NAME_$dep -e PATH_$dep -e SHA256_$dep"
102+
done
103+
104+
python3 $READIES/xtx \
105+
$xtx_vars \
106+
-e DEVICE -e NUMVER -e SEMVER \
107+
$ROOT/ramp.yml > /tmp/ramp.yml
108+
rm -f /tmp/ramp.fname $packfile
109+
$RAMP_PROG pack -m /tmp/ramp.yml --packname-file /tmp/ramp.fname --verbose --debug -o $packfile $product_so >/tmp/ramp.err 2>&1 || true
110+
if [[ ! -e $packfile ]]; then
111+
>&2 echo "Error generating RAMP file:"
112+
>&2 cat /tmp/ramp.err
113+
exit 1
52114
fi
53115

54116
echo "Done."
55117
}
56118

57-
pack_deps() {
58-
echo "Building dependencies file ..."
119+
#----------------------------------------------------------------------------------------------
59120

121+
pack_deps() {
122+
local depname="$1"
123+
60124
cd $ROOT
61125

62-
SEMVER=$(cat $BINDIR/VERSION)
63-
64-
local STEM=$PRODUCT-$DEVICE-dependencies.$OS-$OSNICK-$ARCH
65-
local FQ_PACKAGE
66-
if [[ -z $BRANCH ]]; then
67-
FQ_PACKAGE=$STEM.$VERSION
68-
else
69-
FQ_PACKAGE=$STEM.$BRANCH
70-
fi
71-
72-
cd $INSTALL_DIR
73-
local BACKENDS_DIR=$PRODUCT-$DEVICE-$SEMVER
74-
if [[ ! -h backends ]]; then
75-
[[ ! -d backends ]] && { echo "install-$DEVICE/backend directory not found." ; exit 1; }
76-
rm -rf $BACKENDS_DIR
77-
mkdir $BACKENDS_DIR
126+
local platform="$OS-$OSNICK-$ARCH"
127+
local stem=${PACKAGE_NAME}-${DEVICE}-${depname}.${platform}
128+
local fq_package=$stem.${SEMVER}${VARIANT}.tgz
129+
local tar_path=$BINDIR/$fq_package
130+
local backends_prefix_dir=""
78131

79-
mv backends $BACKENDS_DIR
80-
ln -s $BACKENDS_DIR/backends backends
132+
if [[ $depname == all ]]; then
133+
local backens_dir=.
134+
else
135+
local backens_dir=${PRODUCT}_$depname
81136
fi
82-
find $BACKENDS_DIR -name "*.so*" | xargs tar pczf $BINDIR/$FQ_PACKAGE.tgz
83137

84-
cd "$BINDIR"
85-
if [[ -z $BRANCH ]]; then
86-
ln -sf $FQ_PACKAGE.tgz $STEM.latest.tgz
138+
cd $INSTALL_DIR/backends
139+
{ find $backens_dir -name "*.so*" | \
140+
xargs tar -c --sort=name --owner=root:0 --group=root:0 --mtime='UTC 1970-01-01' --transform "s,^,$backends_prefix_dir," 2>> /tmp/pack.err | \
141+
gzip -n - > $tar_path ; E=$?; } || true
142+
sha256sum $tar_path | gawk '{print $1}' > $tar_path.sha256
143+
144+
mkdir -p $BINDIR/snapshots
145+
cd $BINDIR/snapshots
146+
if [[ ! -z $BRANCH ]]; then
147+
local snap_package=$stem.${BRANCH}${VARIANT}.tgz
148+
ln -sf ../$fq_package $snap_package
149+
ln -sf ../$fq_package.sha256 $snap_package.sha256
87150
fi
88-
89-
echo "Done."
90151
}
91152

92-
if [[ $1 == --help || $1 == help ]]; then
93-
cat <<-END
94-
pack.sh [cpu|gpu] [--help|help]
95-
96-
Argument variables:
97-
BINDIR=dir directory in which packages are created
98-
INSTALL_DIR=dir directory in which artifacts are found
99-
DEVICE=cpu|gpu
100-
BRANCH=branch branch names to serve as an exta package tag
101-
RAMP=1 build RAMP file
102-
DEPS=1 build dependencies file
103-
RAMP_PROG path to RAMP program
153+
#----------------------------------------------------------------------------------------------
104154

105-
END
106-
exit 0
155+
export NUMVER=$(NUMERIC=1 $ROOT/opt/getver)
156+
export SEMVER=$($ROOT/opt/getver)
157+
158+
if [[ ! -z $VARIANT ]]; then
159+
VARIANT=-${VARIANT}
107160
fi
108161

109-
PRODUCT=redisai
110-
PRODUCT_LIB=$PRODUCT.so
162+
[[ -z $BRANCH ]] && BRANCH=${CIRCLE_BRANCH:-`git rev-parse --abbrev-ref HEAD`}
163+
BRANCH=${BRANCH//[^A-Za-z0-9._-]/_}
164+
if [[ $GITSHA == 1 ]]; then
165+
GIT_COMMIT=$(git describe --always --abbrev=7 --dirty="+" 2>/dev/null || git rev-parse --short HEAD)
166+
BRANCH="${BRANCH}-${GIT_COMMIT}"
167+
fi
168+
export BRANCH
111169

112-
if [[ -z $BRANCH ]]; then
113-
tag=`git describe --abbrev=0 2> /dev/null | sed 's/^v\(.*\)/\1/'`
114-
if [[ $? != 0 || -z $tag ]]; then
115-
BRANCH=`git rev-parse --abbrev-ref HEAD`
116-
VERSION=
117-
else
118-
VERSION=$tag
170+
[[ $DEPS == 1 ]] && echo "Building dependencies ..."
171+
for dep in $BACKENDS; do
172+
if [[ $DEPS == 1 ]]; then
173+
echo "$dep ..."
174+
pack_deps $dep
119175
fi
120-
else
121-
VERSION=
122-
fi
176+
done
123177

124-
# GIT_VER=""
125-
# if [[ -d $ROOT/.git ]]; then
126-
# if [[ ! -z $BRANCH ]]; then
127-
# GIT_BRANCH="$BRANCH"
128-
# else
129-
# GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
130-
# fi
131-
# GIT_COMMIT=$(git describe --always --abbrev=7 --dirty="+")
132-
# # GIT_VER="${GIT_BRANCH}-${GIT_COMMIT}"
133-
# GIT_VER="${GIT_BRANCH}"
134-
# else
135-
# if [[ ! -z $BRANCH ]]; then
136-
# GIT_BRANCH="$BRANCH"
137-
# else
138-
# GIT_BRANCH=unknown
139-
# fi
140-
# GIT_VER="$GIT_BRANCH"
141-
# fi
142-
143-
if ! command -v redis-server > /dev/null; then
144-
echo "Cannot find redis-server. Aborting."
145-
exit 1
146-
fi
178+
if [[ $RAMP == 1 ]]; then
179+
if ! command -v redis-server > /dev/null; then
180+
>&2 echo "$0: Cannot find redis-server. Aborting."
181+
exit 1
182+
fi
147183

148-
pack_ramp
149-
[[ $DEPS == 1 ]] && pack_deps
184+
echo "Building RAMP files ..."
185+
SNAPSHOT=0 pack_ramp
186+
SNAPSHOT=1 pack_ramp
187+
echo "Done."
188+
fi
150189

151190
exit 0

‎opt/readies

‎opt/system-setup.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ def common_first(self):
2828

2929
def debian_compat(self):
3030
self.pip_install("-IU --force-reinstall setuptools")
31+
self.install("gawk")
3132
self.install("build-essential cmake")
3233
self.install("python3-regex")
3334
self.install("python3-venv python3-psutil python3-networkx python3-numpy") # python3-skimage

‎ramp.yml

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ email: support@redislabs.com
44
description: Serving tensors and executing deep learning graphs
55
homepage: https://oss.redislabs.com/redisai/
66
license: Redis Source Available License v1.0
7-
command_line_args: ""
7+
command_line_args: "BACKENDSPATH /var/opt/redislabs/modules/ai/{{NUMVER}}/deps"
8+
# command_line_args: "BACKENDSPATH /var/opt/redislabs/modules/ai/{{NUMVER}}/deps/backends"
89
min_redis_version: "5.0.7"
910
min_redis_pack_version: "5.4.11"
1011
capabilities:
@@ -22,3 +23,19 @@ exclude_commands:
2223
- ai.modeldel
2324
- ai.scriptset
2425
- ai.scriptdel
26+
dependencies:
27+
{{NAME_tensorflow}}:
28+
url: http://redismodules.s3.amazonaws.com/redisai/{{PATH_tensorflow}}
29+
sha256: {{SHA256_tensorflow}}
30+
{{NAME_torch}}:
31+
url: http://redismodules.s3.amazonaws.com/redisai/{{PATH_torch}}
32+
sha256: {{SHA256_torch}}
33+
{{NAME_onnxruntime}}:
34+
url: http://redismodules.s3.amazonaws.com/redisai/{{PATH_onnxruntime}}
35+
sha256: {{SHA256_onnxruntime}}
36+
{{NAME_tflite}}:
37+
url: http://redismodules.s3.amazonaws.com/redisai/{{PATH_tflite}}
38+
sha256: {{SHA256_tflite}}
39+
# {{NAME_all}}:
40+
# url: http://redismodules.s3.amazonaws.com/redisai/{{PATH_all}}
41+
# sha256: {{SHA256_all}}

0 commit comments

Comments
 (0)