Skip to content

Commit e339bfd

Browse files
rafierafie
committed
Build fixes #24 
1 parent a2db3d2 commit e339bfd

File tree

3 files changed

+70
-69
lines changed

3 files changed

+70
-69
lines changed

‎automation/Makefile

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,10 @@ make setup # install prerequisited (CAUTION: THIS WILL MODIFY YOUR SYSTEM)
1010
make fetch # download and prepare dependant modules
1111
make build # compile and link
1212
make clean # remove build artifacts
13+
ALL=1 # remove entire artifacts directory
1314
make test # run tests
1415
make pack # create installation packages
16+
PACK_DEPS=0 # do not pack dependencies
1517
make deploy # copy packages to S3
1618
make release # release a version
1719
endef
@@ -87,16 +89,14 @@ fetch deps:
8789

8890
#----------------------------------------------------------------------------------------------
8991

90-
pack: $(BINDIR)/BINDIR pack_ramp pack_deps
91-
92-
$(BINDIR)/BINDIR: $(BIN_DIRS)
93-
$(SHOW)echo $(BINDIR)>$(BINDIR)/BINDIR
94-
95-
pack_ramp:
96-
$(SHOW)RAMP=1 DEVICE=$(DEVICE) BINDIR=$(BINROOT) ./pack.sh
92+
pack: $(BINDIR)/BINDIR
93+
$(SHOW)DEVICE=$(DEVICE) BINDIR=$(BINROOT) INSTALL_DIR=$(INSTALL_DIR) DEPS=0 ./pack.sh
94+
ifneq ($(PACK_DEPS),0)
95+
$(SHOW)DEVICE=$(DEVICE) BINDIR=$(BINROOT) INSTALL_DIR=$(INSTALL_DIR) DEPS=1 ./pack.sh
96+
endif
9797

98-
pack_deps: pack_ramp
99-
$(SHOW)DEPS=1 DEVICE=$(DEVICE) BINDIR=$(BINROOT) ./pack.sh
98+
#$(BINDIR)/BINDIR: $(BIN_DIRS)
99+
# $(SHOW)echo $(BINDIR)>$(BINDIR)/BINDIR
100100

101101
#----------------------------------------------------------------------------------------------
102102

@@ -107,7 +107,4 @@ test:
107107
$(SHOW)cd $(ROOT); git lfs pull
108108
$(SHOW)set -e ;\
109109
cd $(ROOT)/test ;\
110-
python3 -m RLTest $(TEST_ARGS) --test basic_tests.py \
111-
--module $(INSTALL_DIR)/redisai.so
112-
113-
# --junitxml=$(TEST_REPORT_DIR)/tests_report.xml
110+
python3 -m RLTest $(TEST_ARGS) --test basic_tests.py --module $(INSTALL_DIR)/redisai.so

‎automation/pack.sh

Lines changed: 59 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
[[ -z $DEVICE ]] && { echo DEVICE undefined; exit 1; }
44
[[ -z $BINDIR ]] && { echo BINDIR undefined; exit 1; }
5+
[[ -z $INSTALL_DIR ]] && { echo INSTALL_DIR undefined; exit 1; }
6+
[[ ! -z $INTO ]] && INTO=$(realpath $INTO)
57

68
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
79
. $HERE/../automation/readies/shibumi/functions
@@ -11,19 +13,33 @@ RAMP_PROG=ramp
1113
REDIS_ENT_LIB_PATH=/opt/redislabs/lib
1214

1315
BINDIR=$(realpath $BINDIR)
16+
INSTALL_DIR=$(realpath $INSTALL_DIR)
1417

1518
pack_ramp() {
1619
echo "Building RAMP file ..."
1720
cd $ROOT
1821
RAMPOUT=$(mktemp /tmp/ramp.XXXXXX)
19-
echo BINDIR=$BINDIR
20-
$RAMP_PROG pack -m $ROOT/ramp.yml -o $BINDIR/$PRODUCT.{os}-{architecture}.{semantic_version}.zip $INSTALL_DIR/redisai.so 2> /dev/null | grep '.zip' > $RAMPOUT
22+
$RAMP_PROG pack -m $ROOT/ramp.yml -o $BINDIR/$PRODUCT.{os}-{architecture}.{semantic_version}.zip $INSTALL_DIR/$PRODUCT.so 2> /dev/null | grep '.zip' > $RAMPOUT
2123
realpath $(tail -1 $RAMPOUT) > $BINDIR/PACKAGE
2224
cat $BINDIR/PACKAGE | sed -e "s/[^.]*\.[^.]*\.\(.*\)\.zip/\1/" > $BINDIR/VERSION
25+
cat $BINDIR/PACKAGE | sed -e "s/[^.]*\.\([^.]*\)\..*\.zip/\1/" > $BINDIR/OSARCH
26+
PACKAGE=$(cat $BINDIR/PACKAGE)
2327
VERSION=$(cat $BINDIR/VERSION)
24-
$RAMP_PROG pack -m $ROOT/ramp.yml -o "build/redisai.{os}-{architecture}.{semantic_version}.zip" \
25-
-c "BACKENDSPATH $REDIS_ENT_LIB_PATH/redisai-cpu-$VERSION/backends" $INSTALL_DIR/redisai.so 2> /dev/null | grep '.zip' > $RAMPOUT
28+
OSARCH=$(cat $BINDIR/OSARCH)
29+
$RAMP_PROG pack -m $ROOT/ramp.yml -o "build/$PRODUCT.{os}-{architecture}.{semantic_version}.zip" \
30+
-c "BACKENDSPATH $REDIS_ENT_LIB_PATH/$PRODUCT-$DEVICE-$VERSION/backends" $INSTALL_DIR/$PRODUCT.so 2> /dev/null | grep '.zip' > $RAMPOUT
2631
rm -f $RAMPOUT
32+
export PACK_FNAME="$(basename $PACKAGE)"
33+
34+
cd "$BINDIR"
35+
RAMP_STEM=$PRODUCT.$OSARCH
36+
ln -sf $PACK_FNAME $RAMP_STEM.latest.zip
37+
[[ ! -z $BRANCH ]] && ln -sf $PACK_FNAME $RAMP_STEM.${BRANCH}.zip
38+
ln -sf $PACK_FNAME $RAMP_STEM.${GIT_VER}.zip
39+
40+
export RELEASE_ARTIFACTS="$RELEASE_ARTIFACTS $PACK_FNAME $RAMP_STEM.latest.zip"
41+
export DEV_ARTIFACTS="$DEV_ARTIFACTS $RAMP_STEM.${BRANCH}.zip $RAMP_STEM.${GIT_VER}.zip"
42+
2743
echo "Done."
2844
}
2945

@@ -32,18 +48,29 @@ pack_deps() {
3248

3349
cd $ROOT
3450
PACK_FNAME=$(basename `cat $BINDIR/PACKAGE`)
35-
ARCHOSVER=$(echo "$PACK_FNAME" | sed -e "s/^redisai\.\([^.]*\..*\)\.zip/\1/")
51+
ARCHOSVER=$(echo "$PACK_FNAME" | sed -e "s/^[^.]*\.\([^.]*\..*\)\.zip/\1/")
3652
VERSION=$(cat $BINDIR/VERSION)
37-
cd install-$DEVICE
53+
cd $INSTALL_DIR
3854
if [[ ! -h backends ]]; then
3955
[[ ! -d backends ]] && { echo "install-$DEVICE/backend directory not found." ; exit 1; }
40-
rm -rf redisai-$DEVICE-$VERSION
41-
mkdir redisai-$DEVICE-$VERSION
56+
rm -rf $PRODUCT-$DEVICE-$VERSION
57+
mkdir $PRODUCT-$DEVICE-$VERSION
4258

43-
mv backends redisai-$DEVICE-$VERSION
44-
ln -s redisai-$DEVICE-$VERSION/backends backends
59+
mv backends $PRODUCT-$DEVICE-$VERSION
60+
ln -s $PRODUCT-$DEVICE-$VERSION/backends backends
4561
fi
46-
find redisai-$DEVICE-$VERSION -name "*.so*" | xargs tar pczf redisai-$DEVICE-dependencies.$ARCHOSVER.tgz
62+
find $PRODUCT-$DEVICE-$VERSION -name "*.so*" | xargs tar pczf $BINDIR/$PRODUCT-$DEVICE-dependencies.$ARCHOSVER.tgz
63+
export DEPS_FNAME="$PRODUCT-$DEVICE-dependencies.$ARCHOSVER.tgz"
64+
65+
cd "$BINDIR"
66+
DEPS_STEM=$PRODUCT-$DEVICE-dependencies.$OSARCH
67+
ln -sf $DEPS_FNAME $DEPS_STEM.latest.tgz
68+
[[ ! -z $BRANCH ]] && ln -sf $DEPS_FNAME $DEPS_STEM.${BRANCH}.tgz
69+
ln -sf $DEPS_FNAME $DEPS_STEM.${GIT_VER}.tgz
70+
71+
export RELEASE_ARTIFACTS="$RELEASE_ARTIFACTS $DEPS_FNAME $DEPS_STEM.latest.zip"
72+
export DEV_ARTIFACTS="$DEV_ARTIFACTS $DEPS_STEM.${BRANCH}.zip $DEPS_STEM.${GIT_VER}.zip"
73+
4774
echo "Done."
4875
}
4976

@@ -55,21 +82,19 @@ if [[ $1 == --help || $1 == help ]]; then
5582
pack.sh [cpu|gpu] [--help|help]
5683
5784
Argument variables:
85+
BINDIR=dir directory in which packages are created
86+
INSTALL_DIR=dir directory in which artifacts are found
5887
DEVICE=cpu|gpu
59-
BRANCH=branch branch names to serve as an exta package tag
60-
INTO=dir package destination directory (optinal)
61-
RAMP=1 build RAMP file
62-
DEPS=1 build dependencies file
63-
RAMP_PROG path to RAMP program
88+
BRANCH=branch branch names to serve as an exta package tag
89+
INTO=dir package destination directory (optinal)
90+
RAMP=1 build RAMP file
91+
DEPS=1 build dependencies file
92+
RAMP_PROG path to RAMP program
6493
6594
END
6695
exit 0
6796
fi
6897

69-
# BINDIR=$(cat $ROOT/BINDIR)
70-
BIN=$ROOT/bin
71-
INSTALL_DIR=$ROOT/install
72-
7398
PRODUCT=redisai
7499
PRODUCT_LIB=$PRODUCT.so
75100

@@ -80,54 +105,32 @@ if [[ -d $ROOT/.git ]]; then
80105
GIT_VER="${GIT_BRANCH}-${GIT_COMMIT}"
81106
fi
82107

83-
# OSX=""
84-
# if [[ $(./deps/readies/bin/platform --os) == macosx ]]; then
85-
# OSX=1
86-
# export PATH=$PATH:$HOME/Library/Python/2.7/bin
87-
# fi
108+
OSX=""
109+
if [[ $(./readies/bin/platform --os) == macosx ]]; then
110+
# macOS: ramp is installed here
111+
OSX=1
112+
export PATH=$PATH:$HOME/Library/Python/2.7/bin
113+
fi
88114

89115
if ! command -v redis-server > /dev/null; then
90116
echo "Cannot find redis-server. Aborting."
91117
exit 1
92118
fi
93119

94120
pack_ramp
95-
pack_deps
96-
echo "Done."
97-
exit 0
98-
99-
100-
cd "$BINDIR"
101-
ln -s $PACK_FNAME $PRODUCT.latest.zip
102-
ln -s $DEPS_FNAME $PRODUCT-dependencies.latest.tgz
103-
104-
if [[ ! -z $BRANCH ]]; then
105-
ln -s $PACK_FNAME $PRODUCT.${BRANCH}.zip
106-
ln -s $DEPS_FNAME $PRODUCT-dependencies.${BRANCH}.tgz
107-
fi
108-
ln -s $PACK_FNAME $PRODUCT.${GIT_VER}.zip
109-
ln -s $DEPS_FNAME $PRODUCT-dependencies.${GIT_VER}.tgz
110-
111-
RELEASE_ARTIFACTS=\
112-
$PACK_FNAME $DEPS_FNAME \
113-
$PRODUCT.latest.zip $PRODUCT-dependencies.latest.tgz
114-
115-
DEV_ARTIFACTS=\
116-
$PRODUCT.${BRANCH}.zip $PRODUCT-dependencies.${BRANCH}.tgz \
117-
$PRODUCT.${GIT_VER}.zip $PRODUCT-dependencies.${GIT_VER}.tgz
121+
[[ $DEPS == 1 ]] && pack_deps
118122

119123
if [[ ! -z $INTO ]]; then
120-
INTO=$(realpath $INTO)
121124
mkdir -p $INTO/release $INTO/branch
122125
cd $INTO/release
123-
foreach f in ($RELEASE_ARTIFACTS)
124-
ln -s $f
125-
end
126+
for f in $RELEASE_ARTIFACTS; do
127+
[[ -f $BINDIR/$f ]] && ln -sf $(realpath --relative-to . $BINDIR/$f)
128+
done
126129

127130
cd $INTO/branch
128-
foreach f in ($DEV_ARTIFACTS)
129-
ln -s $f
130-
end
131+
for f in $DEV_ARTIFACTS; do
132+
[[ -f $BINDIR/$f ]] && ln -sf $(realpath --relative-to . $BINDIR/$f)
133+
done
131134
fi
132135

133-
echo "Done."
136+
exit 0

‎automation/system-setup.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ def common_first(self):
2323
if self.os == 'linux':
2424
self.install("ca-certificates")
2525
self.install("git cmake unzip wget patchelf awscli")
26+
self.install("coreutils") # for realpath
2627

2728
def debian_compat(self):
2829
self.install("build-essential")

0 commit comments

Comments
 (0)