Skip to content

Commit 5045c12

Browse files
rafierafie
authored
CircleCI: added QA automation (#660)
1 parent 2cf7050 commit 5045c12

File tree

11 files changed

+227
-35
lines changed

11 files changed

+227
-35
lines changed

‎.circleci/config.yml

Lines changed: 45 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ jobs:
167167
168168
build-and-test:
169169
docker:
170-
- image: redisfab/rmbuilder:6.0.9-x64-buster
170+
- image: redisfab/rmbuilder:6.2.1-x64-buster
171171
steps:
172172
- build-steps:
173173
platform: debian
@@ -183,7 +183,7 @@ jobs:
183183

184184
coverage:
185185
docker:
186-
- image: redisfab/rmbuilder:6.0.9-x64-buster
186+
- image: redisfab/rmbuilder:6.2.1-x64-buster
187187
steps:
188188
- abort_for_docs
189189
- checkout
@@ -217,7 +217,7 @@ jobs:
217217
type: string
218218
default: "CLUSTER=0 AOF=0"
219219
docker:
220-
- image: redisfab/rmbuilder:6.0.9-x64-buster
220+
- image: redisfab/rmbuilder:6.2.1-x64-buster
221221
steps:
222222
- abort_for_docs
223223
- checkout
@@ -325,7 +325,7 @@ jobs:
325325
location:
326326
type: string
327327
docker:
328-
- image: redisfab/rmbuilder:6.0.9-x64-buster
328+
- image: redisfab/rmbuilder:6.2.1-x64-buster
329329
steps:
330330
- abort_for_docs
331331
- early_return_for_forked_pull_requests
@@ -342,7 +342,7 @@ jobs:
342342
343343
deploy-snapshot:
344344
docker:
345-
- image: redisfab/rmbuilder:6.0.9-x64-buster
345+
- image: redisfab/rmbuilder:6.2.1-x64-buster
346346
steps:
347347
- abort_for_docs
348348
- early_return_for_forked_pull_requests
@@ -365,7 +365,7 @@ jobs:
365365
366366
deploy-release:
367367
docker:
368-
- image: redisfab/rmbuilder:6.0.9-x64-buster
368+
- image: redisfab/rmbuilder:6.2.1-x64-buster
369369
steps:
370370
- abort_for_docs
371371
- early_return_for_forked_pull_requests
@@ -380,6 +380,26 @@ jobs:
380380
aws s3 cp --no-progress $f s3://redismodules/$PACKAGE_NAME/ --acl public-read
381381
done
382382
383+
release-automation:
384+
docker:
385+
- image: redisfab/rmbuilder:6.2.1-x64-buster
386+
steps:
387+
- checkout
388+
- setup-automation
389+
- run:
390+
name: Run QA Automation
391+
command: MODULE_VERSION=$CIRCLE_TAG VERBOSE=1 TEST=release ./tests/qa/run
392+
393+
nightly-automation:
394+
docker:
395+
- image: redisfab/rmbuilder:6.2.1-x64-buster
396+
steps:
397+
- checkout
398+
- setup-automation
399+
- run:
400+
name: Run QA Automation
401+
command: MODULE_VERSION=$CIRCLE_BRANCH VERBOSE=1 TEST=nightly QUICK=1 ./tests/qa/run
402+
383403

384404
on-any-branch: &on-any-branch
385405
filters:
@@ -408,6 +428,17 @@ on-integ-branch: &on-integ-branch
408428
only:
409429
- master
410430
- /^\d+\.\d+.*$/
431+
- /^feature-.*$/
432+
tags:
433+
ignore: /.*/
434+
435+
not-on-integ-branch: &not-on-integ-branch
436+
filters:
437+
branches:
438+
ignore:
439+
- master
440+
- /^\d+\.\d+.*$/
441+
- /^feature-.*$/
411442
tags:
412443
ignore: /.*/
413444

@@ -424,6 +455,7 @@ on-integ-and-version-tags: &on-integ-and-version-tags
424455
only:
425456
- master
426457
- /^\d+\.\d+.*$/
458+
- /^feature-.*$/
427459
tags:
428460
only: /^v[0-9].*/
429461

@@ -493,6 +525,11 @@ workflows:
493525
context: common
494526
<<: *after-platform-builds
495527
<<: *on-version-tags
528+
- release-automation:
529+
context: common
530+
<<: *on-version-tags
531+
requires:
532+
- deploy-release
496533

497534
nightly:
498535
triggers:
@@ -504,3 +541,5 @@ workflows:
504541
jobs:
505542
- build-macos:
506543
<<: *never # temporarily disabled
544+
- nightly-automation:
545+
context: common

‎Dockerfile

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,20 +40,18 @@ RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh cpu; else env $DEPS_ARGS ./get_
4040

4141
ARG BUILD_ARGS=""
4242
ADD ./ /build
43-
RUN bash -c "set -e ;\
44-
. ./opt/readies/bin/sourced ./profile.d ;\
45-
make -C opt build $BUILD_ARGS SHOW=1"
43+
RUN bash -l -c "make -C opt build $BUILD_ARGS SHOW=1"
4644

4745
ARG PACK
4846
ARG TEST
4947

5048
RUN mkdir -p bin/artifacts
5149
RUN set -e ;\
52-
if [ "$PACK" = "1" ]; then make -C opt pack; fi
50+
if [ "$PACK" = "1" ]; then bash -l -c "make -C opt pack"; fi
5351

5452
RUN set -e ;\
5553
if [ "$TEST" = "1" ]; then \
56-
TEST= make -C opt test $BUILD_ARGS NO_LFS=1 ;\
54+
bash -l -c "TEST= make -C opt test $BUILD_ARGS NO_LFS=1" ;\
5755
if [[ -d test/logs ]]; then \
5856
tar -C test/logs -czf bin/artifacts/test-logs-cpu.tgz . ;\
5957
fi ;\

‎Dockerfile.gpu

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -52,23 +52,21 @@ RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh gpu; else env $DEPS_ARGS ./get_
5252

5353
ARG BUILD_ARGS=""
5454
ADD ./ /build
55-
RUN bash -c "set -e ;\
56-
. ./opt/readies/bin/sourced ./profile.d ;\
57-
make -C opt build GPU=1 $BUILD_ARGS SHOW=1"
55+
RUN bash -l -c "make -C opt build GPU=1 $BUILD_ARGS SHOW=1"
5856

5957
ARG PACK
6058
ARG TEST
6159

6260
RUN mkdir -p bin/artifacts
6361
RUN set -e ;\
64-
if [ "$PACK" = "1" ]; then make -C opt pack GPU=1; fi
62+
if [ "$PACK" = "1" ]; then bash -l -c "make -C opt pack GPU=1"; fi
6563

6664
RUN set -e ;\
6765
if [ "$TEST" = "1" ]; then \
68-
TEST= make -C opt test GPU=1 $BUILD_ARGS NO_LFS=1 ;\
69-
if [[ -d test/logs ]]; then \
70-
tar -C test/logs -czf bin/artifacts/test-logs-gpu.tgz . ;\
71-
fi ;\
66+
bash -l -c "TEST= make -C opt test GPU=1 $BUILD_ARGS NO_LFS=1" ;\
67+
if [[ -d test/logs ]]; then \
68+
tar -C test/logs -czf bin/artifacts/test-logs-gpu.tgz . ;\
69+
fi ;\
7270
fi
7371

7472
#----------------------------------------------------------------------------------------------

‎Dockerfile.gpu-test

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -57,15 +57,14 @@ ARG BUILD_ARGS=""
5757
ADD ./ /build
5858
RUN set -e ;\
5959
. venv/bin/activate ;\
60-
. ./opt/readies/bin/sourced ./profile.d ;\
61-
make -C opt build GPU=1 $BUILD_ARGS SHOW=1
60+
bash -l -c "make -C opt build GPU=1 $BUILD_ARGS SHOW=1"
6261

6362
ARG PACK
6463

6564
RUN set -e ;\
6665
if [[ $PACK == 1 ]]; then \
67-
. venv/bin/activate ;\
68-
make -C opt pack GPU=1 VERBOSE=1 ;\
66+
. venv/bin/activate ;\
67+
bash -l -c "make -C opt pack GPU=1 VERBOSE=1" ;\
6968
fi
7069

7170
RUN git remote set-url origin https://github.com/RedisAI/RedisAI

‎opt/pack.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,8 @@ RAMP_PROG="python3 -m RAMP.ramp"
5050
BINDIR=$(realpath $BINDIR)
5151
INSTALL_DIR=$(realpath $INSTALL_DIR)
5252

53-
. $READIES/enable-utf8
53+
$READIES/enable-utf8
54+
source /etc/profile.d/utf8.sh
5455

5556
export ARCH=$($READIES/platform --arch)
5657
export OS=$($READIES/platform --os)

‎opt/readies

‎opt/system-setup.py

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@
55
import argparse
66

77
HERE = os.path.abspath(os.path.dirname(__file__))
8-
READIES = os.path.join(HERE, "readies")
98
ROOT = os.path.abspath(os.path.join(HERE, ".."))
9+
READIES = os.path.join(ROOT, "opt/readies")
1010
sys.path.insert(0, READIES)
1111
import paella
1212

@@ -21,43 +21,43 @@ def common_first(self):
2121
self.pip_install("wheel")
2222

2323
self.install("git unzip patchelf")
24-
self.install("coreutils") # for realpath
24+
if self.osnick != 'centos8':
25+
self.install("coreutils") # for realpath
2526

2627
def debian_compat(self):
28+
self.run("%s/bin/enable-utf8" % READIES)
29+
self.run("%s/bin/getgcc" % READIES)
2730
self.install("gawk")
28-
self.install("build-essential")
2931
self.install("libssl-dev")
3032
self.install("python3-regex")
31-
self.install("python3-psutil python3-networkx python3-numpy")
33+
self.install("python3-networkx python3-numpy")
3234
if self.platform.is_arm():
3335
self.install("python3-dev") # python3-skimage
3436
self.install("libmpich-dev libopenblas-dev") # for libtorch
3537
self.install_git_lfs_on_linux()
3638

3739
def redhat_compat(self):
38-
self.install("redhat-lsb-core")
3940
self.run("%s/bin/enable-utf8" % READIES)
41+
self.run("%s/bin/getepel" % READIES)
42+
self.install("redhat-lsb-core")
4043

4144
self.run("%s/bin/getgcc --modern" % READIES)
42-
# self.install("llvm-toolset-7")
4345

4446
if self.arch == 'x64':
4547
self.install_linux_gnu_tar()
4648

4749
if not self.dist == "amzn":
4850
self.install("epel-release")
4951
self.install("python3-devel libaec-devel")
50-
self.install("python36-psutil")
5152
else:
5253
self.run("amazon-linux-extras install epel", output_on_error=True)
5354
self.install("python3-devel")
54-
self.pip_install("psutil")
5555

5656
self.install_git_lfs_on_linux()
5757

5858
def fedora(self):
59-
self.group_install("'Development Tools'")
60-
self.install("python3 python3-psutil python3-networkx")
59+
self.run("%s/bin/getepel" % READIES)
60+
self.install("python3-networkx")
6161
self.install_git_lfs_on_linux()
6262

6363
def linux_last(self):

‎tests/qa/RS_VERSIONS

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
6.0.12-41
2+
5.4.14-37
3+
6.0.8-30
4+
100.0.0-2199

‎tests/qa/nightly.json

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{
2+
"service_id": "single_module_test_cycle",
3+
"name": "redisai automation-testing",
4+
"properties": {
5+
"sut_version": "master",
6+
"email_recipients": "redisaidev-aaaacob2o7eeecrhkqwua77kku@redislabs.slack.com",
7+
"sut_environments": [],
8+
"tools_environment": {},
9+
"modules_version": "master",
10+
"test_names_modules": [
11+
"{{RS_MODULE}}"
12+
],
13+
"cycle_environments_setup": [
14+
{
15+
"teardown": true,
16+
"name": "bionic-amd64-aws",
17+
"concurrency": 1,
18+
"module_url": "http://redismodules.s3.amazonaws.com/{{RS_MODULE_FILE_PREFIX}}/snapshots/{{RS_MODULE_FILE_PREFIX}}-cpu.linux-bionic-x64.master.zip"
19+
}
20+
]
21+
}
22+
}

‎tests/qa/release.json

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
{
2+
"service_id": "single_module_test_cycle",
3+
"name": "redisai automation-testing",
4+
"properties": {
5+
"sut_version": "{{RS_VERSION}}",
6+
"email_recipients": "redisaidev-aaaacob2o7eeecrhkqwua77kku@redislabs.slack.com",
7+
"sut_environments": [],
8+
"tools_environment": {},
9+
"modules_version": "{{MODULE_VERSION}}",
10+
"test_names_modules": [
11+
"{{RS_MODULE}}"
12+
],
13+
"cycle_environments_setup": [
14+
{
15+
"teardown": true,
16+
"name": "xenial-amd64-aws",
17+
"concurrency": 1,
18+
"module_url": "http://redismodules.s3.amazonaws.com/{{RS_MODULE_FILE_PREFIX}}/{{RS_MODULE_FILE_PREFIX}}-cpu.linux-xenial-x64.{{MODULE_VERSION}}.zip"
19+
},
20+
{
21+
"teardown": true,
22+
"name": "bionic-amd64-aws",
23+
"concurrency": 1,
24+
"module_url": "http://redismodules.s3.amazonaws.com/{{RS_MODULE_FILE_PREFIX}}/{{RS_MODULE_FILE_PREFIX}}-cpu.linux-bionic-x64.{{MODULE_VERSION}}.zip"
25+
}
26+
]
27+
}
28+
}

0 commit comments

Comments
 (0)