diff --git a/.circleci/config.yml b/.circleci/config.yml index da95b42ed..56fe7236c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,8 +1,8 @@ -version: 2 +version: 2.1 jobs: build: docker: - - image: circleci/python:3.6.8 + - image: circleci/python:3.7.4-buster steps: - checkout @@ -12,58 +12,38 @@ jobs: keys: - v1-dependencies-{{ checksum "get_deps.sh" }} # fallback to using the latest cache if no exact match is found - - v1-dependencies- + # - v1-dependencies- - run: name: install dependencies command: | - curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash - sudo apt-get install -y git-lfs patchelf - git lfs install - curl -o cmake-3.12.4-Linux-x86_64.sh https://cmake.org/files/v3.12/cmake-3.12.4-Linux-x86_64.sh - mkdir /home/circleci/project/cmake-bin - sh cmake-3.12.4-Linux-x86_64.sh --prefix=/home/circleci/project/cmake-bin --skip-license - sudo ln -s /home/circleci/project/cmake-bin/bin/cmake /usr/local/bin/cmake - echo `cmake --version` - rm -rf /home/circleci/project/deps - bash get_deps.sh cpu + sudo ./automation/readies/bin/getpy + sudo ./automation/system-setup.py + ./get_deps.sh cpu git clone git://github.com/antirez/redis.git --branch 5.0.3 (cd redis && make malloc=libc -j4 && sudo make install) - save_cache: paths: - deps - key: v1-dependencies-{{ checksum "get_deps.sh" }} + key: v2-dependencies-{{ checksum "get_deps.sh" }} - run: name: Build command: | - rm -rf build - mkdir build - cd build - cmake -DDEPS_PATH=../deps/install .. - make && make install - cd .. - make rlec_runpath_fix + make -C automation all + make -C automation pack - run: name: Test - command: | - python3 -m venv venv - . venv/bin/activate - pip -q uninstall -y RLTest || true - pip -q install git+https://github.com/RedisLabsModules/RLTest@py3 - pip -q install -r test/test_requirements.txt - git lfs pull - python3 -m RLTest --test test/basic_tests.py --module install/redisai.so + command: make -C automation test - run: name: Persist Artifacts command: | mkdir -p ~/workspace/build - cp install/ramp/$MODULE_ARTIFACT ~/workspace/ - # cp deps/install/lib/*.so* ~/workspace/ - cp -r install/backends ~/workspace/ + cp install-cpu/$MODULE_ARTIFACT ~/workspace/ + cp -r install-cpu/backends ~/workspace/ cp ramp.yml ~/workspace/ - persist_to_workspace: root: ~/workspace diff --git a/.dockerignore b/.dockerignore index caac77506..9418ba6fc 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,7 @@ -deps/* -install/* +/deps/ +/build/ +/install*/ **/*.o -**/*.so \ No newline at end of file +**/*.so +.venv/ +venv*/ diff --git a/.gitignore b/.gitignore index 7d100019b..0bb1f7455 100644 --- a/.gitignore +++ b/.gitignore @@ -7,11 +7,16 @@ __pycache__ *.pyc # Dirs -install/ -deps/ -build/ +/install*/ +/deps/* +/build/ examples/js/node_modules/ examples/js/package-lock.json +/test/venv/ +/test/logs/ +/1/ +/venv/ +/.venv/ # Eclipse (if that's your jam...) .project diff --git a/CMakeLists.txt b/CMakeLists.txt index 7188de92d..58cc0cc4f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,6 +7,10 @@ IF (NOT DEPS_PATH) # MESSAGE(FATAL_ERROR "DEPS PATH Missing!") ENDIF() +IF (NOT DEVICE) + SET(DEVICE cpu) +ENDIF() + FUNCTION(ADD_LDFLAGS _TARGET NEW_FLAGS) GET_TARGET_PROPERTY(LD_FLAGS ${_TARGET} LINK_FLAGS) IF(LD_FLAGS) @@ -19,6 +23,8 @@ GET_FILENAME_COMPONENT(depsAbs "${DEPS_PATH}" REALPATH BASE_DIR "${CMAKE_BINARY_DIR}") +# SET(CUDA_TOOLKIT_ROOT_DIR /usr/local/cuda-9.0) + INCLUDE_DIRECTORIES(${depsAbs}/dlpack/include) INCLUDE_DIRECTORIES(${depsAbs}/libtensorflow/include) INCLUDE_DIRECTORIES(${depsAbs}/libtorch/include) @@ -60,7 +66,7 @@ IF (APPLE) LINK_FLAGS "-undefined dynamic_lookup") ENDIF() -SET(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/install) +SET(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/install-${DEVICE}) INSTALL(TARGETS redisai LIBRARY DESTINATION .) diff --git a/Dockerfile b/Dockerfile old mode 100644 new mode 100755 index 24889bc42..e01c43072 --- a/Dockerfile +++ b/Dockerfile @@ -1,45 +1,38 @@ -FROM redis AS builder +ARG OS=debian:buster -ENV DEPS "build-essential git ca-certificates curl unzip wget libgomp1 patchelf" +#---------------------------------------------------------------------------------------------- +FROM redis AS redis +FROM ${OS} AS builder -# install latest cmake -ADD https://cmake.org/files/v3.12/cmake-3.12.4-Linux-x86_64.sh /cmake-3.12.4-Linux-x86_64.sh -RUN mkdir /opt/cmake -RUN sh /cmake-3.12.4-Linux-x86_64.sh --prefix=/opt/cmake --skip-license -RUN ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake -RUN cmake --version +WORKDIR /redisai +COPY --from=redis /usr/local/ /usr/local/ -# Set up a build environment -RUN set -ex;\ - deps="$DEPS";\ - apt-get update;\ - apt-get install -y --no-install-recommends $deps +COPY ./automation/ automation/ +COPY ./test/test_requirements.txt test/ + +RUN ./automation/readies/bin/getpy +RUN ./automation/system-setup.py + +COPY ./get_deps.sh . +RUN ./get_deps.sh cpu -# Get the dependencies -WORKDIR /redisai ADD ./ /redisai -RUN set -ex;\ - mkdir -p deps;\ - DEPS_DIRECTORY=deps bash ./get_deps.sh cpu - -# Build the source -RUN set -ex;\ - rm -rf build;\ - mkdir -p build;\ - cd build;\ - cmake -DDEPS_PATH=../deps/install ..;\ - make && make install;\ - cd .. - -# Package the runner +RUN make -C automation all + +ARG PACK=0 +ARG TEST=0 + +RUN if [ "$PACK" = "1" ]; then make -C automation pack; fi +RUN if [ "$TEST" = "1" ]; then make -C automation test; fi + +#---------------------------------------------------------------------------------------------- FROM redis RUN set -e; apt-get -qq update; apt-get install -y libgomp1 -RUN set -ex;\ - mkdir -p /usr/lib/redis/modules/; +RUN mkdir -p /usr/lib/redis/modules/ -COPY --from=builder /redisai/install/ /usr/lib/redis/modules/ +COPY --from=builder /redisai/install-cpu/ /usr/lib/redis/modules/ WORKDIR /data EXPOSE 6379 diff --git a/Makefile b/Makefile deleted file mode 100755 index 642bc4cb5..000000000 --- a/Makefile +++ /dev/null @@ -1,67 +0,0 @@ - -ifeq ($(CUDA),1) -DEPS_FLAGS= -else -DEPS_FLAGS=cpu -endif - -export REDIS_ENT_LIB_PATH=/opt/redislabs/lib - -GIT_BRANCH:=$(shell git rev-parse --abbrev-ref HEAD) -GIT_COMMIT:=$(shell git describe --always --abbrev=7 --dirty="+") - -ifeq ($(VERSION),) -PACK_VER:=$(GIT_BRANCH)-$(GIT_COMMIT) -else -PACK_VER:=$(VERSION) -endif - -BINDIR=$(PWD)/install - -.PHONY: all clean deps pack rlec_runpath_fix - -all: -ifeq ($(wildcard build/.),) - mkdir -p build - cd build; \ - cmake -DDEPS_PATH=../deps/install .. -endif - $(MAKE) -C build && $(MAKE) -C build install - -clean: -ifeq ($(ALL),1) - rm -rf build deps -else - $(MAKE) -C build clean -endif - -deps: - @echo Fetching dependencies... - @./get_deps.sh $(DEPS_FLAGS) - -# in pack: create ramp/redisai.so with RUNPATH set to /opt/redislabs/lib for RLEC compliance -rlec_runpath_fix: - @echo Fixing RLEC RUNPATH... - @mkdir -p $(BINDIR)/ramp - @cp -f $(BINDIR)/redisai.so $(BINDIR)/ramp/ - # @patchelf --set-rpath $(REDIS_ENT_LIB_PATH) $(BINDIR)/ramp/redisai.so - -pack: rlec_runpath_fix - @[ ! -z `command -v redis-server` ] || { echo "Cannot find redis-server - aborting."; exit 1; } - @[ ! -e $(REDIS_ENT_LIB_PATH) ] || { echo "$(REDIS_ENT_LIB_PATH) exists - aborting."; exit 1; } -ifeq ($(wildcard build/pyenv/.),) - @virtualenv build/pyenv ;\ - . ./build/pyenv/bin/activate ;\ - pip install git+https://github.com/RedisLabs/RAMP -endif - @echo "Building RAMP file ..." - @set -e ;\ - . ./build/pyenv/bin/activate ;\ - ln -fs $(PWD)/deps/install/lib/ $(REDIS_ENT_LIB_PATH) ;\ - ramp pack -m $(PWD)/ramp.yml -o "build/redisai.{os}-{architecture}.${PACK_VER}.zip" $(BINDIR)/ramp/redisai.so 2>&1 > /dev/null ;\ - rm $(REDIS_ENT_LIB_PATH) - @echo Done. - @echo "Building dependencies file redisai-dependencies.${PACK_VER}.tgz ..." - @cd deps/install/lib; \ - tar pczf ../../../build/redisai-dependencies.${PACK_VER}.tgz *.so* - @echo Done. diff --git a/automation/Makefile b/automation/Makefile new file mode 100755 index 000000000..f4398ce9c --- /dev/null +++ b/automation/Makefile @@ -0,0 +1,103 @@ + +ROOT:=.. + +override GPU:=$(or $(findstring $(CUDA),1),$(findstring $(GPU),1)) + +ifeq ($(GPU),1) +ifeq ($(CPU),1) +$(error CPU=1 and GPU=1 (or CUDA=1) are conflicting) +endif +DEPS_FLAGS=gpu +DEVICE=gpu +else +DEPS_FLAGS=cpu +DEVICE=cpu +endif + +GIT_BRANCH:=$(shell git rev-parse --abbrev-ref HEAD) +GIT_COMMIT:=$(shell git describe --always --abbrev=7 --dirty="+") + +ifeq ($(VERSION),) +PACK_VER:=$(GIT_BRANCH)-$(GIT_COMMIT) +else +PACK_VER:=$(VERSION) +endif + +BINDIR=$(PWD)/build +INSTALL_DIR=$(PWD)/install-$(DEVICE) + +BACKENDS_PATH ?= $(INSTALL_DIR)/backends + +RAMP:=ramp + +ifeq ($(DEBUG),1) +CMAKE_FLAGS += -DCMAKE_BUILD_TYPE=Debug +endif + +#---------------------------------------------------------------------------------------------- + +.PHONY: all setup build clean deps pack pack_ramp pack_deps test + +all: build + +build: +ifeq ($(wildcard build/Makefile),) + mkdir -p $(ROOT)/build + cd $(ROOT)/build; \ + cmake -DDEVICE=$(DEVICE) -DDEPS_PATH=$(ROOT)/deps/install $(CMAKE_FLAGS) $(ROOT) +endif + $(MAKE) -C $(ROOT)/build + $(MAKE) -C $(ROOT)/build install + cd $(ROOT) ;\ + [ ! -e install ] && ln -sf install-$(DEVICE) install + +clean: +ifeq ($(ALL),1) + cd $(ROOT) ;\ + rm -rf build install deps/dlpack deps/install-$(DEVICE) deps/*.tar.gz deps/*.zip deps/*.tgz +else + $(MAKE) -C $(ROOT)/build clean +endif + +deps fetch: + @echo Fetching dependencies... + $(ROOT)/get_deps.sh $(DEPS_FLAGS) + +pack: pack_ramp pack_deps + +pack_ramp: + @[ ! -z `command -v redis-server` ] || { echo "Cannot find redis-server - aborting."; exit 1; } + @echo "Building RAMP file ..." + @set -e ;\ + cd $(ROOT) ;\ + RAMPOUT=$$(mktemp /tmp/ramp.XXXXXX) ;\ + $(RAMP) pack -m $(PWD)/ramp.yml -o "build/redisai.{os}-{architecture}.{semantic_version}.zip" $(INSTALL_DIR)/redisai.so 2> /dev/null | grep '.zip' > $$RAMPOUT ;\ + tail -1 $$RAMPOUT > $(BINDIR)/PACKAGE ;\ + cat $(BINDIR)/PACKAGE | sed -e "s/[^.]*\.[^.]*\.\(.*\)\.zip/\1/" > $(BINDIR)/VERSION ;\ + VERSION=$$(cat $(BINDIR)/VERSION) ;\ + $(RAMP) pack -m $(PWD)/ramp.yml -o "build/redisai.{os}-{architecture}.{semantic_version}.zip" \ + -c "BACKENDSPATH /opt/redislabs/lib/redisai-cpu-$$VERSION/backends" $(INSTALL_DIR)/redisai.so 2> /dev/null | grep '.zip' > $$RAMPOUT ;\ + rm -f $$RAMPOUT ;\ + echo "Done." + +pack_deps: pack_ramp + @echo "Building dependencies file ..." + @set -e ;\ + cd $(ROOT) ;\ + PACK_FNAME=$$(basename `cat $(BINDIR)/PACKAGE`) ;\ + ARCHOSVER=$$(echo "$$PACK_FNAME" | sed -e "s/^redisai\.\([^.]*\..*\)\.zip/\1/") ;\ + VERSION=$$(cat $(BINDIR)/VERSION) ;\ + cd install-$(DEVICE) ;\ + rm -rf redisai-$(DEVICE)-$$VERSION ;\ + mkdir redisai-$(DEVICE)-$$VERSION ;\ + mv backends redisai-$(DEVICE)-$$VERSION ;\ + ln -sf redisai-$(DEVICE)-$$VERSION/backends backends ;\ + find redisai-$(DEVICE)-$$VERSION -name "*.so*" | xargs tar pczf redisai-$(DEVICE)-dependencies.$$ARCHOSVER.tgz ;\ + echo "Done." + +test: + @git lfs install + @git lfs pull + @set -e ;\ + cd $(ROOT)/test ;\ + python3 -m RLTest $(TEST_ARGS) --test basic_tests.py --module $(INSTALL_DIR)/redisai.so diff --git a/automation/readies/.gitignore b/automation/readies/.gitignore new file mode 100755 index 000000000..5ff250e4c --- /dev/null +++ b/automation/readies/.gitignore @@ -0,0 +1,36 @@ +# Object files +*.o +*.ko +*.obj +*.elf +*.lib +*.a +*.la +*.lo +*.dll +*.so +*.so.* +*.dylib +*.exe +*.out +*.app +*.i*86 +*.x86_64 +*.hex + +# C/C++ +*.gch +*.pch +*.d + +# Python +*.pyc + +# CMake +CMakeFiles/ +*.cmake +CMakeCache.txt + +# Eclipse +.cproject +.project diff --git a/automation/readies/LICENSE b/automation/readies/LICENSE new file mode 100755 index 000000000..bf39fd1a1 --- /dev/null +++ b/automation/readies/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2019, Redis Labs Modules +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/automation/readies/README.md b/automation/readies/README.md new file mode 100755 index 000000000..a490b311a --- /dev/null +++ b/automation/readies/README.md @@ -0,0 +1,9 @@ +# Library cluster of common Redis Modules code + +Cetara: C/C++ + +Paella: Python + +Shibumi: Bash + +mk: GNU Make diff --git a/automation/readies/bin/getpy b/automation/readies/bin/getpy new file mode 100755 index 000000000..edf875647 --- /dev/null +++ b/automation/readies/bin/getpy @@ -0,0 +1,37 @@ +#!/bin/bash + +[ "$VERBOSE" = "1" ] && set -x + +runn() { + [[ $NOP == 1 ]] && { echo "${@:1}"; return; } + __runn_log=$(mktemp /tmp/run.XXXXX) + { "${@:1}"; } > $__runn_log 2>&1 + [ $? != 0 ] && cat $__runn_log + rm -f $__runn_log +} + +if [ ! -z $(command -v python) ]; then + [ "$(python --version 2>&1 | cut -d" " -f2 | cut -d. -f1)" = "3" ] && exit 0 +fi + +[ ! -z $(command -v python3) ] && exit 0 + +if [ ! -z $(command -v apt-get) ]; then + runn apt-get -qq update + runn apt-get -qq install -y python3 +elif [ ! -z $(command -v dnf) ]; then + runn dnf install -y python3 +elif [ ! -z $(command -v yum) ]; then + runn yum install -y epel-release + runn yum install -y python36 +elif [ ! -z $(command -v apk) ]; then + runn apk update + runn apk add python3 +elif [ ! -z $(command -v brew) ]; then + runn brew install python3 +fi + +if [ -z $(command -v python3) ]; then + >&2 echo "Cannot install Python3. Aborting." + exit 1 +fi diff --git a/automation/readies/bin/getpy2 b/automation/readies/bin/getpy2 new file mode 100755 index 000000000..f957042ca --- /dev/null +++ b/automation/readies/bin/getpy2 @@ -0,0 +1,34 @@ +#!/bin/bash + +[ "$VERBOSE" = "1" ] && set -x + +show_if_error() { + { "${@:1}"; } > /tmp/py2.log 2>&1 + [ $? != 0 ] && cat /tmp/py2.log + rm -f /tmp/py2.log +} + +if [ ! -z $(command -v python) ]; then + [ "$(python --version 2>&1 | cut -d" " -f2 | cut -d. -f1)" = "2" ] && exit 0 +fi + +[ ! -z $(command -v python2) ] && exit 0 + +if [ ! -z $(command -v apt-get) ]; then + show_if_error apt-get -qq update + show_if_error apt-get -qq install -y python +elif [ ! -z $(command -v dnf) ]; then + show_if_error dnf install -y python2 +elif [ ! -z $(command -v yum) ]; then + show_if_error yum install -y python2 +elif [ ! -z $(command -v apk) ]; then + show_if_error apk update + show_if_error apk add python2 +elif [ ! -z $(command -v brew) ]; then + show_if_error brew install python2 +fi + +if [ -z $(command -v python) ]; then + >&2 echo "Cannot install Python2. Aborting." + exit 1 +fi diff --git a/automation/readies/bin/getredis5 b/automation/readies/bin/getredis5 new file mode 100755 index 000000000..caabb8ddf --- /dev/null +++ b/automation/readies/bin/getredis5 @@ -0,0 +1,57 @@ +#!/usr/bin/env python2 + +import sys +import os +import argparse + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +import paella + +os.environ["PYTHONWARNINGS"] = 'ignore:DEPRECATION::pip._internal.cli.base_command' + +#---------------------------------------------------------------------------------------------- + +class Redis5Setup(paella.Setup): + def __init__(self, nop=False): + paella.Setup.__init__(self, nop) + + def common_first(self): + pass + + def debian_compat(self): + # https://chilts.org/installing-redis-from-chris-leas-ppa/ + self.run("add-apt-repository -y ppa:chris-lea/redis-server") + self.install("redis-server") + # if not removed, might break apt-get update + self.run("add-apt-repository -r -y ppa:chris-lea/redis-server") + + def redhat_compat(self): + # https://linuxize.com/post/how-to-install-and-configure-redis-on-centos-7/ + self.install("epel-release yum-utils") + + self.install("http://rpms.remirepo.net/enterprise/remi-release-7.rpm") + self.run("yum-config-manager -y --enable remi") + self.install("redis") + + def fedora(self): + self.install("dnf-plugins-core") + + self.install("https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm") + self.install("--allowerasing https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm") + self.install("http://rpms.remirepo.net/enterprise/remi-release-7.rpm") + self.run("dnf config-manager -y --set-enabled remi") + self.install("redis") + + def macosx(self): + self.install("redis") + + def common_last(self): + pass + +#---------------------------------------------------------------------------------------------- + +parser = argparse.ArgumentParser(description='Set up system for build.') +parser.add_argument('-n', '--nop', action="store_true", help='no operation') +args = parser.parse_args() + +RedisTimeSeriesSetup(nop = args.nop).setup() diff --git a/automation/readies/bin/platform b/automation/readies/bin/platform new file mode 100755 index 000000000..5a355337b --- /dev/null +++ b/automation/readies/bin/platform @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +import sys +import os +import argparse + +READIES_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), "..")) +sys.path.insert(0, READIES_PATH) +from paella import Platform + +parser = argparse.ArgumentParser(description='Report platform characteristics.') +parser.add_argument('--os', action="store_true", help='Operating system') +parser.add_argument('--version', action="store_true", help='OS/Distribution version') +parser.add_argument('--dist', action="store_true", help='Linux distribution (if applicable)') +parser.add_argument('--arch', action="store_true", help='CPU Architecture') +parser.add_argument('--kernel', action="store_true", help='Kernel version (if applicable)') +parser.add_argument('--glibc', action="store_true", help='GLIBC version (if applicable)') +args = parser.parse_args() + +platform = Platform() +ret = "" +if args.os: + ret += " " + platform.os +if args.dist: + ret += " " + platform.dist +if args.version: + ret += " " + platform.os_ver +if args.arch: + ret += " " + platform.arch +if args.kernel: + pass +if args.glibc: + pass +if ret == "": + os = platform.os + dist = platform.dist + if dist != "": + os = dist + " " + os + ret = os + " " + platform.os_ver + " " + platform.arch +print(ret.strip()) diff --git a/automation/readies/bin/python b/automation/readies/bin/python new file mode 100755 index 000000000..814ff07ce --- /dev/null +++ b/automation/readies/bin/python @@ -0,0 +1,7 @@ +#!/bin/sh + +if [ ! -z $(command -v python3) ]; then + python3 "$@" +else + python "$@" +fi diff --git a/automation/readies/bin/python2 b/automation/readies/bin/python2 new file mode 100755 index 000000000..f46ed5a8e --- /dev/null +++ b/automation/readies/bin/python2 @@ -0,0 +1,7 @@ +#!/bin/sh + +if [ ! -z $(command -v python2) ]; then + python2 "$@" +else + python "$@" +fi diff --git a/automation/readies/cetara/diag/gdb.c b/automation/readies/cetara/diag/gdb.c new file mode 100755 index 000000000..8afa1697d --- /dev/null +++ b/automation/readies/cetara/diag/gdb.c @@ -0,0 +1,130 @@ + +#include +#include + +#include "readies/cetara/diag/gdb.h" + +#include +#include +#include +#include +#include +#include + +/////////////////////////////////////////////////////////////////////////////////////////////// + +#ifdef __linux__ + +#if 1 + +static inline bool _via_gdb() +{ + int pid; + int from_child[2] = {-1, -1}; + + if (pipe(from_child) < 0) { + fprintf(stderr, "Debugger check failed: Error opening internal pipe: %s", strerror(errno)); + return false; + } + + pid = fork(); + if (pid == -1) { + fprintf(stderr, "Debugger check failed: Error forking: %s", strerror(errno)); + return false; + } + + if (pid == 0) // child + { + uint8_t ret = 0; + int ppid = getppid(); + + close(from_child[0]); // close parent's side + + if (ptrace(PTRACE_ATTACH, ppid, NULL, NULL) == 0) + { + waitpid(ppid, NULL, 0); // wait for the parent to stop + write(from_child[1], &ret, sizeof(ret)); // tell the parent what happened + + ptrace(PTRACE_DETACH, ppid, NULL, NULL); + exit(0); + } + + ret = 1; + write(from_child[1], &ret, sizeof(ret)); // tell the parent what happened + + exit(0); + + } + else // parent + { + uint8_t ret = -1; + + // child writes a 1 if pattach failed else 0. + // read may be interrupted by pattach, hence the loop. + while (read(from_child[0], &ret, sizeof(ret)) < 0 && errno == EINTR) + ; + + // ret not updated + if (ret < 0) + fprintf(stderr, "Debugger check failed: Error getting status from child: %s", strerror(errno)); + + // close the pipes here, to avoid races with pattach (if we did it above) + close(from_child[1]); + close(from_child[0]); + + waitpid(pid, NULL, 0); // collect the status of the child + + return ret == 1; + } +} + +#else + +static inline bool _via_gdb() +{ + const int status_fd = open("/proc/self/status", O_RDONLY); + if (status_fd == -1) + return false; + + char buf[4096]; + const ssize_t num_read = read(status_fd, buf, sizeof(buf) - 1); + if (num_read <= 0) + return false; + + buf[num_read] = '\0'; + constexpr char tracer_pid[] = "TracerPid:"; + const auto tracer_pid_p = strstr(buf, tracer_pid); + if (!tracer_pid_p) + return false; + + for (const char *p = tracer_pid_p + sizeof(tracer_pid) - 1; p <= buf + num_read; ++p) + { + if (isspace(*p)) + continue; + return isdigit(*p) && *p != '0'; + } + + return false; +} + +#endif // 1 + +#elif defined(__APPLE__) + +static inline bool _via_gdb() +{ +} + +#endif + +//--------------------------------------------------------------------------------------------- + +bool __via_gdb = false; + +__attribute__((constructor)) +static void initialize(void) +{ + __via_gdb = _via_gdb(); +} + +/////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/automation/readies/cetara/diag/gdb.h b/automation/readies/cetara/diag/gdb.h new file mode 100755 index 000000000..458a7e15b --- /dev/null +++ b/automation/readies/cetara/diag/gdb.h @@ -0,0 +1,12 @@ + +#pragma once + +#include + +extern bool __via_gdb; + +#ifdef __arm__ +#define BB do { if (__via_gdb) { __asm__("trap"); } } while(0) +#else +#define BB do { if (__via_gdb) { __asm__("int $3"); } } while(0) +#endif \ No newline at end of file diff --git a/automation/readies/mk/bindirs.defs b/automation/readies/mk/bindirs.defs new file mode 100755 index 000000000..1ff57c788 --- /dev/null +++ b/automation/readies/mk/bindirs.defs @@ -0,0 +1,8 @@ + +BINROOT=$(ROOT)/bin/$(FULL_VARIANT) +BIN_DIRS=$(sort $(patsubst %/,%,$(BINDIR) $(dir $(OBJECTS)))) + +define mkdir_rule +$(1): + $$(SHOW)mkdir -p $(1) +endef diff --git a/automation/readies/mk/bindirs.rules b/automation/readies/mk/bindirs.rules new file mode 100755 index 000000000..4b57e8d49 --- /dev/null +++ b/automation/readies/mk/bindirs.rules @@ -0,0 +1,10 @@ + +ifeq ($(BINDIR),) +$(error BINDIR is undefined) +endif + +.PHONY: bindirs + +bindirs: $(BIN_DIRS) + +$(foreach DIR,$(BIN_DIRS),$(eval $(call mkdir_rule,$(DIR)))) diff --git a/automation/readies/mk/common.defs b/automation/readies/mk/common.defs new file mode 100755 index 000000000..206cd969d --- /dev/null +++ b/automation/readies/mk/common.defs @@ -0,0 +1,14 @@ + +_SHOW:=$(SHOW) +ifeq ($(SHOW),1) +override SHOW:= +else +override SHOW:=@ +endif + +MAKEFLAGS += --no-builtin-rules --no-print-directory +# --no-builtin-variables + +define __SEP +import os; rows, cols = os.popen('stty size', 'r').read().split(); print(\"\n\" + '-' * (int(cols) - 1) + \"\n\") +endef diff --git a/automation/readies/mk/common.rules b/automation/readies/mk/common.rules new file mode 100755 index 000000000..633cf2f6b --- /dev/null +++ b/automation/readies/mk/common.rules @@ -0,0 +1,6 @@ + +.PHONY: __sep + +__sep: ; +# @python -c "$(__SEP)" + diff --git a/automation/readies/mk/configure.defs b/automation/readies/mk/configure.defs new file mode 100755 index 000000000..da576887b --- /dev/null +++ b/automation/readies/mk/configure.defs @@ -0,0 +1,2 @@ + +CONFIGURE_FLAGS= diff --git a/automation/readies/mk/configure.rules b/automation/readies/mk/configure.rules new file mode 100755 index 000000000..baf4ce4d4 --- /dev/null +++ b/automation/readies/mk/configure.rules @@ -0,0 +1,13 @@ + +build: __sep bindirs +ifeq (,$(wildcard $(BUILD_DIR)/Makefile)) + $(SHOW)cd $(BUILD_DIR); $(realpath $(SRCDIR))/configure $(CONFIGURE_FLAGS) +endif + @make -C $(BUILD_DIR) + +clean: +ifeq ($(ALL),1) + $(SHOW)rm -rf $(BINDIR) +else + $(SHOW)$(MAKE) clean -C $(BUILD_DIR) +endif diff --git a/automation/readies/mk/defs b/automation/readies/mk/defs new file mode 100755 index 000000000..819d0d30c --- /dev/null +++ b/automation/readies/mk/defs @@ -0,0 +1,6 @@ + +MK=$(ROOT)/automation/readies/mk + +include $(MK)/common.defs +include $(MK)/variant.defs +include $(MK)/bindirs.defs diff --git a/automation/readies/mk/help.defs b/automation/readies/mk/help.defs new file mode 100755 index 000000000..0641d1684 --- /dev/null +++ b/automation/readies/mk/help.defs @@ -0,0 +1,4 @@ + +ifneq ($(HELP),) +HELPFILE:=$(shell mktemp /tmp/make.help.XXXX) +endif diff --git a/automation/readies/mk/help.rules b/automation/readies/mk/help.rules new file mode 100755 index 000000000..eae2fed21 --- /dev/null +++ b/automation/readies/mk/help.rules @@ -0,0 +1,11 @@ + +ifneq ($(HELP),) + +.PHONY: help + +help: + $(file >$(HELPFILE),$(HELP)) + @cat $(HELPFILE) + @-rm -f $(HELPFILE) + +endif diff --git a/automation/readies/mk/rules b/automation/readies/mk/rules new file mode 100755 index 000000000..1aa2985e3 --- /dev/null +++ b/automation/readies/mk/rules @@ -0,0 +1,4 @@ + +include $(MK)/common.rules +include $(MK)/variant.rules +include $(MK)/bindirs.rules diff --git a/automation/readies/mk/variant.defs b/automation/readies/mk/variant.defs new file mode 100755 index 000000000..aaccb3849 --- /dev/null +++ b/automation/readies/mk/variant.defs @@ -0,0 +1,42 @@ + +OS:=$(shell $(ROOT)/automation/readies/bin/platform --os) +# ifeq ($(OS),linux) +# OS:=$(shell $(ROOT)/automation/readies/bin/platform --dist) +# endif + +ARCH=$(shell $(ROOT)/automation/readies/bin/platform --arch) + +#---------------------------------------------------------------------------------------------- + +GIT_SHA := $(shell git rev-parse HEAD) +GIT_COMMIT := $(shell git describe --always --abbrev=7 --dirty="+") + +#---------------------------------------------------------------------------------------------- + +ifeq ($(DEBUG),1) +FLAVOR=debug +else +FLAVOR=release +endif + +#---------------------------------------------------------------------------------------------- + +__VARIANT__=$(shell if [ -f $(ROOT)/VARIANT ]; then cat $(ROOT)/VARIANT; fi) + +ifeq ($(origin VARIANT),undefined) +ifneq ($(__VARIANT__),) +VARIANT:=$(__VARIANT__) +endif +endif + +ifeq ($(VARIANT),) +__VARIANT:= +else +__VARIANT:=-$(VARIANT) +endif +FULL_VARIANT:=$(OS)-$(ARCH)-$(FLAVOR)$(__VARIANT) +FULL_VARIANT_REL:=$(OS)-$(ARCH)-release$(__VARIANT) + +ifneq ($(origin VARIANT),) +$(eval $(shell if [ -z $(VARIANT) ]; then rm -f $(ROOT)/VARIANT; else echo $(VARIANT)>$(ROOT)/VARIANT; fi)) +endif diff --git a/automation/readies/mk/variant.rules b/automation/readies/mk/variant.rules new file mode 100755 index 000000000..c7580000e --- /dev/null +++ b/automation/readies/mk/variant.rules @@ -0,0 +1,3 @@ + +show-variant: + @cat $(ROOT)/VARIANT \ No newline at end of file diff --git a/automation/readies/paella/__init__.py b/automation/readies/paella/__init__.py new file mode 100755 index 000000000..286fecdd3 --- /dev/null +++ b/automation/readies/paella/__init__.py @@ -0,0 +1,32 @@ + +from .debug import * +from .utils import * +from .files import * +# from .docopt import docopt +from .log import * +from .platform import * +from .setup import * + +#---------------------------------------------------------------------------------------------- + +import sys + +class global_injector: + def __init__(self): + try: + # Python 2 + self.__dict__['builtin'] = sys.modules['__builtin__'].__dict__ + except KeyError: + # Python 3 + self.__dict__['builtin'] = sys.modules['builtins'].__dict__ + def __setattr__(self,name,value): + self.builtin[name] = value + +Global = global_injector() + +#---------------------------------------------------------------------------------------------- + +Global.bb = bb +Global.eprint = eprint +Global.fatal = fatal +Global.cwd = cwd diff --git a/automation/readies/paella/debug.py b/automation/readies/paella/debug.py new file mode 100755 index 000000000..1b60ab7a9 --- /dev/null +++ b/automation/readies/paella/debug.py @@ -0,0 +1,11 @@ + +import os + +#---------------------------------------------------------------------------------------------- + +if 'PYDEBUG' in os.environ: + from pdb import set_trace as bb +else: + def bb(): pass + +#---------------------------------------------------------------------------------------------- diff --git a/automation/readies/paella/docopt1.py b/automation/readies/paella/docopt1.py new file mode 100755 index 000000000..dc08ee3c5 --- /dev/null +++ b/automation/readies/paella/docopt1.py @@ -0,0 +1,25 @@ + +from docopt import docopt as docopt1 +from collections import namedtuple +import re + +def dict_to_obj(d): + def add_to_dict_if_match(r, k, v, d): + m = re.match(r, k) + if m: + d[m[1].replace('-', '_')] = v + return not not m + + d1 = dict() + for k, v in d.items(): + if isinstance(v, dict): + d1[k] = dict_to_obj(v) + elif add_to_dict_if_match('--(.*)', k, v, d1): + pass + elif add_to_dict_if_match('\<(.*)\>', k, v, d1): + pass + return namedtuple('object', d1.keys())(*d1.values()) + +def docopt(*args, **kwargs): + a = docopt1(*args, **kwargs) + return dict_to_obj(a) diff --git a/automation/readies/paella/files.py b/automation/readies/paella/files.py new file mode 100755 index 000000000..b38199803 --- /dev/null +++ b/automation/readies/paella/files.py @@ -0,0 +1,23 @@ + +from contextlib import contextmanager +import os + +def fread(fname, mode = 'rb'): + with open(fname, mode) as file: + return file.read() + +def flines(fname, mode = 'rb'): + return [line.rstrip() for line in open(fname)] + +@contextmanager +def cwd(path): + d0 = os.getcwd() + os.chdir(str(path)) + try: + yield + finally: + os.chdir(d0) + +def mkdir_p(dir): + if dir != '': + os.makedirs(dir, exist_ok=True) diff --git a/automation/readies/paella/log.py b/automation/readies/paella/log.py new file mode 100755 index 000000000..3edaec3ee --- /dev/null +++ b/automation/readies/paella/log.py @@ -0,0 +1,6 @@ + +import os + +def fatal(text): + eprint("%s: %s" %(os.path.basename(__file__), text)) + exit(1) diff --git a/automation/readies/paella/platform.py b/automation/readies/paella/platform.py new file mode 100755 index 000000000..e748cf728 --- /dev/null +++ b/automation/readies/paella/platform.py @@ -0,0 +1,165 @@ + +from __future__ import absolute_import +import platform + +#---------------------------------------------------------------------------------------------- + +class Platform: + def __init__(self): + self.os = self.dist = self.os_ver = self.full_os_ver = self.os_nick = self.arch = '?' + + self.os = platform.system().lower() + dist = platform.linux_distribution() + distname = dist[0].lower() + self.os_ver = self.full_os_ver = dist[1] + if self.os == 'linux': + if distname == 'fedora' or distname == 'ubuntu' or distname == 'debian' or distname == 'arch': + pass + elif distname == 'centos linux': + distname = 'centos' + elif distname.startswith('redhat'): + distname = 'redhat' + elif distname.startswith('suse'): + distname = 'suse' + else: + Assert(False), "Cannot determine distribution" + self.dist = distname + elif self.os == 'darwin': + self.os = 'macosx' + self.dist = '' + mac_ver = platform.mac_ver() + self.full_os_ver = mac_ver[0] # e.g. 10.14, but also 10.5.8 + self.os_ver = '.'.join(self.full_os_ver.split('.')[:2]) # major.minor + # self.arch = mac_ver[2] # e.g. x64_64 + elif self.os == 'windows': + self.dist = self.os + self.os_ver = platform.release() + self.full_os_ver = os.version() + elif self.os == 'sunos': + self.os = 'solaris' + self.os_ver = '' + self.dist = '' + else: + Assert(False), "Cannot determine OS" + + self.arch = platform.machine().lower() + if self.arch == 'amd64' or self.arch == 'x86_64': + self.arch = 'x64' + elif self.arch == 'i386' or self.arch == 'i686' or self.arch == 'i86pc': + self.arch = 'x86' + elif self.arch == 'aarch64': + self.arch = 'arm64v8' + elif self.arch == 'armv7l': + self.arch = 'arm32v7' + + def is_debian_compat(self): + return self.dist == 'debian' or self.dist == 'ubuntu' + + def is_redhat_compat(self): + return self.dist == 'redhat' or self.dist == 'centos' + + def is_container(self): + with open('/proc/1/cgroups', 'r') as conf: + for line in conf: + if re.search('docker', line): + return True + return False + + def report(self): + print("This system is " + self.distname + " " + self.distver + ".\n") + +#---------------------------------------------------------------------------------------------- + +class OnPlatform: + def __init__(self): + self.stages = [0] + self.platform = Platform() + + def invoke(self): + os = self.os = self.platform.os + dist = self.dist = self.platform.dist + self.ver = self.platform.os_ver + self.common_first() + + for stage in self.stages: + self.stage = stage + self.common() + if os == 'linux': + self.linux() + + if self.platform.is_debian_compat(): + self.debian_compat() + if self.platform.is_redhat_compat(): + self.redhat_compat() + + if dist == 'fedora': + self.fedora() + elif dist == 'ubuntu': + self.ubuntu() + elif dist == 'debian': + self.debian() + elif dist == 'centos': + self.centos() + elif dist == 'redhat': + self.redhat() + elif dist == 'suse': + self.suse() + elif dist == 'arch': + self.arch() + else: + Assert(False), "Cannot determine installer" + elif os == 'macosx': + self.macosx() + + self.common_last() + + def common(self): + pass + + def common_first(self): + pass + + def common_last(self): + pass + + def linux(self): + pass + + def arch(self): + pass + + def debian_compat(self): # debian, ubuntu, etc + pass + + def debian(self): + pass + + def centos(self): + pass + + def fedora(self): + pass + + def redhat_compat(self): # centos, rhel + pass + + def redhat(self): + pass + + def ubuntu(self): + pass + + def suse(self): + pass + + def macosx(self): + pass + + def windows(self): + pass + + def bsd_compat(self): + pass + + def freebsd(self): + pass diff --git a/automation/readies/paella/setup.py b/automation/readies/paella/setup.py new file mode 100755 index 000000000..643042c11 --- /dev/null +++ b/automation/readies/paella/setup.py @@ -0,0 +1,161 @@ + +import os +import sys +import tempfile +from .platform import OnPlatform, Platform + +#---------------------------------------------------------------------------------------------- + +class Runner: + def __init__(self, nop=False): + self.nop = nop + + def run(self, cmd, output_on_error=False): + print(cmd) + sys.stdout.flush() + if self.nop: + return + if output_on_error: + fd, temppath = tempfile.mkstemp() + os.close(fd) + cmd = "{{ {}; }} >{} 2>&1".format(cmd, temppath) + rc = os.system(cmd) + if rc > 0: + if output_on_error: + os.system("cat {}".format(temppath)) + os.remove(temppath) + eprint("command failed: " + cmd) + sys.stderr.flush() + sys.exit(1) + + def has_command(self, cmd): + return os.system("command -v " + cmd + " > /dev/null") == 0 + +#---------------------------------------------------------------------------------------------- + +class RepoRefresh(OnPlatform): + def __init__(self, runner): + OnPlatform.__init__(self) + self.runner = runner + + def redhat_compat(self): + pass + + def debian_compat(self): + self.runner.run("apt-get -qq update -y") + + def macosx(self): + self.runner.run("brew update || true") + +#---------------------------------------------------------------------------------------------- + +class Setup(OnPlatform): + def __init__(self, nop=False): + OnPlatform.__init__(self) + self.runner = Runner(nop) + self.stages = [0] + self.platform = Platform() + self.os = self.platform.os + self.dist = self.platform.dist + self.ver = self.platform.os_ver + + if self.has_command("python"): + self.python = "python" + elif self.has_command("python2"): + self.python = "python2" + elif self.has_command("python3"): + self.python = "python3" + + if self.os == 'macosx': + # this is required because osx pip installed are done with --user + os.environ["PATH"] = os.environ["PATH"] + ':' + '$HOME/Library/Python/2.7/bin' + + if self.platform.is_debian_compat(): + # prevents apt-get from interactively prompting + os.environ["DEBIAN_FRONTEND"] = 'noninteractive' + + os.environ["PYTHONWARNINGS"] = 'ignore:DEPRECATION::pip._internal.cli.base_command' + + def setup(self): + RepoRefresh(self.runner).invoke() + self.invoke() + + def run(self, cmd, output_on_error=False): + return self.runner.run(cmd, output_on_error=output_on_error) + + def has_command(self, cmd): + return self.runner.has_command(cmd) + + #------------------------------------------------------------------------------------------ + + def apt_install(self, packs, group=False): + self.run("apt-get -qq install -y " + packs, output_on_error=True) + + def yum_install(self, packs, group=False): + if not group: + self.run("yum install -q -y " + packs, output_on_error=True) + else: + self.run("yum groupinstall -y " + packs, output_on_error=True) + + def dnf_install(self, packs, group=False): + if not group: + self.run("dnf install -y " + packs, output_on_error=True) + else: + self.run("dnf groupinstall -y " + packs, output_on_error=True) + + def zypper_install(self, packs, group=False): + self.run("zipper --non-interactive install " + packs, output_on_error=True) + + def pacman_install(self, packs, group=False): + self.run("pacman --noconfirm -S " + packs, output_on_error=True) + + def brew_install(self, packs, group=False): + # brew will fail if package is already installed + for pack in packs.split(): + self.run("brew list {} &>/dev/null || brew install {}".format(pack, pack), output_on_error=True) + + def install(self, packs, group=False): + if self.os == 'linux': + if self.dist == 'fedora': + self.dnf_install(packs, group) + elif self.dist == 'ubuntu' or self.dist == 'debian': + self.apt_install(packs, group) + elif self.dist == 'centos' or self.dist == 'redhat': + self.yum_install(packs, group) + elif self.dist == 'suse': + self.zypper_install(packs, group) + elif self.dist == 'arch': + self.pacman_install(packs, group) + else: + Assert(False), "Cannot determine installer" + elif self.os == 'macosx': + self.brew_install(packs, group) + else: + Assert(False), "Cannot determine installer" + + def group_install(self, packs): + self.install(packs, group=True) + + #------------------------------------------------------------------------------------------ + + def pip_install(self, cmd): + pip_user = '' + if self.os == 'macosx': + pip_user = '--user ' + self.run("pip install --disable-pip-version-check " + pip_user + cmd, output_on_error=True) + + def pip3_install(self, cmd): + pip_user = '' + if self.os == 'macosx': + pip_user = '--user ' + self.run("pip3 install --disable-pip-version-check " + pip_user + cmd, output_on_error=True) + + def setup_pip(self): + get_pip = "set -e; cd /tmp; curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py" + if not self.has_command("pip"): + self.install("python3-distutils") + self.install("curl ca-certificates") + self.run(get_pip + "; " + self.python + " get-pip.py", output_on_error=True) + ## fails on ubuntu 18: + # if not has_command("pip3") and has_command("python3"): + # run(get_pip + "; python3 get-pip.py") diff --git a/automation/readies/paella/utils.py b/automation/readies/paella/utils.py new file mode 100755 index 000000000..a46d97576 --- /dev/null +++ b/automation/readies/paella/utils.py @@ -0,0 +1,7 @@ + +import sys + +if (sys.version_info > (3, 0)): + from .utils3 import * +else: + from .utils2 import * diff --git a/automation/readies/paella/utils2.py b/automation/readies/paella/utils2.py new file mode 100755 index 000000000..bad63520e --- /dev/null +++ b/automation/readies/paella/utils2.py @@ -0,0 +1,5 @@ + +import sys + +def eprint(*args, **kwargs): + print >> sys.stderr, ' '.join(map(lambda x: "%s" % x, args)) diff --git a/automation/readies/paella/utils3.py b/automation/readies/paella/utils3.py new file mode 100755 index 000000000..2f552cd34 --- /dev/null +++ b/automation/readies/paella/utils3.py @@ -0,0 +1,5 @@ + +import sys + +def eprint(*args, **kwargs): + print(*args, file = sys.stderr, **kwargs) diff --git a/automation/readies/shibumi/functions b/automation/readies/shibumi/functions new file mode 100755 index 000000000..6eb9450dd --- /dev/null +++ b/automation/readies/shibumi/functions @@ -0,0 +1,21 @@ + +platform_os() { + case "$OSTYPE" in + linux*) echo "linux" ;; + solaris*) echo "solaris" ;; + darwin*) echo "macosx" ;; + bsd*) echo "bsd" ;; + msys*) echo "windows" ;; + *) echo "?" ;; + esac +} + +#---------------------------------------------------------------------------------------------- + +if [[ $(platform_os) == mac ]]; then + +realpath() { + [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}" +} + +fi diff --git a/automation/readies/shibumi/here b/automation/readies/shibumi/here new file mode 100755 index 000000000..a6e56964e --- /dev/null +++ b/automation/readies/shibumi/here @@ -0,0 +1,3 @@ +#!/bin/bash + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" diff --git a/automation/system-setup.py b/automation/system-setup.py new file mode 100755 index 000000000..03fbb8705 --- /dev/null +++ b/automation/system-setup.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 + +import sys +import os +from subprocess import Popen, PIPE +import argparse + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "readies")) +import paella + +#---------------------------------------------------------------------------------------------- + +class RedisAISetup(paella.Setup): + def __init__(self, nop=False): + paella.Setup.__init__(self, nop) + + def common_first(self): + self.setup_pip() + self.pip3_install("wheel") + self.pip3_install("setuptools --upgrade") + + self.install("git cmake ca-certificates curl unzip wget patchelf awscli") + + def debian_compat(self): + self.install("build-essential") + self.install("python3-venv python3-psutil python3-networkx") + self.install_git_lfs_on_linux() + + def redhat_compat(self): + self.group_install("'Development Tools'") + self.install("redhat-lsb-core") + + self.install("epel-release") + self.install("python36 python36-pip") + self.install("python36-psutil") + + self.install_git_lfs_on_linux() + + def fedora(self): + self.group_install("'Development Tools'") + self.install_git_lfs_on_linux() + + def macosx(self): + p = Popen('xcode-select -p', stdout=PIPE, close_fds=True, shell=True) + out, _ = p.communicate() + if out.splitlines() == []: + fatal("Xcode tools are not installed. Please run xcode-select --install.") + self.install("git-lfs") + + def install_git_lfs_on_linux(self): + self.run("curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash") + self.install("git-lfs") + + def common_last(self): + if not self.has_command("ramp"): + self.pip3_install("git+https://github.com/RedisLabs/RAMP --upgrade") + if not self.has_command("RLTest"): + self.pip3_install("git+https://github.com/RedisLabsModules/RLTest.git@master") + self.pip3_install("-r test/test_requirements.txt") + +#---------------------------------------------------------------------------------------------- + +parser = argparse.ArgumentParser(description='Set up system for build.') +parser.add_argument('-n', '--nop', action="store_true", help='no operation') +args = parser.parse_args() + +RedisAISetup(nop = args.nop).setup() diff --git a/get_deps.sh b/get_deps.sh index 62cdd1ad2..2cf745010 100755 --- a/get_deps.sh +++ b/get_deps.sh @@ -1,142 +1,185 @@ #!/usr/bin/env bash -set -e -set -x - -BASE_DIRECTORY=`pwd` +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" -# Allow different deps for different platforms: -if [ -z "$DEPS_DIRECTORY" ]; then - DEPS_DIRECTORY=${BASE_DIRECTORY}/deps +set -e +[[ $VERBOSE == 1 ]] && set -x + +if [[ "$1" == "cpu" ]]; then + GPU=no + DEVICE=cpu +elif [[ "$1" == "gpu" ]]; then + GPU=yes + DEVICE=gpu +else + GPU=${GPU:-no} + if [[ $GPU == 1 ]]; then + DEVICE=gpu + else + DEVICE=cpu + fi fi -mkdir -p ${DEPS_DIRECTORY} -cd ${DEPS_DIRECTORY} -DEPS_DIRECTORY=$PWD # -- to avoid relative/absolute confusion +DEPS_DIR=$HERE/deps +mkdir -p ${DEPS_DIR} +cd ${DEPS_DIR} -PREFIX=${DEPS_DIRECTORY}/install +PREFIX=${DEPS_DIR}/install mkdir -p ${PREFIX} DLPACK_PREFIX=${PREFIX}/dlpack -rm -rf ${DLPACK_PREFIX} -mkdir -p ${DLPACK_PREFIX} - TF_PREFIX=${PREFIX}/libtensorflow -rm -rf ${TF_PREFIX} -mkdir -p ${TF_PREFIX} - TORCH_PREFIX=${PREFIX}/libtorch -rm -rf ${TORCH_PREFIX} -mkdir -p ${TORCH_PREFIX} - ORT_PREFIX=${PREFIX}/onnxruntime -rm -rf ${ORT_PREFIX} -mkdir -p ${ORT_PREFIX} -if [ ! -d dlpack ]; then - echo "Cloning dlpack" +## DLPACK + +[[ $FORCE == 1 ]] && rm -rf ${DLPACK_PREFIX} + +if [[ ! -d dlpack ]]; then + echo "Cloning dlpack ..." git clone --depth 1 https://github.com/dmlc/dlpack.git + echo "Done." +else + echo "dlpack is in place." fi -if [[ "$OSTYPE" == "linux-gnu" ]]; then - cp -d -r --no-preserve=ownership dlpack/include ${DLPACK_PREFIX} -elif [[ "$OSTYPE" == "darwin"* ]]; then - cp -r dlpack/include ${DLPACK_PREFIX} +if [[ ! -d ${DLPACK_PREFIX}/include ]]; then + mkdir -p ${DLPACK_PREFIX} + ln -sf ${DEPS_DIR}/dlpack/include ${DLPACK_PREFIX}/include fi ## TENSORFLOW -TF_VERSION="1.12.0" -if [[ "$OSTYPE" == "linux-gnu" ]]; then - TF_OS="linux" - if [[ "$1" == "cpu" ]]; then - TF_BUILD="cpu" - else - TF_BUILD="gpu" - fi -elif [[ "$OSTYPE" == "darwin"* ]]; then - TF_OS="darwin" - TF_BUILD="cpu" -fi -LIBTF_ARCHIVE=libtensorflow-${TF_BUILD}-${TF_OS}-x86_64-${TF_VERSION}.tar.gz +TF_VERSION="1.12.0" -if [ ! -e ${LIBTF_ARCHIVE} ]; then - echo "Downloading libtensorflow ${TF_VERSION} ${TF_BUILD}" - wget https://storage.googleapis.com/tensorflow/libtensorflow/${LIBTF_ARCHIVE} -fi +[[ $FORCE == 1 ]] && rm -rf ${TF_PREFIX} -tar xf ${LIBTF_ARCHIVE} --no-same-owner --strip-components=1 -C ${TF_PREFIX} +if [[ ! -d ${TF_PREFIX} ]]; then + echo "Installing TensorFlow ..." -## PYTORCH + mkdir -p ${TF_PREFIX} -PT_VERSION="1.1.0" -#PT_VERSION="latest" - -if [[ "$OSTYPE" == "linux-gnu" ]]; then - PT_OS="shared-with-deps" - if [[ "$1" == "cpu" ]]; then - PT_BUILD="cpu" - else - PT_BUILD="cu90" + if [[ "$OSTYPE" == "linux-gnu" ]]; then + TF_OS="linux" + if [[ "$1" == "cpu" ]]; then + TF_BUILD="cpu" + else + TF_BUILD="gpu" + fi + elif [[ "$OSTYPE" == "darwin"* ]]; then + TF_OS="darwin" + TF_BUILD="cpu" fi -elif [[ "$OSTYPE" == "darwin"* ]]; then - PT_OS="macos" - PT_BUILD="cpu" + + LIBTF_ARCHIVE=libtensorflow-${TF_BUILD}-${TF_OS}-x86_64-${TF_VERSION}.tar.gz + + if [ ! -e ${LIBTF_ARCHIVE} ]; then + echo "Downloading libtensorflow ${TF_VERSION} ${TF_BUILD}" + wget -q https://storage.googleapis.com/tensorflow/libtensorflow/${LIBTF_ARCHIVE} + fi + + tar xf ${LIBTF_ARCHIVE} --no-same-owner --strip-components=1 -C ${TF_PREFIX} + + echo "Done." +else + echo "TensorFlow is in place." fi -if [[ "$PT_VERSION" == "latest" ]]; then - PT_BUILD=nightly/${PT_BUILD} -fi +## PYTORCH -# Where to get the archive -LIBTORCH_URL=https://download.pytorch.org/libtorch/${PT_BUILD}/libtorch-${PT_OS}-${PT_VERSION}.zip -# Directory where torch is extracted to -LIBTORCH_DIRECTORY=libtorch-${PT_OS}-${PT_VERSION} +PT_VERSION="1.1.0" +#PT_VERSION="latest" -# Archive - specifically named -LIBTORCH_ARCHIVE=libtorch-${PT_OS}-${PT_BUILD}-${PT_VERSION}.zip +[[ $FORCE == 1 ]] && rm -rf ${TORCH_PREFIX} -if [ ! -e "${LIBTORCH_ARCHIVE}" ]; then - echo "Downloading libtorch ${PT_VERSION} ${PT_BUILD}" - curl -L ${LIBTORCH_URL} > ${LIBTORCH_ARCHIVE} -fi +if [[ ! -d ${TORCH_PREFIX} ]]; then + echo "Installing libtorch ..." -unzip -o ${LIBTORCH_ARCHIVE} -tar cf - libtorch | tar xf - --no-same-owner --strip-components=1 -C ${TORCH_PREFIX} -rm -rf libtorch + mkdir -p ${TORCH_PREFIX} -if [[ "${PT_OS}" == "macos" ]]; then - # also download mkl - MKL_BUNDLE=mklml_mac_2019.0.3.20190220 - if [ ! -e "${MKL_BUNDLE}.tgz" ]; then - wget "https://github.com/intel/mkl-dnn/releases/download/v0.18/${MKL_BUNDLE}.tgz" + if [[ "$OSTYPE" == "linux-gnu" ]]; then + PT_OS="shared-with-deps" + if [[ "$1" == "cpu" ]]; then + PT_BUILD="cpu" + else + PT_BUILD="cu90" + fi + elif [[ "$OSTYPE" == "darwin"* ]]; then + PT_OS="macos" + PT_BUILD="cpu" + fi + + if [[ "$PT_VERSION" == "latest" ]]; then + PT_BUILD=nightly/${PT_BUILD} + fi + + # Where to get the archive + LIBTORCH_URL=https://download.pytorch.org/libtorch/${PT_BUILD}/libtorch-${PT_OS}-${PT_VERSION}.zip + # Directory where torch is extracted to + LIBTORCH_DIRECTORY=libtorch-${PT_OS}-${PT_VERSION} + + # Archive - specifically named + LIBTORCH_ARCHIVE=libtorch-${PT_OS}-${PT_BUILD}-${PT_VERSION}.zip + + if [ ! -e "${LIBTORCH_ARCHIVE}" ]; then + echo "Downloading libtorch ${PT_VERSION} ${PT_BUILD}" + curl -s -L ${LIBTORCH_URL} > ${LIBTORCH_ARCHIVE} + fi + + unzip -q -o ${LIBTORCH_ARCHIVE} -d ${TORCH_PREFIX}/../ + + echo "Done." + + if [[ "${PT_OS}" == "macos" ]]; then + echo "Installing MKL ..." + # also download mkl + MKL_BUNDLE=mklml_mac_2019.0.3.20190220 + if [ ! -e "${MKL_BUNDLE}.tgz" ]; then + wget -q "https://github.com/intel/mkl-dnn/releases/download/v0.18/${MKL_BUNDLE}.tgz" + fi + tar xzf ${MKL_BUNDLE}.tgz --no-same-owner --strip-components=1 -C ${TORCH_PREFIX} + mkdir -p ${ORT_PREFIX} + tar xzf ${MKL_BUNDLE}.tgz --no-same-owner --strip-components=1 -C ${ORT_PREFIX} + echo "Done." fi - tar xf ${MKL_BUNDLE}.tgz --no-same-owner --strip-components=1 -C ${TORCH_PREFIX} - tar xf ${MKL_BUNDLE}.tgz --no-same-owner --strip-components=1 -C ${ORT_PREFIX} +else + echo "librotch is in place." fi ## ONNXRUNTIME ORT_VERSION="0.5.0" +ORT_ARCH="x64" if [[ "$OSTYPE" == "linux-gnu" ]]; then if [[ "$1" == "cpu" ]]; then - ORT_OS="linux-x64" - ORT_BUILD="cpu" + ORT_OS="linux" + ORT_BUILD="" else - ORT_OS="linux-x64-gpu" - ORT_BUILD="gpu" + ORT_OS="linux" + ORT_BUILD="-gpu" fi elif [[ "$OSTYPE" == "darwin"* ]]; then - ORT_OS="osx-x64" + ORT_OS="osx" ORT_BUILD="" fi -ORT_ARCHIVE=onnxruntime-${ORT_OS}-${ORT_VERSION}.tgz +[[ $FORCE == 1 ]] && rm -rf ${ORT_PREFIX} -if [ ! -e ${ORT_ARCHIVE} ]; then - echo "Downloading ONNXRuntime ${ORT_VERSION} ${ORT_BUILD}" - wget https://github.com/Microsoft/onnxruntime/releases/download/v${ORT_VERSION}/${ORT_ARCHIVE} -fi +ORT_ARCHIVE=onnxruntime-${ORT_OS}-${ORT_ARCH}${ORT_BUILD}-${ORT_VERSION}.tgz + +if [[ ! -d ${ORT_PREFIX} ]]; then + echo "Installing ONNXRuntime ..." + + mkdir -p ${ORT_PREFIX} -tar xf ${ORT_ARCHIVE} --no-same-owner --strip-components=1 -C ${ORT_PREFIX} + if [ ! -e ${ORT_ARCHIVE} ]; then + echo "Downloading ONNXRuntime ${ORT_VERSION} ${DEVICE}" + wget -q https://github.com/microsoft/onnxruntime/releases/download/v${ORT_VERSION}/${ORT_ARCHIVE} + fi + + tar xf ${ORT_ARCHIVE} --no-same-owner --strip-components=1 -C ${ORT_PREFIX} +else + echo "onnxruntime is in place." +fi -echo "Done" +echo "Done." diff --git a/ramp.yml b/ramp.yml index a31e3fb4f..895b34470 100644 --- a/ramp.yml +++ b/ramp.yml @@ -6,7 +6,7 @@ homepage: https://oss.redislabs.com/redisai/ license: GNU Affero General Public License v3.0 command_line_args: "" min_redis_version: "5.0" -min_redis_pack_version: "5.5" +min_redis_pack_version: "5.4" capabilities: - types - no_multi_key diff --git a/src/config.h b/src/config.h index 09551a5cd..d4a395813 100644 --- a/src/config.h +++ b/src/config.h @@ -13,7 +13,7 @@ typedef enum { RAI_DEVICE_GPU, } RAI_Device; -#define RAI_ENC_VER 300 +#define RAI_ENC_VER 302 //#define RAI_COPY_RUN_INPUT #define RAI_COPY_RUN_OUTPUT