diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml new file mode 100644 index 0000000000..921ecb65e9 --- /dev/null +++ b/.github/workflows/run_tests.yml @@ -0,0 +1,65 @@ +name: run-tests + +on: [push, pull_request] + +env: + HOMEBREW_NO_ANALYTICS: "ON" # Make Homebrew installation a little quicker + HOMEBREW_NO_AUTO_UPDATE: "ON" + HOMEBREW_NO_BOTTLE_SOURCE_FALLBACK: "ON" + HOMEBREW_NO_GITHUB_API: "ON" + HOMEBREW_NO_INSTALL_CLEANUP: "ON" + + +jobs: + run_tests: + name: Run tests with ${{ matrix.os }}, Python ${{ matrix.py_v}}, RedisAI ${{ matrix.rai }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [macos-10.15, ubuntu-20.04] # Operating systems + compiler: [8] # GNU compiler version + rai: [1.2.3, 1.2.5] # Redis AI versions + py_v: [3.7, 3.8, 3.9] # Python versions + exclude: + - os: macos-10.15 # Do not build with Redis AI 1.2.5 on MacOS + rai: 1.2.5 + + env: + SMARTSIM_REDISAI: ${{ matrix.rai }} + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.py_v }} + + - name: Install build-essentials for Ubuntu + if: contains( matrix.os, 'ubuntu' ) + run: | + sudo add-apt-repository ppa:ubuntu-toolchain-r/test + sudo apt-get update + sudo apt-get install -y build-essential + sudo apt-get install -y wget + + - name: Install GNU make for MacOS and set GITHUB_PATH + if: "contains( matrix.os, 'macos' )" + run: | + brew install make || true + echo "$(brew --prefix)/opt/make/libexec/gnubin" >> $GITHUB_PATH + + - name: Install SmartSim (with ML backends) + run: python -m pip install .[dev,ml,ray] + + - name: Install ML Runtimes with Smart + if: contains( matrix.os, 'macos' ) + run: smart build --device cpu -v + + - name: Install ML Runtimes with Smart (with pt, tf, and onnx support) + if: contains( matrix.os, 'ubuntu' ) + run: smart build --device cpu --onnx -v + + - name: Run Pytest + run: | + echo "SMARTSIM_LOG_LEVEL=debug" >> $GITHUB_ENV + py.test -s --import-mode=importlib -o log_cli=true diff --git a/requirements-dev.txt b/requirements-dev.txt index cace3bc22d..c35d7c182f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,7 +3,6 @@ smartredis>=0.1.1 redis==3.5.3 redis-py-cluster==2.1.3 sphinx==3.1.1 -numpy~=1.19.2 tqdm>=4.50.2 psutil>=5.7.2 tabulate>=0.8.9 diff --git a/setup.py b/setup.py index b9e75673ae..e10b02294c 100644 --- a/setup.py +++ b/setup.py @@ -139,7 +139,6 @@ def has_ext_modules(_placeholder): "tabulate>=0.8.9", "redis-py-cluster==2.1.3", "redis==3.5.3", - "numpy>=1.18.2", "tqdm>=4.50.2" ] diff --git a/smartsim/_core/_install/builder.py b/smartsim/_core/_install/builder.py index 99e7cdaff9..0c7ee550ef 100644 --- a/smartsim/_core/_install/builder.py +++ b/smartsim/_core/_install/builder.py @@ -68,12 +68,11 @@ def is_built(self): def build_from_git(self): raise NotImplementedError - @property - def make(self): - make_bin = shutil.which("make") - if make_bin: - return make_bin - raise BuildError("Could not find Make binary") + def binary_path(self, binary): + binary_ = shutil.which(binary) + if binary_: + return binary_ + raise BuildError(f"{binary} not found in PATH") def copy_file(self, src, dst, set_exe=False): shutil.copyfile(src, dst) @@ -98,7 +97,7 @@ def cleanup(self): if self.build_dir.is_dir(): shutil.rmtree(str(self.build_dir)) - def run_command(self, cmd, shell=True, out=None, cwd=None): + def run_command(self, cmd, shell=False, out=None, cwd=None): # option to manually disable output if necessary if not out: out = self.out @@ -120,10 +119,8 @@ def run_command(self, cmd, shell=True, out=None, cwd=None): class RedisBuilder(Builder): """Class to build Redis from Source - Supported build methods: - from git - See buildenv.py for buildtime configuration of Redis version and url. """ @@ -141,7 +138,6 @@ def is_built(self): def build_from_git(self, git_url, branch): """Build Redis from git - :param git_url: url from which to retrieve Redis :type git_url: str :param branch: branch to checkout @@ -160,7 +156,7 @@ def build_from_git(self, git_url, branch): # clone Redis clone_cmd = [ - "git", + self.binary_path("git"), "clone", git_url, "--branch", @@ -169,11 +165,17 @@ def build_from_git(self, git_url, branch): "1", "redis", ] - self.run_command(" ".join(clone_cmd), shell=True, cwd=self.build_dir) + self.run_command(clone_cmd, cwd=self.build_dir) # build Redis - cmd = [self.make, "-j", str(self.jobs), f"MALLOC={self.malloc}"] - self.run_command(" ".join(cmd), shell=True, cwd=str(redis_build_path)) + build_cmd = [ + self.binary_path("make"), + "--debug=all", + "-j", + str(self.jobs), + f"MALLOC={self.malloc}", + ] + self.run_command(build_cmd, cwd=str(redis_build_path)) # move redis binaries to smartsim/smartsim/_core/bin redis_src_dir = redis_build_path / "src" @@ -187,10 +189,8 @@ def build_from_git(self, git_url, branch): class RedisAIBuilder(Builder): """Class to build RedisAI from Source - Supported build method: - from git - See buildenv.py for buildtime configuration of RedisAI version and url. """ @@ -223,13 +223,11 @@ def is_built(self): def copy_tf_cmake(self): """Copy the FindTensorFlow.cmake file to the build directory as the version included in RedisAI is out of date for us. - Note: opt/cmake/modules removed in RedisAI v1.2.5 """ # remove the previous version - tf_cmake = self.rai_build_path.joinpath( - "opt/cmake/modules/FindTensorFlow.cmake" - ).resolve() + tf_cmake = self.rai_build_path / "opt/cmake/modules/FindTensorFlow.cmake" + tf_cmake.resolve() if tf_cmake.is_file(): tf_cmake.unlink() # copy ours in @@ -239,7 +237,6 @@ def copy_tf_cmake(self): def build_from_git(self, git_url, branch, device): """Build RedisAI from git - :param git_url: url from which to retrieve RedisAI :type git_url: str :param branch: branch to checkout @@ -258,6 +255,7 @@ def build_from_git(self, git_url, branch, device): # clone RedisAI clone_cmd = [ + self.binary_path("env"), "GIT_LFS_SKIP_SMUDGE=1", "git", "clone", @@ -268,34 +266,32 @@ def build_from_git(self, git_url, branch, device): "--depth=1", "RedisAI", ] - clone_cmd = " ".join(clone_cmd) - self.run_command( - clone_cmd, out=subprocess.DEVNULL, shell=True, cwd=self.build_dir - ) + self.run_command(clone_cmd, out=subprocess.DEVNULL, cwd=self.build_dir) # copy FindTensorFlow.cmake to RAI cmake dir self.copy_tf_cmake() # get RedisAI dependencies dep_cmd = [ + self.binary_path("env"), f"WITH_PT=0", # torch is always 0 because we never use the torch from RAI f"WITH_TF={self.tf}", f"WITH_TFLITE=0", # never build with TF lite (for now) f"WITH_ORT={self.onnx}", "VERBOSE=1", - "bash", - "get_deps.sh", + self.binary_path("bash"), + self.rai_build_path / "get_deps.sh", device, ] - dep_cmd = " ".join(dep_cmd) + self.run_command( dep_cmd, - shell=True, out=subprocess.DEVNULL, # suppress this as it's not useful cwd=self.rai_build_path, ) build_cmd = [ + self.binary_path("env"), f"WITH_PT={self.torch}", # but we built it in if the user specified it f"WITH_TF={self.tf}", f"WITH_TFLITE=0", # never build TF Lite @@ -311,12 +307,17 @@ def build_from_git(self, git_url, branch, device): if self.torch_dir: self.env["Torch_DIR"] = str(self.torch_dir) - build_cmd.extend([self.make, "-C", "opt", "-j", f"{self.jobs}", "build"]) - self.run_command( - " ".join(build_cmd), - cwd=self.rai_build_path, - shell=True, + build_cmd.extend( + [ + self.binary_path("make"), + "-C", + str(self.rai_build_path / "opt"), + "-j", + f"{self.jobs}", + "build", + ] ) + self.run_command(build_cmd, cwd=self.rai_build_path) self._install_backends(device) if self.torch: @@ -325,7 +326,6 @@ def build_from_git(self, git_url, branch, device): def _install_backends(self, device): """Move backend libraries to smartsim/_core/lib/ - :param device: cpu or cpu :type device: str """ @@ -341,7 +341,6 @@ def _install_backends(self, device): def _move_torch_libs(self): """Move pip install torch libraries - Since we use pip installed torch libraries for building RedisAI, we need to move them into the LD_runpath of redisai.so in the smartsim/_core/lib directory. diff --git a/smartsim/_core/utils/helpers.py b/smartsim/_core/utils/helpers.py index 736f72d8cf..2c7cd632e2 100644 --- a/smartsim/_core/utils/helpers.py +++ b/smartsim/_core/utils/helpers.py @@ -36,13 +36,12 @@ def get_base_36_repr(positive_int): """Converts a positive integer to its base 36 representation - :param positive_int: the positive integer to convert :type positive_int: int :return: base 36 representation of the given positive int :rtype: str """ - digits = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ' + digits = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" result = [] while positive_int: @@ -50,7 +49,7 @@ def get_base_36_repr(positive_int): result.append(next_digit) positive_int //= 36 - return ''.join(reversed(result)) + return "".join(reversed(result)) def get_ip_from_host(host):