From 45e6bbdf385bbef0ac2076326c239b2e1412aa38 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 22 Jul 2022 13:06:07 +0200 Subject: [PATCH 01/54] adding easyblocks: cudacompat.py --- easybuild/easyblocks/c/cudacompat.py | 133 +++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 easybuild/easyblocks/c/cudacompat.py diff --git a/easybuild/easyblocks/c/cudacompat.py b/easybuild/easyblocks/c/cudacompat.py new file mode 100644 index 0000000000..4b1d14d951 --- /dev/null +++ b/easybuild/easyblocks/c/cudacompat.py @@ -0,0 +1,133 @@ +## +# Copyright 2012-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for CUDA compat libraries, implemented as an easyblock + +Ref: https://docs.nvidia.com/deploy/cuda-compatibility/index.html#manually-installing-from-runfile + +@author: Alexander Grund (TU Dresden) +""" + +import os +from distutils.version import LooseVersion + +from easybuild.easyblocks.generic.binary import Binary +from easybuild.framework.easyconfig import MANDATORY +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import copy_file, find_glob_pattern, mkdir, symlink +from easybuild.tools.run import run_cmd + + +class EB_CUDAcompat(Binary): + """ + Support for installing CUDA compat libraries. + """ + + @staticmethod + def extra_options(): + """Add variable for driver version""" + extra_vars = Binary.extra_options() + extra_vars.update({ + 'nv_version': [None, "Version of the driver package", MANDATORY], + }) + # We don't need the extract and install step from the Binary EasyBlock + del extra_vars['extract_sources'] + del extra_vars['install_cmd'] + # And also no need to modify the PATH + del extra_vars['prepend_to_path'] + return extra_vars + + def __init__(self, *args, **kwargs): + """Initialize custom class variables for Clang.""" + super(EB_CUDAcompat, self).__init__(*args, **kwargs) + + def extract_step(self): + """Extract the files without running the installer.""" + execpath = self.src[0]['path'] + tmpdir = os.path.join(self.builddir, 'tmp') + targetdir = os.path.join(self.builddir, 'extracted') + run_cmd("/bin/sh " + execpath + " --extract-only --tmpdir='%s' --target '%s'" % (tmpdir, targetdir)) + self.src[0]['finalpath'] = targetdir + + def install_step(self): + """Install CUDA compat libraries by copying library files and creating the symlinks.""" + libdir = os.path.join(self.installdir, 'lib') + mkdir(libdir) + + # From https://docs.nvidia.com/deploy/cuda-compatibility/index.html#installing-from-network-repo: + # The cuda-compat package consists of the following files: + # - libcuda.so.* - the CUDA Driver + # - libnvidia-nvvm.so.* - JIT LTO ( CUDA 11.5 and later only) + # - libnvidia-ptxjitcompiler.so.* - the JIT (just-in-time) compiler for PTX files + + library_globs = [ + 'libcuda.so.*', + 'libnvidia-ptxjitcompiler.so.*', + ] + if LooseVersion(self.version) >= '11.5': + library_globs.append('libnvidia-nvvm.so.*') + + startdir = self.cfg['start_dir'] + nv_version = self.cfg['nv_version'] + for library_glob in library_globs: + library_path = find_glob_pattern(os.path.join(startdir, library_glob)) + library = os.path.basename(library_path) + # Sanity check the version + if library_glob == 'libcuda.so.*': + library_version = library.split('.', 2)[2] + if library_version != nv_version: + raise EasyBuildError('Expected driver version %s (from nv_version) but found %s ' + '(determined from file %s)', nv_version, library_version, library_path) + + copy_file(library_path, os.path.join(libdir, library)) + if library.endswith('.' + nv_version): + # E.g. libcuda.so.510.73.08 -> libcuda.so.1 + versioned_symlink = library[:-len(nv_version)] + '1' + else: + # E.g. libnvidia-nvvm.so.4.0.0 -> libnvidia-nvvm.so.4 + versioned_symlink = library.rsplit('.', 2)[0] + symlink(library, os.path.join(libdir, versioned_symlink), use_abspath_source=False) + # E.g. libcuda.so.1 -> libcuda.so + unversioned_symlink = versioned_symlink.rsplit('.', 1)[0] + symlink(versioned_symlink, os.path.join(libdir, unversioned_symlink), use_abspath_source=False) + + def make_module_extra(self): + """Skip the changes from the Binary EasyBlock.""" + + return super(Binary, self).make_module_extra() + + def sanity_check_step(self): + """Check for core files (unversioned libs, symlinks)""" + libraries = [ + 'libcuda.so', + 'libnvidia-ptxjitcompiler.so', + ] + if LooseVersion(self.version) >= '11.5': + libraries.append('libnvidia-nvvm.so') + custom_paths = { + 'files': [os.path.join(self.installdir, 'lib', x) for x in libraries], + 'dirs': ['lib', 'lib64'], + } + super(EB_CUDAcompat, self).sanity_check_step(custom_paths=custom_paths) From e5a65b9a160e089ac633899b99b9d002f0d66f7d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 25 Jul 2022 15:06:00 +0200 Subject: [PATCH 02/54] Check for EULA acceptance before downloading sources --- easybuild/easyblocks/c/cudacompat.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/easybuild/easyblocks/c/cudacompat.py b/easybuild/easyblocks/c/cudacompat.py index 4b1d14d951..4969357b9a 100644 --- a/easybuild/easyblocks/c/cudacompat.py +++ b/easybuild/easyblocks/c/cudacompat.py @@ -63,6 +63,16 @@ def __init__(self, *args, **kwargs): """Initialize custom class variables for Clang.""" super(EB_CUDAcompat, self).__init__(*args, **kwargs) + def fetch_step(self, *args, **kwargs): + """Check for EULA acceptance prior to getting sources.""" + # EULA for NVIDIA driver must be accepted via --accept-eula-for EasyBuild configuration option, + # or via 'accept_eula = True' in easyconfig file + self.check_accepted_eula( + name='NVIDIA-driver', + more_info='https://www.nvidia.com/content/DriverDownload-March2009/licence.php?lang=us' + ) + super(EB_CUDAcompat, self).fetch_step(*args, **kwargs) + def extract_step(self): """Extract the files without running the installer.""" execpath = self.src[0]['path'] From 3d11d24ae40d180ac54498e21e565675d3f56329 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 25 Jul 2022 14:09:05 +0000 Subject: [PATCH 03/54] introduce is_extend, is_dot_org, is_dot_com and simplify logic --- easybuild/easyblocks/o/openfoam.py | 68 +++++++++++++++--------------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/easybuild/easyblocks/o/openfoam.py b/easybuild/easyblocks/o/openfoam.py index bbdfc4e29e..a112f8f063 100644 --- a/easybuild/easyblocks/o/openfoam.py +++ b/easybuild/easyblocks/o/openfoam.py @@ -78,7 +78,11 @@ def __init__(self, *args, **kwargs): self.looseversion = LooseVersion(clean_version) - if 'extend' in self.name.lower(): + self.is_extend = 'extend' in self.name.lower() + self.is_dot_com = self.looseversion >= LooseVersion('1606') + self.is_dot_org = self.looseversion <= LooseVersion('100') + + if self.is_extend: if self.looseversion >= LooseVersion('3.0'): self.openfoamdir = 'foam-extend-%s' % self.version else: @@ -153,7 +157,7 @@ def configure_step(self): extra_flags = '-fuse-ld=bfd' # older versions of OpenFOAM-Extend require -fpermissive - if 'extend' in self.name.lower() and self.looseversion < LooseVersion('2.0'): + if self.is_extend and self.looseversion < LooseVersion('2.0'): extra_flags += ' -fpermissive' if self.looseversion < LooseVersion('3.0'): @@ -174,13 +178,13 @@ def configure_step(self): regex_subs = [(r"^(setenv|export) WM_THIRD_PARTY_USE_.*[ =].*$", r"# \g<0>")] # this does not work for OpenFOAM Extend lower than 2.0 - if 'extend' not in self.name.lower() or self.looseversion >= LooseVersion('2.0'): + if not self.is_extend or self.looseversion >= LooseVersion('2.0'): key = "WM_PROJECT_VERSION" regex_subs += [(r"^(setenv|export) %s=.*$" % key, r"export %s=%s #\g<0>" % (key, self.version))] WM_env_var = ['WM_COMPILER', 'WM_COMPILE_OPTION', 'WM_MPLIB', 'WM_THIRD_PARTY_DIR'] # OpenFOAM >= 3.0.0 can use 64 bit integers - if 'extend' not in self.name.lower() and self.looseversion >= LooseVersion('3.0'): + if not self.is_extend and self.looseversion >= LooseVersion('3.0'): WM_env_var.append('WM_LABEL_SIZE') for env_var in WM_env_var: regex_subs.append((r"^(setenv|export) (?P%s)[ =](?P.*)$" % env_var, @@ -257,14 +261,14 @@ def configure_step(self): env.setvar("WM_NCOMPPROCS", str(self.cfg['parallel'])) # OpenFOAM >= 3.0.0 can use 64 bit integers - if 'extend' not in self.name.lower() and self.looseversion >= LooseVersion('3.0'): + if not self.is_extend and self.looseversion >= LooseVersion('3.0'): if self.toolchain.options['i8']: env.setvar("WM_LABEL_SIZE", '64') else: env.setvar("WM_LABEL_SIZE", '32') # make sure lib/include dirs for dependencies are found - openfoam_extend_v3 = 'extend' in self.name.lower() and self.looseversion >= LooseVersion('3.0') + openfoam_extend_v3 = self.is_extend and self.looseversion >= LooseVersion('3.0') if self.looseversion < LooseVersion("2") or openfoam_extend_v3: self.log.debug("List of deps: %s" % self.cfg.dependencies()) for dep in self.cfg.dependencies(): @@ -297,7 +301,7 @@ def build_step(self): setup_cmake_env(self.toolchain) precmd = "source %s" % os.path.join(self.builddir, self.openfoamdir, "etc", "bashrc") - if 'extend' not in self.name.lower() and self.looseversion >= LooseVersion('4.0'): + if not self.is_extend and self.looseversion >= LooseVersion('4.0'): if self.looseversion >= LooseVersion('2006'): cleancmd = "cd $WM_PROJECT_DIR && wclean -platform -all && cd -" else: @@ -312,7 +316,7 @@ def build_step(self): 'prebuildopts': self.cfg['prebuildopts'], 'makecmd': os.path.join(self.builddir, self.openfoamdir, '%s'), } - if 'extend' in self.name.lower() and self.looseversion >= LooseVersion('3.0'): + if self.is_extend and self.looseversion >= LooseVersion('3.0'): qa = { "Proceed without compiling ParaView [Y/n]": 'Y', "Proceed without compiling cudaSolvers? [Y/n]": 'Y', @@ -340,7 +344,7 @@ def det_psubdir(self): """Determine the platform-specific installation directory for OpenFOAM.""" # OpenFOAM >= 3.0.0 can use 64 bit integers # same goes for OpenFOAM-Extend >= 4.1 - if 'extend' in self.name.lower(): + if self.is_extend: set_int_size = self.looseversion >= LooseVersion('4.1') else: set_int_size = self.looseversion >= LooseVersion('3.0') @@ -357,7 +361,7 @@ def det_psubdir(self): arch = get_cpu_architecture() if arch == AARCH64: # Variants have different abbreviations for ARM64... - if self.looseversion < LooseVersion("100"): + if self.is_dot_org: archpart = 'Arm64' else: archpart = 'ARM64' @@ -386,7 +390,7 @@ def install_step(self): # to make sure they take precedence over the libraries in the dummy subdirectory shlib_ext = get_shared_lib_ext() psubdir = self.det_psubdir() - openfoam_extend_v3 = 'extend' in self.name.lower() and self.looseversion >= LooseVersion('3.0') + openfoam_extend_v3 = self.is_extend and self.looseversion >= LooseVersion('3.0') if openfoam_extend_v3 or self.looseversion < LooseVersion("2"): libdir = os.path.join(self.installdir, self.openfoamdir, "lib", psubdir) else: @@ -410,7 +414,7 @@ def sanity_check_step(self): shlib_ext = get_shared_lib_ext() psubdir = self.det_psubdir() - openfoam_extend_v3 = 'extend' in self.name.lower() and self.looseversion >= LooseVersion('3.0') + openfoam_extend_v3 = self.is_extend and self.looseversion >= LooseVersion('3.0') if openfoam_extend_v3 or self.looseversion < LooseVersion("2"): toolsdir = os.path.join(self.openfoamdir, "applications", "bin", psubdir) libsdir = os.path.join(self.openfoamdir, "lib", psubdir) @@ -422,26 +426,28 @@ def sanity_check_step(self): # some randomly selected binaries # if one of these is missing, it's very likely something went wrong + tools = ["boundaryFoam", "engineFoam", "buoyantSimpleFoam", "buoyantBoussinesqSimpleFoam", "sonicFoam"] + tools += ["surfaceAdd", "surfaceFind", "surfaceSmooth"] + tools += ["blockMesh", "checkMesh", "deformedGeom", "engineSwirl", "modifyMesh", "refineMesh"] + + # surfaceSmooth is replaced by surfaceLambdaMuSmooth is OpenFOAM v2.3.0 + if not self.is_extend and self.looseversion >= LooseVersion("2.3.0"): + tools.remove("surfaceSmooth") + tools.append("surfaceLambdaMuSmooth") + # sonicFoam and buoyantBoussineqSimpleFoam deprecated in version 7+ + if self.is_dot_org and self.looseversion >= LooseVersion('7'): + tools.remove("buoyantBoussinesqSimpleFoam") + tools.remove("sonicFoam") + bins = [os.path.join(self.openfoamdir, "bin", x) for x in ["paraFoam"]] + \ - [os.path.join(toolsdir, "buoyantSimpleFoam")] + \ - [os.path.join(toolsdir, "%sFoam" % x) for x in ["boundary", "engine"]] + \ - [os.path.join(toolsdir, "surface%s" % x) for x in ["Add", "Find", "Smooth"]] + \ - [os.path.join(toolsdir, x) for x in ['blockMesh', 'checkMesh', 'deformedGeom', 'engineSwirl', - 'modifyMesh', 'refineMesh']] + [os.path.join(toolsdir, x) for x in tools] # test setting up the OpenFOAM environment in bash shell load_openfoam_env = "source $FOAM_BASH" custom_commands = [load_openfoam_env] - # only include Boussinesq and sonic since for OpenFOAM < 7, since those solvers have been deprecated - if self.looseversion < LooseVersion('7'): - bins.extend([ - os.path.join(toolsdir, "buoyantBoussinesqSimpleFoam"), - os.path.join(toolsdir, "sonicFoam") - ]) - # check for the Pstream and scotchDecomp libraries, there must be a dummy one and an mpi one - if 'extend' in self.name.lower(): + if self.is_extend: libs = [os.path.join(libsdir, "libscotchDecomp.%s" % shlib_ext), os.path.join(libsdir, "libmetisDecomp.%s" % shlib_ext)] if self.looseversion < LooseVersion('3.2'): @@ -452,7 +458,7 @@ def sanity_check_step(self): libs.extend([os.path.join(libsdir, "libparMetisDecomp.%s" % shlib_ext)]) else: # OpenFOAM v2012 puts mpi into eb-mpi - if self.looseversion >= LooseVersion("2012"): + if self.is_dot_com and self.looseversion >= LooseVersion("2012"): mpilibssubdir = "eb-mpi" else: mpilibssubdir = "mpi" @@ -463,12 +469,8 @@ def sanity_check_step(self): [os.path.join(libsdir, "libscotchDecomp.%s" % shlib_ext)] + \ [os.path.join(libsdir, "dummy", "libscotchDecomp.%s" % shlib_ext)] - if 'extend' not in self.name.lower() and self.looseversion >= LooseVersion("2.3.0"): - # surfaceSmooth is replaced by surfaceLambdaMuSmooth is OpenFOAM v2.3.0 - bins.remove(os.path.join(toolsdir, "surfaceSmooth")) - bins.append(os.path.join(toolsdir, "surfaceLambdaMuSmooth")) - if 'extend' not in self.name.lower() and self.looseversion >= LooseVersion("2.4.0"): + if not self.is_extend and self.looseversion >= LooseVersion("2.4.0"): # also check for foamMonitor for OpenFOAM versions other than OpenFOAM-Extend bins.append(os.path.join(self.openfoamdir, 'bin', 'foamMonitor')) @@ -484,7 +486,7 @@ def sanity_check_step(self): # run motorBike tutorial case to ensure the installation is functional (if it's available); # only for recent (>= v6.0) versions of openfoam.org variant - if self.looseversion >= LooseVersion('6') and self.looseversion < LooseVersion('100'): + if self.is_dot_org and self.looseversion >= LooseVersion('6'): openfoamdir_path = os.path.join(self.installdir, self.openfoamdir) motorbike_path = os.path.join(openfoamdir_path, 'tutorials', 'incompressible', 'simpleFoam', 'motorBike') if os.path.exists(motorbike_path): @@ -537,7 +539,7 @@ def make_module_extra(self, altroot=None, altversion=None): ] # OpenFOAM >= 3.0.0 can use 64 bit integers - if 'extend' not in self.name.lower() and self.looseversion >= LooseVersion('3.0'): + if not self.is_extend and self.looseversion >= LooseVersion('3.0'): if self.toolchain.options['i8']: env_vars += [('WM_LABEL_SIZE', '64')] else: From 3a5e8ccb17bf544a645c7b9ccc33c69bdc4700da Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 25 Jul 2022 14:11:15 +0000 Subject: [PATCH 04/54] update tools for OpenFOAM 10 --- easybuild/easyblocks/o/openfoam.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/easybuild/easyblocks/o/openfoam.py b/easybuild/easyblocks/o/openfoam.py index a112f8f063..67f651f9e3 100644 --- a/easybuild/easyblocks/o/openfoam.py +++ b/easybuild/easyblocks/o/openfoam.py @@ -438,6 +438,14 @@ def sanity_check_step(self): if self.is_dot_org and self.looseversion >= LooseVersion('7'): tools.remove("buoyantBoussinesqSimpleFoam") tools.remove("sonicFoam") + # buoyantSimpleFoam replaced by buoyantFoam in versions 10+ + if self.is_dot_org and self.looseversion >= LooseVersion("10"): + tools.remove("buoyantSimpleFoam") + tools.append("buoyantFoam") + # engineFoam replaced by reactingFoam in versions 10+ + if self.is_dot_org and self.looseversion >= LooseVersion("10"): + tools.remove("engineFoam") + tools.append("reactingFoam") bins = [os.path.join(self.openfoamdir, "bin", x) for x in ["paraFoam"]] + \ [os.path.join(toolsdir, x) for x in tools] From 400235564ccd4f517e706c9bf446481fcfb12964 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 25 Jul 2022 14:14:59 +0000 Subject: [PATCH 05/54] appeasing hound --- easybuild/easyblocks/o/openfoam.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/easyblocks/o/openfoam.py b/easybuild/easyblocks/o/openfoam.py index 67f651f9e3..82327581b0 100644 --- a/easybuild/easyblocks/o/openfoam.py +++ b/easybuild/easyblocks/o/openfoam.py @@ -477,7 +477,6 @@ def sanity_check_step(self): [os.path.join(libsdir, "libscotchDecomp.%s" % shlib_ext)] + \ [os.path.join(libsdir, "dummy", "libscotchDecomp.%s" % shlib_ext)] - if not self.is_extend and self.looseversion >= LooseVersion("2.4.0"): # also check for foamMonitor for OpenFOAM versions other than OpenFOAM-Extend bins.append(os.path.join(self.openfoamdir, 'bin', 'foamMonitor')) From 49d2db06819d45ddb11542c9a8e84077c7529436 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 29 Jul 2022 16:40:36 +0200 Subject: [PATCH 06/54] Add nvidia-smi checks to validate driver requirements and installed module --- easybuild/easyblocks/c/cudacompat.py | 130 +++++++++++++++++++++++++-- 1 file changed, 123 insertions(+), 7 deletions(-) diff --git a/easybuild/easyblocks/c/cudacompat.py b/easybuild/easyblocks/c/cudacompat.py index 4969357b9a..8aea98f23b 100644 --- a/easybuild/easyblocks/c/cudacompat.py +++ b/easybuild/easyblocks/c/cudacompat.py @@ -34,9 +34,10 @@ from distutils.version import LooseVersion from easybuild.easyblocks.generic.binary import Binary -from easybuild.framework.easyconfig import MANDATORY -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import copy_file, find_glob_pattern, mkdir, symlink +from easybuild.framework.easyconfig import CUSTOM, MANDATORY +from easybuild.tools.build_log import EasyBuildError, print_warning +from easybuild.tools.config import build_option, IGNORE +from easybuild.tools.filetools import copy_file, find_glob_pattern, mkdir, symlink, which from easybuild.tools.run import run_cmd @@ -50,6 +51,7 @@ def extra_options(): """Add variable for driver version""" extra_vars = Binary.extra_options() extra_vars.update({ + 'compatible_driver_versions': [None, "Minimum (system) CUDA driver versions which are compatible", CUSTOM], 'nv_version': [None, "Version of the driver package", MANDATORY], }) # We don't need the extract and install step from the Binary EasyBlock @@ -60,8 +62,52 @@ def extra_options(): return extra_vars def __init__(self, *args, **kwargs): - """Initialize custom class variables for Clang.""" + """Initialize custom class variables for CUDACompat.""" super(EB_CUDAcompat, self).__init__(*args, **kwargs) + self._has_nvidia_smi = None + + @property + def has_nvidia_smi(self): + """Return whether the system has nvidia-smi and print a warning once if not""" + if self._has_nvidia_smi is None: + self._has_nvidia_smi = which('nvidia-smi', on_error=IGNORE) is not None + if not self._has_nvidia_smi: + print_warning('Could not find nvidia-smi. Assuming a system without GPUs and skipping checks!') + return self._has_nvidia_smi + + def _run_nvidia_smi(self, args): + """ + Run nvidia-smi with the given argument(s) and return the output. + + Also does sensible logging. + Raises RuntimeError on failure. + """ + if not self.has_nvidia_smi: + raise RuntimeError('Could not find nvidia-smi.') + cmd = 'nvidia-smi ' + args + out, ec = run_cmd(cmd, log_ok=False, log_all=False, regexp=False) + if ec != 0: + raise RuntimeError("`%s` returned exit code %s with output:\n%s" % (cmd, ec, out)) + else: + self.log.info('`%s` succeeded with output:\n%s' % (cmd, out)) + return out.strip().split('\n') + + def prepare_step(self, *args, **kwargs): + """Parse and check the compatible_driver_versions value of the EasyConfig""" + compatible_driver_versions = self.cfg.get('compatible_driver_versions') + if compatible_driver_versions: + try: + # Create a dictionary with the major version as the keys + self.compatible_driver_version_map = { + int(v.split('.', 1)[0]): v + for v in compatible_driver_versions + } + except ValueError: + raise EasyBuildError("Invalid format of compatible_driver_versions. " + "Expected numeric major versions, got '%s'", compatible_driver_versions) + else: + self.compatible_driver_version_map = None + super(EB_CUDAcompat, self).prepare_step(*args, **kwargs) def fetch_step(self, *args, **kwargs): """Check for EULA acceptance prior to getting sources.""" @@ -71,7 +117,7 @@ def fetch_step(self, *args, **kwargs): name='NVIDIA-driver', more_info='https://www.nvidia.com/content/DriverDownload-March2009/licence.php?lang=us' ) - super(EB_CUDAcompat, self).fetch_step(*args, **kwargs) + return super(EB_CUDAcompat, self).fetch_step(*args, **kwargs) def extract_step(self): """Extract the files without running the installer.""" @@ -81,6 +127,44 @@ def extract_step(self): run_cmd("/bin/sh " + execpath + " --extract-only --tmpdir='%s' --target '%s'" % (tmpdir, targetdir)) self.src[0]['finalpath'] = targetdir + def test_step(self): + """ + Check for a compatible driver version if the EC has that information. + + This can be skipped with `--skip-test-step`. + """ + + if self.compatible_driver_version_map and self.has_nvidia_smi: + try: + out_lines = self._run_nvidia_smi('--query-gpu=driver_version --format=csv,noheader') + if not out_lines or not out_lines[0]: + raise RuntimeError('nvidia-smi did not find any GPUs on the system') + driver_version = out_lines[0] + version_parts = driver_version.split('.') + if len(version_parts) < 3 or any(not v.isdigit() for v in version_parts): + raise RuntimeError("Expected the version to be in format x.y.z (all numeric) " + "but got '%s'" % driver_version) + except RuntimeError as err: + self.log.warning("Failed to get the CUDA driver version: %s", err) + driver_version = None + + if not driver_version: + print_warning('Failed to determine the CUDA driver version, so skipping the compatibility check!') + else: + driver_version_major = int(driver_version.split('.', 1)[0]) + compatible_driver_versions = ', '.join(self.compatible_driver_version_map.values()) + if driver_version_major not in self.compatible_driver_version_map: + raise EasyBuildError('The installed CUDA driver %s is not a supported branch/major version for ' + '%s %s. Supported drivers: %s', + driver_version, self.name, self.version, compatible_driver_versions) + elif LooseVersion(driver_version) < self.compatible_driver_version_map[driver_version_major]: + raise EasyBuildError('The installed CUDA driver %s is to old for %s %s. Supported drivers: %s', + driver_version, self.name, self.version, compatible_driver_versions) + else: + self.log.info('The installed CUDA driver %s appears to be supported.', driver_version) + + return super(EB_CUDAcompat, self).test_step() + def install_step(self): """Install CUDA compat libraries by copying library files and creating the symlinks.""" libdir = os.path.join(self.installdir, 'lib') @@ -123,10 +207,16 @@ def install_step(self): unversioned_symlink = versioned_symlink.rsplit('.', 1)[0] symlink(versioned_symlink, os.path.join(libdir, unversioned_symlink), use_abspath_source=False) + def make_module_req_guess(self): + """Don't try to guess anything.""" + return dict() + def make_module_extra(self): - """Skip the changes from the Binary EasyBlock.""" + """Skip the changes from the Binary EasyBlock and (only) set LD_LIBRARY_PATH.""" - return super(Binary, self).make_module_extra() + txt = super(Binary, self).make_module_extra() + txt += self.module_generator.prepend_paths('LD_LIBRARY_PATH', 'lib') + return txt def sanity_check_step(self): """Check for core files (unversioned libs, symlinks)""" @@ -141,3 +231,29 @@ def sanity_check_step(self): 'dirs': ['lib', 'lib64'], } super(EB_CUDAcompat, self).sanity_check_step(custom_paths=custom_paths) + + if self.has_nvidia_smi: + fake_mod_data = None + + # skip loading of fake module when using --sanity-check-only, load real module instead + if build_option('sanity_check_only'): + self.load_module() + elif not self.dry_run: + fake_mod_data = self.load_fake_module(purge=True, verbose=True) + + try: + out_lines = self._run_nvidia_smi('--query --display=COMPUTE') + + if fake_mod_data: + self.clean_up_fake_module(fake_mod_data) + + cuda_version = next((line.rsplit(' ', 1)[1] for line in out_lines if line.startswith('CUDA')), None) + if not cuda_version: + raise RuntimeError('Failed to find CUDA version!') + self.log.info('CUDA version (as per nvidia-smi) after loading the module: ' + cuda_version) + if LooseVersion(cuda_version) != self.version: + raise RuntimeError('Reported CUDA version %s is not %s' % (cuda_version, self.version)) + except RuntimeError as err: + if fake_mod_data: + self.clean_up_fake_module(fake_mod_data) + raise EasyBuildError('Version check via nvidia-smi after loading the module failed: %s', err) From 7b529bc491753238d16b6b89dcd7354ac679155d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 12 Sep 2022 09:04:15 +0200 Subject: [PATCH 07/54] bump version to 4.6.1dev --- easybuild/easyblocks/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/__init__.py b/easybuild/easyblocks/__init__.py index 7aff8318b6..c546b0843b 100644 --- a/easybuild/easyblocks/__init__.py +++ b/easybuild/easyblocks/__init__.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.6.1') +VERSION = LooseVersion('4.6.2.dev0') UNKNOWN = 'UNKNOWN' From caac0f5e5d3c5763369cdd8aec09f4b58d3074b0 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 20 Sep 2022 21:34:38 +0000 Subject: [PATCH 08/54] added support for ESMPy in ESMF --- easybuild/easyblocks/e/esmf.py | 43 ++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/e/esmf.py b/easybuild/easyblocks/e/esmf.py index 9af0a257a1..b3fbe50b69 100644 --- a/easybuild/easyblocks/e/esmf.py +++ b/easybuild/easyblocks/e/esmf.py @@ -35,7 +35,7 @@ import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.modules import get_software_root +from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import get_shared_lib_ext @@ -114,6 +114,41 @@ def configure_step(self): cmd = "make info" run_cmd(cmd, log_all=True, simple=True, log_ok=True) + + def install_step(self): + # first, install the software + super(EB_ESMF, self).install_step() + + python = get_software_version('Python') + if python: + # then, install the python bindings + py_subdir = os.path.join(self.builddir, 'esmf-ESMF_%s' % '_'.join(self.version.split('.')), 'src', 'addon', 'ESMPy') + try: + os.chdir(py_subdir) + except OSError as err: + raise EasyBuildError("Failed to move to: %s", err) + + cmd = "python setup.py build --ESMFMKFILE=%s/lib/esmf.mk && python setup.py install --prefix=%s" % (self.installdir, self.installdir) + run_cmd(cmd, log_all=True, simple=True, log_ok=True) + + + def make_module_extra(self): + """Add install path to PYTHONPATH or EBPYTHONPREFIXES""" + + txt = super(EB_ESMF, self).make_module_extra() + + python = get_software_version('Python') + if python: + if self.cfg['multi_deps'] and 'Python' in self.cfg['multi_deps']: + txt += self.module_generator.prepend_paths('EBPYTHONPREFIXES', []) + else: + pyshortver = '.'.join(get_software_version('Python').split('.')[:2]) + pythonpath = os.path.join('lib', 'python%s' % pyshortver, 'site-packages') + txt += self.module_generator.prepend_paths('PYTHONPATH', [pythonpath]) + + return txt + + def sanity_check_step(self): """Custom sanity check for ESMF.""" @@ -130,4 +165,8 @@ def sanity_check_step(self): 'dirs': ['include', 'mod'], } - super(EB_ESMF, self).sanity_check_step(custom_paths=custom_paths) + custom_commands = [] + if get_software_root('Python'): + custom_commands += ["python -c 'import ESMF'"] + + super(EB_ESMF, self).sanity_check_step(custom_commands=custom_commands, custom_paths=custom_paths) From 8c4a28ca267ae69b7a4d6d26021c2e43f8d0f262 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 20 Sep 2022 21:41:53 +0000 Subject: [PATCH 09/54] appeasing hound --- easybuild/easyblocks/e/esmf.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/e/esmf.py b/easybuild/easyblocks/e/esmf.py index b3fbe50b69..e72fa43b7d 100644 --- a/easybuild/easyblocks/e/esmf.py +++ b/easybuild/easyblocks/e/esmf.py @@ -114,7 +114,6 @@ def configure_step(self): cmd = "make info" run_cmd(cmd, log_all=True, simple=True, log_ok=True) - def install_step(self): # first, install the software super(EB_ESMF, self).install_step() @@ -122,16 +121,17 @@ def install_step(self): python = get_software_version('Python') if python: # then, install the python bindings - py_subdir = os.path.join(self.builddir, 'esmf-ESMF_%s' % '_'.join(self.version.split('.')), 'src', 'addon', 'ESMPy') + py_subdir = os.path.join(self.builddir, 'esmf-ESMF_%s' % '_'.join(self.version.split('.')), + 'src', 'addon', 'ESMPy') try: os.chdir(py_subdir) except OSError as err: raise EasyBuildError("Failed to move to: %s", err) - cmd = "python setup.py build --ESMFMKFILE=%s/lib/esmf.mk && python setup.py install --prefix=%s" % (self.installdir, self.installdir) + cmd = "python setup.py build --ESMFMKFILE=%s/lib/esmf.mk " % self.installdir + cmd += " && python setup.py install --prefix=%s" % self.installdir run_cmd(cmd, log_all=True, simple=True, log_ok=True) - def make_module_extra(self): """Add install path to PYTHONPATH or EBPYTHONPREFIXES""" From e38dc59d56d8eedeba1bda89878a072131512a6c Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 20 Sep 2022 21:44:53 +0000 Subject: [PATCH 10/54] appeasing lint --- easybuild/easyblocks/e/esmf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/easyblocks/e/esmf.py b/easybuild/easyblocks/e/esmf.py index e72fa43b7d..c6051610fe 100644 --- a/easybuild/easyblocks/e/esmf.py +++ b/easybuild/easyblocks/e/esmf.py @@ -148,7 +148,6 @@ def make_module_extra(self): return txt - def sanity_check_step(self): """Custom sanity check for ESMF.""" From 6177e62c47f4e048cc4c1d4cbff62e9800354734 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 20 Sep 2022 21:48:07 +0000 Subject: [PATCH 11/54] add authorship --- easybuild/easyblocks/e/esmf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/easyblocks/e/esmf.py b/easybuild/easyblocks/e/esmf.py index c6051610fe..2e795d1642 100644 --- a/easybuild/easyblocks/e/esmf.py +++ b/easybuild/easyblocks/e/esmf.py @@ -27,6 +27,7 @@ @author: Kenneth Hoste (Ghent University) @author: Damian Alvarez (Forschungszentrum Juelich GmbH) +@author: Maxime Boissonneault (Digital Research Alliance of Canada) """ import os from distutils.version import LooseVersion From 3156ab3e2c904f632102489af0f290a56d6d5d6c Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Wed, 21 Sep 2022 13:17:26 +0000 Subject: [PATCH 12/54] fix EBPYTHONPREFIXES for multi_deps --- easybuild/easyblocks/e/esmf.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/e/esmf.py b/easybuild/easyblocks/e/esmf.py index 2e795d1642..a5d09277c4 100644 --- a/easybuild/easyblocks/e/esmf.py +++ b/easybuild/easyblocks/e/esmf.py @@ -138,11 +138,11 @@ def make_module_extra(self): txt = super(EB_ESMF, self).make_module_extra() - python = get_software_version('Python') - if python: - if self.cfg['multi_deps'] and 'Python' in self.cfg['multi_deps']: - txt += self.module_generator.prepend_paths('EBPYTHONPREFIXES', []) - else: + if self.cfg['multi_deps'] and 'Python' in self.cfg['multi_deps']: + txt += self.module_generator.prepend_paths('EBPYTHONPREFIXES', []) + else: + python = get_software_version('Python') + if python: pyshortver = '.'.join(get_software_version('Python').split('.')[:2]) pythonpath = os.path.join('lib', 'python%s' % pyshortver, 'site-packages') txt += self.module_generator.prepend_paths('PYTHONPATH', [pythonpath]) From 74da80c9e5bb75fce31f9505bdc0c51d0c21a2fc Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Wed, 21 Sep 2022 14:10:14 +0000 Subject: [PATCH 13/54] actually fix EBPYTHONPREFIXES --- easybuild/easyblocks/e/esmf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/easyblocks/e/esmf.py b/easybuild/easyblocks/e/esmf.py index a5d09277c4..cc3986ca45 100644 --- a/easybuild/easyblocks/e/esmf.py +++ b/easybuild/easyblocks/e/esmf.py @@ -135,11 +135,10 @@ def install_step(self): def make_module_extra(self): """Add install path to PYTHONPATH or EBPYTHONPREFIXES""" - txt = super(EB_ESMF, self).make_module_extra() if self.cfg['multi_deps'] and 'Python' in self.cfg['multi_deps']: - txt += self.module_generator.prepend_paths('EBPYTHONPREFIXES', []) + txt += self.module_generator.prepend_paths('EBPYTHONPREFIXES', '') else: python = get_software_version('Python') if python: From 127a09bb8396bfb2965acb3679f49c5dff8c9d18 Mon Sep 17 00:00:00 2001 From: satishk Date: Wed, 21 Sep 2022 17:53:43 +0200 Subject: [PATCH 14/54] symlink points to CUDA folder not created for non GPU nodes. shutil throws an error in this --- easybuild/easyblocks/s/suitesparse.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/s/suitesparse.py b/easybuild/easyblocks/s/suitesparse.py index 807fb835eb..0648b1d8cd 100644 --- a/easybuild/easyblocks/s/suitesparse.py +++ b/easybuild/easyblocks/s/suitesparse.py @@ -42,7 +42,7 @@ from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import mkdir, write_file, adjust_permissions +from easybuild.tools.filetools import mkdir, write_file, adjust_permissions, copy_dir from easybuild.tools.modules import get_software_root from easybuild.tools.modules import get_software_libdir from easybuild.tools.systemtools import get_shared_lib_ext @@ -150,7 +150,10 @@ def install_step(self): dst = os.path.join(self.installdir, x) try: if os.path.isdir(src): - shutil.copytree(src, dst) + # symlink points to CUDA folder not + # created for non GPU nodes.shutil + # throws an error in this case. + copy_dir(src, dst, symlinks=True) # symlink # - dst/Lib to dst/lib # - dst/Include to dst/include From 861f0ac3fed151b2cf8b332b65929148fb4c1c2b Mon Sep 17 00:00:00 2001 From: satishk Date: Wed, 21 Sep 2022 18:07:01 +0200 Subject: [PATCH 15/54] Removed trailing white space. --- easybuild/easyblocks/s/suitesparse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/s/suitesparse.py b/easybuild/easyblocks/s/suitesparse.py index 0648b1d8cd..2cff1575ce 100644 --- a/easybuild/easyblocks/s/suitesparse.py +++ b/easybuild/easyblocks/s/suitesparse.py @@ -150,8 +150,8 @@ def install_step(self): dst = os.path.join(self.installdir, x) try: if os.path.isdir(src): - # symlink points to CUDA folder not - # created for non GPU nodes.shutil + # symlink points to CUDA folder that is + # not created for non GPU nodes. shutil # throws an error in this case. copy_dir(src, dst, symlinks=True) # symlink From 6ff70d1a7f511e6d7db1fa8393e7f4cc8117c865 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Mon, 26 Sep 2022 16:38:42 +0200 Subject: [PATCH 16/54] Remove docs and java and add Rocky support for ABAQUS --- easybuild/easyblocks/a/abaqus.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/easybuild/easyblocks/a/abaqus.py b/easybuild/easyblocks/a/abaqus.py index 91ba89d56d..fe8ed67a5a 100644 --- a/easybuild/easyblocks/a/abaqus.py +++ b/easybuild/easyblocks/a/abaqus.py @@ -44,6 +44,7 @@ from easybuild.tools.filetools import change_dir, symlink, write_file from easybuild.tools.modules import get_software_root from easybuild.tools.run import run_cmd_qa +from easybuild.tools.systemtools import get_os_name from easybuild.tools.py2vs3 import OrderedDict @@ -78,6 +79,9 @@ def extract_step(self): def configure_step(self): """Configure ABAQUS installation.""" if LooseVersion(self.version) >= LooseVersion('2016'): + # Rocky Linux isn't recognized; faking it as RHEL + if get_os_name() == 'Rocky Linux': + setvar('DISTRIB_ID', 'RedHatEnterpriseServer') # skip checking of Linux version setvar('DSY_Force_OS', 'linux_a64') # skip checking of license server @@ -126,10 +130,9 @@ def install_step(self): # rather than a regular dictionary (where there's no guarantee on key order in general) std_qa = OrderedDict() - # Enable Extended Product Documentation - std_qa[selectionstr % (r"\[ \]", "Extended Product Documentation")] = "%(nr)s" - # Enable Abaqus CAE (docs) - std_qa[selectionstr % (r"\[ \]", "Abaqus CAE")] = "%(nr)s" + # Disable Extended Product Documentation (because it has a troublesome Java dependency) + std_qa[selectionstr % (r"\[*\]", "Extended Product Documentation")] = "%(nr)s" + installed_docs = False # enable all ABAQUS components std_qa[selectionstr % (r"\[ \]", "Abaqus.*")] = "%(nr)s" @@ -146,9 +149,9 @@ def install_step(self): # Disable/enable Tosca if self.cfg['with_tosca']: - std_qa[selectionstr % (r"\[\ \]", "Tosca")] = "%(nr)s" + std_qa[selectionstr % (r"\[\ \]", "Tosca.*")] = "%(nr)s" else: - std_qa[selectionstr % (r"\[\*\]", "Tosca")] = "%(nr)s" + std_qa[selectionstr % (r"\[\*\]", "Tosca.*")] = "%(nr)s" # disable CloudView std_qa[r"(?P[0-9]+) \[X\] Search using CloudView\nEnter selection:"] = '%(cloudview)s\n\n' @@ -170,7 +173,7 @@ def install_step(self): cae_subdir = os.path.join(self.installdir, 'cae') sim_subdir = os.path.join(self.installdir, 'sim') std_qa[r"Default.*SIMULIA/EstProducts.*:"] = cae_subdir - std_qa[r"SIMULIA[0-9]*doc.*:"] = os.path.join(self.installdir, 'doc') + std_qa[r"SIMULIA[0-9]*doc.*:"] = os.path.join(self.installdir, 'doc') # if docs are installed std_qa[r"SimulationServices.*:"] = sim_subdir std_qa[r"Choose the CODE installation directory.*:\n.*\n\n.*:"] = sim_subdir std_qa[r"SIMULIA/CAE.*:"] = cae_subdir @@ -197,13 +200,13 @@ def install_step(self): std_qa[r"Please choose an action:"] = '1' - if LooseVersion(self.version) >= LooseVersion('2022'): + if LooseVersion(self.version) >= LooseVersion('2022') and installed_docs: java_root = get_software_root('Java') if java_root: std_qa[r"Please enter .*Java Runtime Environment.* path.(\n.*)+Default \[\]:"] = java_root std_qa[r"Please enter .*Java Runtime Environment.* path.(\n.*)+Default \[.+\]:"] = '' else: - raise EasyBuildError("Java is a required dependency for ABAQUS versions >= 2022, but it is missing") + raise EasyBuildError("Java is a required dependency for ABAQUS docs versions >= 2022, but it is missing") # Continue std_qa[nextstr] = '' From 0df3f3acecca82bf4ae855d55b1428d5d6e80e09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Mon, 26 Sep 2022 16:41:03 +0200 Subject: [PATCH 17/54] Fix long line --- easybuild/easyblocks/a/abaqus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/a/abaqus.py b/easybuild/easyblocks/a/abaqus.py index fe8ed67a5a..eff6c668c3 100644 --- a/easybuild/easyblocks/a/abaqus.py +++ b/easybuild/easyblocks/a/abaqus.py @@ -206,7 +206,7 @@ def install_step(self): std_qa[r"Please enter .*Java Runtime Environment.* path.(\n.*)+Default \[\]:"] = java_root std_qa[r"Please enter .*Java Runtime Environment.* path.(\n.*)+Default \[.+\]:"] = '' else: - raise EasyBuildError("Java is a required dependency for ABAQUS docs versions >= 2022, but it is missing") + raise EasyBuildError("Java is required for ABAQUS docs versions >= 2022, but it is missing") # Continue std_qa[nextstr] = '' From 5e9c2538a18b9a760274a2bac21f4508b903cbcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Mon, 26 Sep 2022 17:06:13 +0200 Subject: [PATCH 18/54] Remove doc sanity check --- easybuild/easyblocks/a/abaqus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/a/abaqus.py b/easybuild/easyblocks/a/abaqus.py index eff6c668c3..c8bc797e0b 100644 --- a/easybuild/easyblocks/a/abaqus.py +++ b/easybuild/easyblocks/a/abaqus.py @@ -333,7 +333,7 @@ def sanity_check_step(self): custom_commands = [] if LooseVersion(self.version) >= LooseVersion('2016'): - custom_paths['dirs'].extend(['cae', 'Commands', 'doc']) + custom_paths['dirs'].extend(['cae', 'Commands']) if LooseVersion(self.version) < LooseVersion('2020'): custom_paths['dirs'].extend(['sim']) # 'all' also check license server, but lmstat is usually not available From cf0eeaf99d8fd045d23e07876103f6788d544e44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Wed, 28 Sep 2022 14:07:49 +0200 Subject: [PATCH 19/54] Include AlmaLinux in abaqus block as well Co-authored-by: Sam Moors --- easybuild/easyblocks/a/abaqus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/a/abaqus.py b/easybuild/easyblocks/a/abaqus.py index c8bc797e0b..be0013795b 100644 --- a/easybuild/easyblocks/a/abaqus.py +++ b/easybuild/easyblocks/a/abaqus.py @@ -80,7 +80,7 @@ def configure_step(self): """Configure ABAQUS installation.""" if LooseVersion(self.version) >= LooseVersion('2016'): # Rocky Linux isn't recognized; faking it as RHEL - if get_os_name() == 'Rocky Linux': + if get_os_name() in ['Rocky Linux', 'AlmaLinux']: setvar('DISTRIB_ID', 'RedHatEnterpriseServer') # skip checking of Linux version setvar('DSY_Force_OS', 'linux_a64') From db706d7039b8c7c3cff9959886e3e6a087ca2b7f Mon Sep 17 00:00:00 2001 From: casparl Date: Thu, 29 Sep 2022 16:54:06 +0200 Subject: [PATCH 20/54] Correctly count the number of failing tests, not failing test suites. Improve error reporting to provide more insight into which test suites fail --- easybuild/easyblocks/p/pytorch.py | 73 ++++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 11 deletions(-) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index 36aa710cb2..f200234209 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -264,17 +264,68 @@ def test_step(self): (tests_out, tests_ec) = super(EB_PyTorch, self).test_step(return_output_ec=True) ran_tests_hits = re.findall(r"^Ran (?P[0-9]+) tests in", tests_out, re.M) - test_cnt = sum(int(hit) for hit in ran_tests_hits) - - failed_tests = nub(re.findall(r"^(?P.*) failed!(?: Received signal: \w+)?\s*$", tests_out, re.M)) - failed_test_cnt = len(failed_tests) - - if failed_test_cnt: - max_failed_tests = self.cfg['max_failed_tests'] - - test_or_tests = 'tests' if failed_test_cnt > 1 else 'test' - msg = "%d %s (out of %d) failed:\n" % (failed_test_cnt, test_or_tests, test_cnt) - msg += '\n'.join('* %s' % t for t in sorted(failed_tests)) + test_cnt = 0 + for hit in ran_tests_hits: + test_cnt += int(hit) + + # Get matches to create clear summary report, greps for patterns like: + # FAILED (errors=10, skipped=190, expected failures=6) + # test_fx failed! + summary_matches=re.findall(r"^Ran (?P[0-9]+) tests.*$\n\nFAILED \((?P.*)\)$\n(?:^(?:(?!failed!).)*$\n)*(?P.*) failed!$", tests_out, re.M) + + # Get matches to create clear summary report, greps for patterns like: + # ===================== 2 failed, 128 passed, 2 skipped, 2 warnings in 3.43s ===================== + summary_matches_pattern2=re.findall(r"^=+ (?P.*) in [0-9]+\.*[0-9]*[a-zA-Z]* =+$\n(?P.*) failed!$", tests_out, re.M) + + # Count failures and errors + def get_count_for_pattern(regex, text): + match = re.findall(regex, text, re.M) + if len(match) == 0: + return 0 + elif len(match) == 1: + return int(match[0]) + elif len(match) > 1: + # Shouldn't happen, throw warning (we don't need to error on a failed grep, the build could very well be fine) + # TODO: THROW WARNING + print("WARNING") + + failure_cnt=0 + error_cnt=0 + # Loop over first pattern to count failures/errors: + for summary in summary_matches: + print("Summary: %s" % summary[1]) + failures = get_count_for_pattern(r"^.*(?[0-9]+).*$", summary[1]) + print("Failures: %s" % failures) + failure_cnt += failures + errs = get_count_for_pattern(r"^.*errors=(?P[0-9]+).*$", summary[1]) + print("Errors: %s" % errs) + error_cnt += errs + + # Loop over the second pattern to count failures/errors + for summary in summary_matches_pattern2: + print("Summary: %s" % summary[0]) + failures = get_count_for_pattern(r"^.*(?P[0-9]+) failed.*$", summary[0]) + print("Failures: %s" % failures) + failure_cnt += failures + errs = get_count_for_pattern(r"^.*(?P[0-9]+) error.*$", summary[0]) + print("Errors: %s" % errs) + error_cnt += errs + + print("Total failures: %s" % failure_cnt) + print("Total errors: %s" % error_cnt) + + # Calculate total number of unsuccesful tests + failed_test_cnt = failure_cnt + error_cnt + + if failed_test_cnt > 0: + + failure_or_failures = 'failures' if failure_cnt > 1 else 'failure' + error_or_errors = 'errors' if error_cnt > 1 else 'error' + msg = "%d test %s, %d test %s (out of %d):\n" % (failure_cnt, failure_or_failures, error_cnt, error_or_errors, test_cnt) + for summary in summary_matches_pattern2: + msg += "{test_suite} ({failure_summary})\n".format(test_suite=summary[1], failure_summary=summary[0]) + for summary in summary_matches: + msg += "{test_suite} ({total} total tests, {failure_summary})\n".format(test_suite=summary[2], total=summary[0], failure_summary=summary[1]) if max_failed_tests == 0: raise EasyBuildError(msg) From 7c94d972d33eaf530ab8a8735bc31da0b0c84fe9 Mon Sep 17 00:00:00 2001 From: casparl Date: Thu, 29 Sep 2022 17:07:22 +0200 Subject: [PATCH 21/54] Fix formatting issues --- easybuild/easyblocks/p/pytorch.py | 49 +++++++++++++++---------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index f200234209..64bd3e88cb 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -271,12 +271,14 @@ def test_step(self): # Get matches to create clear summary report, greps for patterns like: # FAILED (errors=10, skipped=190, expected failures=6) # test_fx failed! - summary_matches=re.findall(r"^Ran (?P[0-9]+) tests.*$\n\nFAILED \((?P.*)\)$\n(?:^(?:(?!failed!).)*$\n)*(?P.*) failed!$", tests_out, re.M) - + regex = r"^Ran (?P[0-9]+) tests.*$\n\nFAILED \((?P.*)\)$\n(?:^(?:(?!failed!).)*$\n)*(?P.*) failed!$" + summary_matches = re.findall(regex, tests_out, re.M) + # Get matches to create clear summary report, greps for patterns like: # ===================== 2 failed, 128 passed, 2 skipped, 2 warnings in 3.43s ===================== - summary_matches_pattern2=re.findall(r"^=+ (?P.*) in [0-9]+\.*[0-9]*[a-zA-Z]* =+$\n(?P.*) failed!$", tests_out, re.M) - + regex = r"^=+ (?P.*) in [0-9]+\.*[0-9]*[a-zA-Z]* =+$\n(?P.*) failed!$" + summary_matches_pattern2 = re.findall(regex, tests_out, re.M) + # Count failures and errors def get_count_for_pattern(regex, text): match = re.findall(regex, text, re.M) @@ -285,47 +287,44 @@ def get_count_for_pattern(regex, text): elif len(match) == 1: return int(match[0]) elif len(match) > 1: - # Shouldn't happen, throw warning (we don't need to error on a failed grep, the build could very well be fine) - # TODO: THROW WARNING - print("WARNING") - - failure_cnt=0 - error_cnt=0 + # Shouldn't happen, but means something went wrong with the regular expressions. + # Throw warning, as the build might be fine, no need to error on this. + warn_msg = "Error in counting the number of test failures in the output of the PyTorch test suite.\n" + warn_msg += "Please check the EasyBuild log to verify the number of failures (if any) was acceptable." + print_warning(warning_msg) + + failure_cnt = 0 + error_cnt = 0 # Loop over first pattern to count failures/errors: for summary in summary_matches: - print("Summary: %s" % summary[1]) failures = get_count_for_pattern(r"^.*(?[0-9]+).*$", summary[1]) - print("Failures: %s" % failures) failure_cnt += failures errs = get_count_for_pattern(r"^.*errors=(?P[0-9]+).*$", summary[1]) - print("Errors: %s" % errs) error_cnt += errs - + # Loop over the second pattern to count failures/errors for summary in summary_matches_pattern2: - print("Summary: %s" % summary[0]) failures = get_count_for_pattern(r"^.*(?P[0-9]+) failed.*$", summary[0]) - print("Failures: %s" % failures) failure_cnt += failures errs = get_count_for_pattern(r"^.*(?P[0-9]+) error.*$", summary[0]) - print("Errors: %s" % errs) error_cnt += errs - - print("Total failures: %s" % failure_cnt) - print("Total errors: %s" % error_cnt) - + # Calculate total number of unsuccesful tests failed_test_cnt = failure_cnt + error_cnt - + if failed_test_cnt > 0: - + failure_or_failures = 'failures' if failure_cnt > 1 else 'failure' error_or_errors = 'errors' if error_cnt > 1 else 'error' - msg = "%d test %s, %d test %s (out of %d):\n" % (failure_cnt, failure_or_failures, error_cnt, error_or_errors, test_cnt) + msg = "%d test %s, %d test %s (out of %d):\n" % ( + failure_cnt, failure_or_failures, error_cnt, error_or_errors, test_cnt + ) for summary in summary_matches_pattern2: msg += "{test_suite} ({failure_summary})\n".format(test_suite=summary[1], failure_summary=summary[0]) for summary in summary_matches: - msg += "{test_suite} ({total} total tests, {failure_summary})\n".format(test_suite=summary[2], total=summary[0], failure_summary=summary[1]) + msg += "{test_suite} ({total} total tests, {failure_summary})\n".format( + test_suite=summary[2], total=summary[0], failure_summary=summary[1] + ) if max_failed_tests == 0: raise EasyBuildError(msg) From 9ea72be727282ee509a1c007ba3252319a19b68a Mon Sep 17 00:00:00 2001 From: casparl Date: Thu, 29 Sep 2022 17:08:05 +0200 Subject: [PATCH 22/54] Fix typo --- easybuild/easyblocks/p/pytorch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index 64bd3e88cb..6384bdda07 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -291,7 +291,7 @@ def get_count_for_pattern(regex, text): # Throw warning, as the build might be fine, no need to error on this. warn_msg = "Error in counting the number of test failures in the output of the PyTorch test suite.\n" warn_msg += "Please check the EasyBuild log to verify the number of failures (if any) was acceptable." - print_warning(warning_msg) + print_warning(warn_msg) failure_cnt = 0 error_cnt = 0 From 8b96b6978ad59e4d7ca43482b57644f2f24ef99f Mon Sep 17 00:00:00 2001 From: casparl Date: Thu, 29 Sep 2022 17:09:53 +0200 Subject: [PATCH 23/54] Further formatting fixes --- easybuild/easyblocks/p/pytorch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index 6384bdda07..0b407190fc 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -271,9 +271,9 @@ def test_step(self): # Get matches to create clear summary report, greps for patterns like: # FAILED (errors=10, skipped=190, expected failures=6) # test_fx failed! - regex = r"^Ran (?P[0-9]+) tests.*$\n\nFAILED \((?P.*)\)$\n(?:^(?:(?!failed!).)*$\n)*(?P.*) failed!$" + regex = r"^Ran (?P[0-9]+) tests.*$\n\nFAILED \((?P.*)\)$\n(?:^(?:(?!failed!).)*$\n)*(?P.*) failed!$" # noqa: W503 summary_matches = re.findall(regex, tests_out, re.M) - + # Get matches to create clear summary report, greps for patterns like: # ===================== 2 failed, 128 passed, 2 skipped, 2 warnings in 3.43s ===================== regex = r"^=+ (?P.*) in [0-9]+\.*[0-9]*[a-zA-Z]* =+$\n(?P.*) failed!$" From 9be147e2527f631e00c8166c71fd0d71f75e58e6 Mon Sep 17 00:00:00 2001 From: casparl Date: Thu, 29 Sep 2022 17:11:31 +0200 Subject: [PATCH 24/54] Further formatting fixes --- easybuild/easyblocks/p/pytorch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index 0b407190fc..0045910cd5 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -271,7 +271,7 @@ def test_step(self): # Get matches to create clear summary report, greps for patterns like: # FAILED (errors=10, skipped=190, expected failures=6) # test_fx failed! - regex = r"^Ran (?P[0-9]+) tests.*$\n\nFAILED \((?P.*)\)$\n(?:^(?:(?!failed!).)*$\n)*(?P.*) failed!$" # noqa: W503 + regex = r"^Ran (?P[0-9]+) tests.*$\n\nFAILED \((?P.*)\)$\n(?:^(?:(?!failed!).)*$\n)*(?P.*) failed!$" # noqa: W501 summary_matches = re.findall(regex, tests_out, re.M) # Get matches to create clear summary report, greps for patterns like: From 6bab8b77c598e5409c39f45e368213579def2860 Mon Sep 17 00:00:00 2001 From: casparl Date: Fri, 30 Sep 2022 10:32:14 +0200 Subject: [PATCH 25/54] Forgot to assign max_failed_tests variable --- easybuild/easyblocks/p/pytorch.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index 0045910cd5..df29fbfd02 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -313,6 +313,7 @@ def get_count_for_pattern(regex, text): failed_test_cnt = failure_cnt + error_cnt if failed_test_cnt > 0: + max_failed_tests = self.cfg['max_failed_tests'] failure_or_failures = 'failures' if failure_cnt > 1 else 'failure' error_or_errors = 'errors' if error_cnt > 1 else 'error' From adbd9115be9b7bed8bd36d7ec252c4ddcba6f1cc Mon Sep 17 00:00:00 2001 From: casparl Date: Fri, 30 Sep 2022 10:36:27 +0200 Subject: [PATCH 26/54] Removed include that is no longer needed --- easybuild/easyblocks/p/pytorch.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index df29fbfd02..4b2a993ec5 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -40,7 +40,6 @@ from easybuild.tools.filetools import symlink, apply_regex_substitutions from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.systemtools import POWER, get_cpu_architecture -from easybuild.tools.utilities import nub class EB_PyTorch(PythonPackage): From 1e6b4a7975f29ee46e47accfeb4c24b9f76caac9 Mon Sep 17 00:00:00 2001 From: casparl Date: Fri, 30 Sep 2022 10:41:13 +0200 Subject: [PATCH 27/54] Accept longer line, don't want to break a single regex in the middle --- easybuild/easyblocks/p/pytorch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index 4b2a993ec5..57a9ea65c0 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -270,7 +270,7 @@ def test_step(self): # Get matches to create clear summary report, greps for patterns like: # FAILED (errors=10, skipped=190, expected failures=6) # test_fx failed! - regex = r"^Ran (?P[0-9]+) tests.*$\n\nFAILED \((?P.*)\)$\n(?:^(?:(?!failed!).)*$\n)*(?P.*) failed!$" # noqa: W501 + regex = r"^Ran (?P[0-9]+) tests.*$\n\nFAILED \((?P.*)\)$\n(?:^(?:(?!failed!).)*$\n)*(?P.*) failed!$" # noqa: E501 summary_matches = re.findall(regex, tests_out, re.M) # Get matches to create clear summary report, greps for patterns like: From dab687999313a8e448e4735458e211bbb44d45f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Fri, 30 Sep 2022 15:49:29 +0200 Subject: [PATCH 28/54] Update easybuild/easyblocks/a/abaqus.py Co-authored-by: ocaisa --- easybuild/easyblocks/a/abaqus.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/a/abaqus.py b/easybuild/easyblocks/a/abaqus.py index be0013795b..56d15b93be 100644 --- a/easybuild/easyblocks/a/abaqus.py +++ b/easybuild/easyblocks/a/abaqus.py @@ -130,9 +130,9 @@ def install_step(self): # rather than a regular dictionary (where there's no guarantee on key order in general) std_qa = OrderedDict() - # Disable Extended Product Documentation (because it has a troublesome Java dependency) + # Disable Extended Product Documentation because it has a troublesome Java dependency std_qa[selectionstr % (r"\[*\]", "Extended Product Documentation")] = "%(nr)s" - installed_docs = False + installed_docs = False # hard disabled, previous support was actually incomplete # enable all ABAQUS components std_qa[selectionstr % (r"\[ \]", "Abaqus.*")] = "%(nr)s" From ec7912624400649bfdcc491fed23d105eafcf58c Mon Sep 17 00:00:00 2001 From: Viktor Rehnberg <35767167+VRehnberg@users.noreply.github.com> Date: Fri, 30 Sep 2022 16:39:03 +0200 Subject: [PATCH 29/54] Fix typo TensorFlow -> PyTorch --- easybuild/easyblocks/p/pytorch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index 36aa710cb2..fcf264095d 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -44,7 +44,7 @@ class EB_PyTorch(PythonPackage): - """Support for building/installing TensorFlow.""" + """Support for building/installing PyTorch.""" @staticmethod def extra_options(): From 3ac00ffb34ad1c47153ce494ab6f8bf1a4536036 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 30 Sep 2022 18:00:02 +0200 Subject: [PATCH 30/54] Fix typo Co-authored-by: ocaisa --- easybuild/easyblocks/c/cudacompat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/c/cudacompat.py b/easybuild/easyblocks/c/cudacompat.py index 8aea98f23b..5feaf25964 100644 --- a/easybuild/easyblocks/c/cudacompat.py +++ b/easybuild/easyblocks/c/cudacompat.py @@ -158,7 +158,7 @@ def test_step(self): '%s %s. Supported drivers: %s', driver_version, self.name, self.version, compatible_driver_versions) elif LooseVersion(driver_version) < self.compatible_driver_version_map[driver_version_major]: - raise EasyBuildError('The installed CUDA driver %s is to old for %s %s. Supported drivers: %s', + raise EasyBuildError('The installed CUDA driver %s is too old for %s %s. Supported drivers: %s', driver_version, self.name, self.version, compatible_driver_versions) else: self.log.info('The installed CUDA driver %s appears to be supported.', driver_version) From e1c4517fa1f543d974e8e3620537ee4bc392ba1b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 4 Oct 2022 16:15:34 +0200 Subject: [PATCH 31/54] Also log to easybuild log file. The message contains a much easier to read overview of failing tests etc than what is logged in the logfile by default. --- easybuild/easyblocks/p/pytorch.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index 57a9ea65c0..6dc8492b9a 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -337,8 +337,12 @@ def get_count_for_pattern(regex, text): "are known to be flaky, or do not affect your intended usage of PyTorch.", "In case of doubt, reach out to the EasyBuild community (via GitHub, Slack, or mailing list).", ]) + # Print to console, the user should really be aware that we are accepting failing tests here... print_warning(msg) + # Also log this warning in the file log + self.log.warning(msg) + if failed_test_cnt > max_failed_tests: raise EasyBuildError("Too many failed tests (%d), maximum allowed is %d", failed_test_cnt, max_failed_tests) From 05b215d12f01deeeb5c8affe510cfa2b6d960e75 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 5 Oct 2022 17:19:00 +0200 Subject: [PATCH 32/54] Improve error (sorted versions, show min. compatible version) --- easybuild/easyblocks/c/cudacompat.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/c/cudacompat.py b/easybuild/easyblocks/c/cudacompat.py index 5feaf25964..6d8563b795 100644 --- a/easybuild/easyblocks/c/cudacompat.py +++ b/easybuild/easyblocks/c/cudacompat.py @@ -152,14 +152,18 @@ def test_step(self): print_warning('Failed to determine the CUDA driver version, so skipping the compatibility check!') else: driver_version_major = int(driver_version.split('.', 1)[0]) - compatible_driver_versions = ', '.join(self.compatible_driver_version_map.values()) - if driver_version_major not in self.compatible_driver_version_map: + compatible_driver_versions = ', '.join(sorted(self.compatible_driver_version_map.values())) + try: + min_required_version = self.compatible_driver_version_map[driver_version_major] + except KeyError: raise EasyBuildError('The installed CUDA driver %s is not a supported branch/major version for ' '%s %s. Supported drivers: %s', driver_version, self.name, self.version, compatible_driver_versions) - elif LooseVersion(driver_version) < self.compatible_driver_version_map[driver_version_major]: - raise EasyBuildError('The installed CUDA driver %s is too old for %s %s. Supported drivers: %s', - driver_version, self.name, self.version, compatible_driver_versions) + if LooseVersion(driver_version) < min_required_version: + raise EasyBuildError('The installed CUDA driver %s is to old for %s %s, ' + 'need at least %s. Supported drivers: %s', + driver_version, self.name, self.version, + min_required_version, compatible_driver_versions) else: self.log.info('The installed CUDA driver %s appears to be supported.', driver_version) From fd0caab5f58401fae490555390a0a67d800621dc Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 6 Oct 2022 09:39:32 +0200 Subject: [PATCH 33/54] Make the test output from PythonPackage less verbose When the EasyBlock wants to handle the test output itself it doesn't make sense to search it for potential errors using the generic RegExp. This applies e.g. to PyTorch and TensorFlow which search for test failures themselves. So set `regexp=False` to avoid making the log much harder to read due to duplicated and, more annoyingly, partial test (fail) output. --- easybuild/easyblocks/generic/pythonpackage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index f84b601ef0..77c493b56e 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -684,8 +684,8 @@ def test_step(self, return_output_ec=False): ]) if return_output_ec: - (out, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False) - # need to log seperately, since log_all and log_ok need to be false to retreive out and ec + (out, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, regexp=False) + # need to log seperately, since log_all and log_ok need to be false to retrieve out and ec self.log.info("cmd '%s' exited with exit code %s and output:\n%s", cmd, ec, out) else: run_cmd(cmd, log_all=True, simple=True) From 502f3085f0e86d27e8a03f1e276df0e187c84ace Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Thu, 6 Oct 2022 15:05:20 +0000 Subject: [PATCH 34/54] handle iterative builds with MakeCP EasyBlock --- easybuild/easyblocks/generic/makecp.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/makecp.py b/easybuild/easyblocks/generic/makecp.py index f8603445b7..55a223e755 100644 --- a/easybuild/easyblocks/generic/makecp.py +++ b/easybuild/easyblocks/generic/makecp.py @@ -26,6 +26,7 @@ @author: George Tsouloupas (The Cyprus Institute) @author: Fotis Georgatos (Uni.Lu, NTUA) @author: Kenneth Hoste (Ghent University) +@author: Maxime Boissonneault (Digital Research Alliance of Canada, Universite Laval) """ import os import glob @@ -70,6 +71,10 @@ def install_step(self): files_to_copy = self.cfg.get('files_to_copy') or [] self.log.debug("Starting install_step with files_to_copy: %s", files_to_copy) + + # if buildopts is a list, this is an iterative build, and directories will be copied multiple times + dirs_exist_ok = isinstance(self.cfg.get('buildopts'), list) + for fil in files_to_copy: if isinstance(fil, tuple): # ([src1, src2], targetdir) @@ -128,6 +133,6 @@ def install_step(self): elif os.path.isdir(filepath): self.log.debug("Copying directory %s to %s", filepath, target) fulltarget = os.path.join(target, os.path.basename(filepath)) - copy_dir(filepath, fulltarget, symlinks=self.cfg['keepsymlinks']) + copy_dir(filepath, fulltarget, symlinks=self.cfg['keepsymlinks'], dirs_exist_ok=dirs_exist_ok) else: raise EasyBuildError("Can't copy non-existing path %s to %s", filepath, target) From c8344f9040146b962371754506c92cdb03a3ac9c Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Thu, 6 Oct 2022 15:30:33 +0000 Subject: [PATCH 35/54] properly check if iterative build --- easybuild/easyblocks/generic/makecp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/makecp.py b/easybuild/easyblocks/generic/makecp.py index 55a223e755..02b2ad12d0 100644 --- a/easybuild/easyblocks/generic/makecp.py +++ b/easybuild/easyblocks/generic/makecp.py @@ -72,8 +72,8 @@ def install_step(self): files_to_copy = self.cfg.get('files_to_copy') or [] self.log.debug("Starting install_step with files_to_copy: %s", files_to_copy) - # if buildopts is a list, this is an iterative build, and directories will be copied multiple times - dirs_exist_ok = isinstance(self.cfg.get('buildopts'), list) + # if this is an iterative build directories will be copied multiple times + dirs_exist_ok = True if self.iter_opts else False for fil in files_to_copy: if isinstance(fil, tuple): From c693551347b4972cff0343be1340675d31f8c513 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 14 Oct 2022 11:26:13 +0200 Subject: [PATCH 36/54] enhance OpenBLAS easyblock to support running LAPACK test suite + checking how many tests fail --- easybuild/easyblocks/o/openblas.py | 63 +++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/o/openblas.py b/easybuild/easyblocks/o/openblas.py index 4973fa0df1..be12dbd08a 100644 --- a/easybuild/easyblocks/o/openblas.py +++ b/easybuild/easyblocks/o/openblas.py @@ -3,21 +3,40 @@ @author: Andrew Edmondson (University of Birmingham) @author: Alex Domingo (Vrije Universiteit Brussel) +@author: Kenneth Hoste (Ghent University) """ import os +import re from distutils.version import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake +from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.systemtools import POWER, get_cpu_architecture, get_shared_lib_ext -from easybuild.tools.build_log import print_warning +from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import ERROR from easybuild.tools.run import run_cmd, check_log_for_errors +LAPACK_TEST_TARGET = 'lapack-test' TARGET = 'TARGET' class EB_OpenBLAS(ConfigureMake): """Support for building/installing OpenBLAS.""" + @staticmethod + def extra_options(): + """Custom easyconfig parameters for OpenBLAS easyblock.""" + extra_vars = { + 'max_failing_lapack_tests_num_errors': [0, "Maximum number of LAPACK tests failing " + "due to numerical errors", CUSTOM], + 'max_failing_lapack_tests_other_errors': [0, "Maximum number of LAPACK tests failing " + "due to non-numerical errors", CUSTOM], + 'run_lapack_tests': [False, "Run LAPACK tests during test step, " + "and check whether failing tests exceeds threshold", CUSTOM], + } + + return ConfigureMake.extra_options(extra_vars) + + def configure_step(self): """ set up some options - but no configure command to run""" @@ -72,10 +91,47 @@ def build_step(self): cmd = ' '.join([self.cfg['prebuildopts'], makecmd, ' '.join(build_parts), self.cfg['buildopts']]) run_cmd(cmd, log_all=True, simple=True) + def check_lapack_test_results(self, test_output): + """Check output of OpenBLAS' LAPACK test suite ('make lapack-test').""" + + # example: + # --> LAPACK TESTING SUMMARY <-- + # SUMMARY nb test run numerical error other error + # ================ =========== ================= ================ + # ... + # --> ALL PRECISIONS 4116982 4172 (0.101%) 0 (0.000%) + test_summary_pattern = r'\s+'.join([ + r"^--> ALL PRECISIONS", + r"(?P[0-9]+)", + r"(?P[0-9]+)\s+\([0-9.]+\%\)", + r"(?P[0-9]+)\s+\([0-9.]+\%\)", + ]) + regex = re.compile(test_summary_pattern, re.M) + res = regex.search(test_output) + if res: + (tot_cnt, fail_cnt_num_errors, fail_cnt_other_errors) = [int(x) for x in res.groups()] + msg = "%d LAPACK tests run - %d failed due to numerical errors - %d failed due to other errors" + self.log.info(msg, tot_cnt, fail_cnt_num_errors, fail_cnt_other_errors) + + if fail_cnt_other_errors > self.cfg['max_failing_lapack_tests_other_errors']: + raise EasyBuildError("Too many LAPACK tests failed due to non-numerical errors: %d (> %d)", + fail_cnt_other_errors, self.cfg['max_failing_lapack_tests_other_errors']) + + if fail_cnt_num_errors > self.cfg['max_failing_lapack_tests_num_errors']: + raise EasyBuildError("Too many LAPACK tests failed due to numerical errors: %d (> %d)", + fail_cnt_num_errors, self.cfg['max_failing_lapack_tests_num_errors']) + else: + raise EasyBuildError("Failed to find test summary using pattern '%s' in test output: %s", + test_summary_pattern, test_output) + def test_step(self): """ Mandatory test step plus optional runtest""" run_tests = ['tests'] + + if self.cfg['run_lapack_tests']: + run_tests += [LAPACK_TEST_TARGET] + if self.cfg['runtest']: run_tests += [self.cfg['runtest']] @@ -86,6 +142,11 @@ def test_step(self): # Raise an error if any test failed check_log_for_errors(out, [('FATAL ERROR', ERROR)]) + # check number of failing LAPACK tests more closely + if runtest == LAPACK_TEST_TARGET: + self.check_lapack_test_results(out) + + def sanity_check_step(self): """ Custom sanity check for OpenBLAS """ custom_paths = { From 25d2e06f2b8aa6be633a73a92cd2a0ca16fafff2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 14 Oct 2022 12:30:45 +0200 Subject: [PATCH 37/54] trivial style fixes for OpenBLAS easyblock --- easybuild/easyblocks/o/openblas.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/o/openblas.py b/easybuild/easyblocks/o/openblas.py index be12dbd08a..9ce5034ba0 100644 --- a/easybuild/easyblocks/o/openblas.py +++ b/easybuild/easyblocks/o/openblas.py @@ -27,16 +27,15 @@ def extra_options(): """Custom easyconfig parameters for OpenBLAS easyblock.""" extra_vars = { 'max_failing_lapack_tests_num_errors': [0, "Maximum number of LAPACK tests failing " - "due to numerical errors", CUSTOM], + "due to numerical errors", CUSTOM], 'max_failing_lapack_tests_other_errors': [0, "Maximum number of LAPACK tests failing " - "due to non-numerical errors", CUSTOM], + "due to non-numerical errors", CUSTOM], 'run_lapack_tests': [False, "Run LAPACK tests during test step, " "and check whether failing tests exceeds threshold", CUSTOM], } return ConfigureMake.extra_options(extra_vars) - def configure_step(self): """ set up some options - but no configure command to run""" @@ -146,7 +145,6 @@ def test_step(self): if runtest == LAPACK_TEST_TARGET: self.check_lapack_test_results(out) - def sanity_check_step(self): """ Custom sanity check for OpenBLAS """ custom_paths = { From a1426529608c7ca3d8fed4fb971d1dc9db121c0d Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 14 Oct 2022 17:40:31 +0200 Subject: [PATCH 38/54] Fix https://github.com/easybuilders/easybuild-easyblocks/issues/2802 using improved regex --- easybuild/easyblocks/p/pytorch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index ec4ee9ad97..42ed999e85 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -303,9 +303,9 @@ def get_count_for_pattern(regex, text): # Loop over the second pattern to count failures/errors for summary in summary_matches_pattern2: - failures = get_count_for_pattern(r"^.*(?P[0-9]+) failed.*$", summary[0]) + failures = get_count_for_pattern(r"^.*[^0-9](?P[0-9]+) failed.*$", summary[0]) failure_cnt += failures - errs = get_count_for_pattern(r"^.*(?P[0-9]+) error.*$", summary[0]) + errs = get_count_for_pattern(r"^.*[^0-9](?P[0-9]+) error.*$", summary[0]) error_cnt += errs # Calculate total number of unsuccesful tests From b8cdfdb6545b88641ff703dcd2e4fc7aed62a319 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 17 Oct 2022 13:39:49 +0200 Subject: [PATCH 39/54] Check `filter_env_vars` for $LD_LIBRARY_PATH --- easybuild/easyblocks/c/cudacompat.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/easyblocks/c/cudacompat.py b/easybuild/easyblocks/c/cudacompat.py index 6d8563b795..07e7bdb175 100644 --- a/easybuild/easyblocks/c/cudacompat.py +++ b/easybuild/easyblocks/c/cudacompat.py @@ -107,6 +107,9 @@ def prepare_step(self, *args, **kwargs): "Expected numeric major versions, got '%s'", compatible_driver_versions) else: self.compatible_driver_version_map = None + if 'LD_LIBRARY_PATH' in (build_option('filter_env_vars') or []): + raise EasyBuildError("This module relies on setting $LD_LIBRARY_PATH, " + "so you need to remove this variable from --filter-env-vars") super(EB_CUDAcompat, self).prepare_step(*args, **kwargs) def fetch_step(self, *args, **kwargs): From 018d685dc0849ba1de6b4fed873f9c11aa4a58de Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Mon, 17 Oct 2022 14:03:00 +0200 Subject: [PATCH 40/54] allow empty optarch string for opencv --- easybuild/easyblocks/o/opencv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/o/opencv.py b/easybuild/easyblocks/o/opencv.py index 7d18d0906e..035fd751a3 100644 --- a/easybuild/easyblocks/o/opencv.py +++ b/easybuild/easyblocks/o/opencv.py @@ -165,7 +165,7 @@ def configure_step(self): # see https://github.com/opencv/opencv/wiki/CPU-optimizations-build-options if self.toolchain.options.get('optarch') and 'CPU_BASELINE' not in self.cfg['configopts']: optarch = build_option('optarch') - if optarch is None: + if not optarch: # optimize for host arch (let OpenCV detect it) self.cfg.update('configopts', '-DCPU_BASELINE=DETECT') elif optarch == OPTARCH_GENERIC: From 7e08937d22adb49a0c37d4b41ede1f62e3c3f8f2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 17 Oct 2022 22:21:04 +0200 Subject: [PATCH 41/54] enhance EasyBuildMeta easyblock to auto-enable installing with pip rather than 'setup.py install' when using Python >= 3.6 and if pip >= 21.0 is available --- easybuild/easyblocks/e/easybuildmeta.py | 26 +++++++++++++++++-- easybuild/easyblocks/generic/pythonpackage.py | 8 +++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/e/easybuildmeta.py b/easybuild/easyblocks/e/easybuildmeta.py index 5949a181f8..86ceb75d73 100644 --- a/easybuild/easyblocks/e/easybuildmeta.py +++ b/easybuild/easyblocks/e/easybuildmeta.py @@ -33,9 +33,9 @@ import sys from distutils.version import LooseVersion -from easybuild.easyblocks.generic.pythonpackage import PythonPackage +from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_pip_version from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import read_file +from easybuild.tools.filetools import read_file, which from easybuild.tools.modules import get_software_root_env_var_name from easybuild.tools.py2vs3 import OrderedDict from easybuild.tools.utilities import flatten @@ -66,6 +66,28 @@ def __init__(self, *args, **kwargs): # consider setuptools first, in case it is listed as a sources self.easybuild_pkgs.insert(0, 'setuptools') + # opt-in to using pip for recent version of EasyBuild, if: + # - EasyBuild is being installed for Python >= 3.6; + # - pip is available, and recent enough (>= 21.0); + # - use_pip is not specified; + pyver = sys.version.split(' ')[0] + pip = which('pip') + self.log.info("Python version: %s - pip: %s", pyver, pip) + if sys.version_info >= (3, 6) and pip and self.cfg['use_pip'] is None: + # try to determine pip version, ignore any failures that occur while doing so; + # problems may occur due changes in environment ($PYTHONPATH, etc.) + pip_version = None + try: + pip_version = det_pip_version() + self.log.info("Found Python v%s + pip: %s", pyver, pip_version) + except Exception as err: + self.log.warning("Failed to determine pip version: %s", err) + + if pip_version and LooseVersion(pip_version) >= LooseVersion('21.0'): + self.log.info("Auto-enabling use of pip to install EasyBuild!") + self.cfg['use_pip'] = True + self.determine_install_command() + # Override this function since we want to respect the user choice for the python installation to use # (which can be influenced by EB_PYTHON and EB_INSTALLPYTHON) def prepare_python(self): diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 77c493b56e..5788c05bed 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -318,6 +318,12 @@ def __init__(self, *args, **kwargs): # determine install command self.use_setup_py = False + self.determine_install_command() + + def determine_install_command(self): + """ + Determine install command to use. + """ if self.cfg.get('use_pip', False) or self.cfg.get('use_pip_editable', False): self.install_cmd = PIP_INSTALL_CMD @@ -361,7 +367,7 @@ def __init__(self, *args, **kwargs): else: raise EasyBuildError("Installing zipped eggs requires using easy_install or pip") - self.log.debug("Using '%s' as install command", self.install_cmd) + self.log.info("Using '%s' as install command", self.install_cmd) def set_pylibdirs(self): """Set Python lib directory-related class variables.""" From fd77f29c259109fef9c794275e2bf24e93cee4b3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 17 Oct 2022 22:22:14 +0200 Subject: [PATCH 42/54] enhance EasyBuildMeta easyblock to fix easybuild-easyconfigs's setup.py so installation with setuptools >= 61.0 works --- easybuild/easyblocks/e/easybuildmeta.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/e/easybuildmeta.py b/easybuild/easyblocks/e/easybuildmeta.py index 86ceb75d73..abb7b89837 100644 --- a/easybuild/easyblocks/e/easybuildmeta.py +++ b/easybuild/easyblocks/e/easybuildmeta.py @@ -35,7 +35,7 @@ from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_pip_version from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import read_file, which +from easybuild.tools.filetools import apply_regex_substitutions, change_dir, read_file, which from easybuild.tools.modules import get_software_root_env_var_name from easybuild.tools.py2vs3 import OrderedDict from easybuild.tools.utilities import flatten @@ -114,6 +114,21 @@ def build_step(self): """No building for EasyBuild packages.""" pass + def fix_easyconfigs_setup_py_setuptools61(self): + """ + Patch setup.py of easybuild-easyconfigs package if needed to make sure that installation works + for recent setuptools versions (>= 61.0). + """ + # cfr. https://github.com/easybuilders/easybuild-easyconfigs/pull/15206 + cwd = os.getcwd() + regex = re.compile(r'packages=\[\]') + setup_py_txt = read_file('setup.py') + if regex.search(setup_py_txt) is None: + self.log.info("setup.py at %s needs to be fixed to install with setuptools >= 61.0", cwd) + apply_regex_substitutions('setup.py', [(r'^setup\(', 'setup(packages=[],')]) + else: + self.log.info("setup.py at %s does not need to be fixed to install with setuptools >= 61.0", cwd) + def install_step(self): """Install EasyBuild packages one by one.""" try: @@ -130,7 +145,11 @@ def install_step(self): else: self.log.info("Installing package %s", pkg) - os.chdir(os.path.join(self.builddir, seldirs[0])) + change_dir(os.path.join(self.builddir, seldirs[0])) + + if pkg == 'easybuild-easyconfigs': + self.fix_easyconfigs_setup_py_setuptools61() + super(EB_EasyBuildMeta, self).install_step() except OSError as err: From db889b62385bf991eaab45b8cb88d445c79782a1 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 18 Oct 2022 15:20:24 +0200 Subject: [PATCH 43/54] Prefer `python -m pip` over `pip` for Python packages --- easybuild/easyblocks/d/dm_reverb.py | 1 + easybuild/easyblocks/generic/pythonpackage.py | 36 ++++++++++--------- easybuild/easyblocks/t/tensorrt.py | 1 + 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/easybuild/easyblocks/d/dm_reverb.py b/easybuild/easyblocks/d/dm_reverb.py index 18012d98bb..60b012f7a2 100644 --- a/easybuild/easyblocks/d/dm_reverb.py +++ b/easybuild/easyblocks/d/dm_reverb.py @@ -121,6 +121,7 @@ def install_step(self, *args, **kwargs): 'installopts': self.cfg['installopts'], 'loc': whl_path, 'prefix': self.installdir, + 'python': self.python_cmd, } return super(EB_dm_minus_reverb, self).install_step(*args, **kwargs) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 77c493b56e..b95c043393 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -60,7 +60,7 @@ # not 'easy_install' deliberately, to avoid that pkg installations listed in easy-install.pth get preference # '.' is required at the end when using easy_install/pip in unpacked source dir EASY_INSTALL_TARGET = "easy_install" -PIP_INSTALL_CMD = "pip install --prefix=%(prefix)s %(installopts)s %(loc)s" +PIP_INSTALL_CMD = "%(python)s -m pip install --prefix=%(prefix)s %(installopts)s %(loc)s" SETUP_PY_INSTALL_CMD = "%(python)s setup.py %(install_target)s --prefix=%(prefix)s %(installopts)s" UNKNOWN = 'UNKNOWN' @@ -204,14 +204,14 @@ def get_pylibdirs(python_cmd): return all_pylibdirs -def det_pip_version(): - """Determine version of currently active 'pip' command.""" +def det_pip_version(python_cmd='python'): + """Determine version of currently active 'pip' module.""" pip_version = None log = fancylogger.getLogger('det_pip_version', fname=False) log.info("Determining pip version...") - out, _ = run_cmd("pip --version", verbose=False, simple=False, trace=False) + out, _ = run_cmd("%s -m pip --version" % python_cmd, verbose=False, simple=False, trace=False) pip_version_regex = re.compile('^pip ([0-9.]+)') res = pip_version_regex.search(out) @@ -247,8 +247,8 @@ def extra_options(extra_vars=None): "the pip version check. Enabled by default when pip_ignore_installed=True", CUSTOM], 'req_py_majver': [None, "Required major Python version (only relevant when using system Python)", CUSTOM], 'req_py_minver': [None, "Required minor Python version (only relevant when using system Python)", CUSTOM], - 'sanity_pip_check': [False, "Run 'pip check' to ensure all required Python packages are installed " - "and check for any package with an invalid (0.0.0) version.", CUSTOM], + 'sanity_pip_check': [False, "Run 'python -m pip check' to ensure all required Python packages are " + "installed and check for any package with an invalid (0.0.0) version.", CUSTOM], 'runtest': [True, "Run unit tests.", CUSTOM], # overrides default 'unpack_sources': [None, "Unpack sources prior to build/install. Defaults to 'True' except for whl files", CUSTOM], @@ -261,8 +261,8 @@ def extra_options(extra_vars=None): # see https://packaging.python.org/tutorials/installing-packages/#installing-setuptools-extras 'use_pip_extras': [None, "String with comma-separated list of 'extras' to install via pip", CUSTOM], 'use_pip_for_deps': [False, "Install dependencies using '%s'" % PIP_INSTALL_CMD, CUSTOM], - 'use_pip_requirement': [False, "Install using 'pip install --requirement'. The sources is expected " + - "to be the requirements file.", CUSTOM], + 'use_pip_requirement': [False, "Install using 'python -m pip install --requirement'. The sources is " + + "expected to be the requirements file.", CUSTOM], 'zipped_egg': [False, "Install as a zipped eggs", CUSTOM], }) # Use PYPI_SOURCE as the default for source_urls. @@ -468,7 +468,7 @@ def compose_install_command(self, prefix, extrapath=None, installopts=None): using_pip = self.install_cmd.startswith(PIP_INSTALL_CMD) if using_pip: - pip_version = det_pip_version() + pip_version = det_pip_version(python_cmd=self.python_cmd) if pip_version: # pip 8.x or newer required, because of --prefix option being used if LooseVersion(pip_version) >= LooseVersion('8.0'): @@ -834,8 +834,11 @@ def sanity_check_step(self, *args, **kwargs): kwargs.update({'exts_filter': exts_filter}) if self.cfg.get('sanity_pip_check', False): - pip_version = det_pip_version() + pip_version = det_pip_version(python_cmd=python_cmd) + if pip_version: + pip_check_command = "%s -m pip check" % python_cmd + if LooseVersion(pip_version) >= LooseVersion('9.0.0'): if not self.is_extension: @@ -846,11 +849,11 @@ def sanity_check_step(self, *args, **kwargs): pip_check_errors = [] - pip_check_msg, ec = run_cmd("pip check", log_ok=False) + pip_check_msg, ec = run_cmd(pip_check_command, log_ok=False) if ec: - pip_check_errors.append('`pip check` failed:\n%s' % pip_check_msg) + pip_check_errors.append('`%s` failed:\n%s' % (pip_check_command, pip_check_msg)) else: - self.log.info('`pip check` completed successfully') + self.log.info('`%s` completed successfully' % pip_check_command) # Also check for a common issue where the package version shows up as 0.0.0 often caused # by using setup.py as the installation method for a package which is released as a generic wheel @@ -869,8 +872,8 @@ def sanity_check_step(self, *args, **kwargs): version = next(pkg['version'] for pkg in pkgs if pkg['name'] == unversioned_package) except StopIteration: msg = ('Package %s in unversioned_packages was not found in the installed packages. ' - 'Check that the name from `pip list` is used which may be different than the ' - 'module name.' % unversioned_package) + 'Check that the name from `python -m pip list` is used which may be different ' + 'than the module name.' % unversioned_package) else: msg = ('Package %s in unversioned_packages has a version of %s which is valid. ' 'Please remove it from unversioned_packages.' % (unversioned_package, version)) @@ -894,7 +897,8 @@ def sanity_check_step(self, *args, **kwargs): if pip_check_errors: raise EasyBuildError('\n'.join(pip_check_errors)) else: - raise EasyBuildError("pip >= 9.0.0 is required for running 'pip check', found %s", pip_version) + raise EasyBuildError("pip >= 9.0.0 is required for running '%s', found %s", (pip_check_command, + pip_version)) else: raise EasyBuildError("Failed to determine pip version!") diff --git a/easybuild/easyblocks/t/tensorrt.py b/easybuild/easyblocks/t/tensorrt.py index 3d5489b13c..e438658803 100644 --- a/easybuild/easyblocks/t/tensorrt.py +++ b/easybuild/easyblocks/t/tensorrt.py @@ -108,6 +108,7 @@ def extensions_step(self): 'installopts': self.cfg['installopts'], 'loc': whl_paths[0], 'prefix': self.installdir, + 'python': self.python_cmd, } # Use --no-deps to prevent pip from downloading & installing From 9fd3361a10f45d4bbb5fa417d5cb73326e8ebe63 Mon Sep 17 00:00:00 2001 From: casparl Date: Tue, 18 Oct 2022 15:32:41 +0200 Subject: [PATCH 44/54] Add mamba EasyBlock --- easybuild/easyblocks/m/mamba.py | 52 +++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 easybuild/easyblocks/m/mamba.py diff --git a/easybuild/easyblocks/m/mamba.py b/easybuild/easyblocks/m/mamba.py new file mode 100644 index 0000000000..cf65e3e62a --- /dev/null +++ b/easybuild/easyblocks/m/mamba.py @@ -0,0 +1,52 @@ +## +# Copyright 2009-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for building and installing Mamba, implemented as an easyblock + +@author: Caspar van Leeuwen (SURF) +@author: Kenneth Hoste (HPC-UGent) +""" + +import os +import stat + +from easybuild.easyblocks.a.anaconda import EB_Anaconda +from easybuild.tools.filetools import adjust_permissions, remove_dir +from easybuild.tools.run import run_cmd + + +class EB_Mamba(EB_Anaconda): + """Support for building/installing Mamba.""" + + def sanity_check_step(self): + """ + Custom sanity check for Anaconda and Miniconda + """ + custom_paths = { + 'files': [os.path.join('bin', x) for x in ['2to3', 'conda', 'pydoc', 'python', 'mamba']], + 'dirs': ['bin', 'etc', 'lib', 'pkgs'], + } + # Directly call EB_Anaconda's super, as this sanity_check_step should _overwrite_ Anaconda's (not call it) + super(EB_Anaconda, self).sanity_check_step(custom_paths=custom_paths) From 8617bd33db82f1d32d61e90728835ef45d05ab40 Mon Sep 17 00:00:00 2001 From: casparl Date: Tue, 18 Oct 2022 15:40:00 +0200 Subject: [PATCH 45/54] Removed unneeded imports --- easybuild/easyblocks/m/mamba.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/easybuild/easyblocks/m/mamba.py b/easybuild/easyblocks/m/mamba.py index cf65e3e62a..8c6c5afd3e 100644 --- a/easybuild/easyblocks/m/mamba.py +++ b/easybuild/easyblocks/m/mamba.py @@ -33,8 +33,6 @@ import stat from easybuild.easyblocks.a.anaconda import EB_Anaconda -from easybuild.tools.filetools import adjust_permissions, remove_dir -from easybuild.tools.run import run_cmd class EB_Mamba(EB_Anaconda): From 662667c28ea14e9fe82d3ab14394f531e9168a9b Mon Sep 17 00:00:00 2001 From: casparl Date: Tue, 18 Oct 2022 15:40:32 +0200 Subject: [PATCH 46/54] Removed more unneeded imports --- easybuild/easyblocks/m/mamba.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/easyblocks/m/mamba.py b/easybuild/easyblocks/m/mamba.py index 8c6c5afd3e..beebea0245 100644 --- a/easybuild/easyblocks/m/mamba.py +++ b/easybuild/easyblocks/m/mamba.py @@ -30,7 +30,6 @@ """ import os -import stat from easybuild.easyblocks.a.anaconda import EB_Anaconda From 595b3be62c710e14f8453b91c42037568f647f2f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 18 Oct 2022 16:30:58 +0200 Subject: [PATCH 47/54] Make dm_reverb aware of prebuildopts --- easybuild/easyblocks/d/dm_reverb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/d/dm_reverb.py b/easybuild/easyblocks/d/dm_reverb.py index 60b012f7a2..c847aea331 100644 --- a/easybuild/easyblocks/d/dm_reverb.py +++ b/easybuild/easyblocks/d/dm_reverb.py @@ -94,7 +94,8 @@ def build_step(self, *args, **kwargs): # print full compilation commands bazel_build_opts += " --subcommands" - bazel_cmd = "bazel %s build %s %s" % (bazel_opts, bazel_build_opts, bazel_build_pkg) + bazel_cmd = "%s bazel %s build %s %s" % (self.cfg['prebuildopts'], bazel_opts, bazel_build_opts, + bazel_build_pkg) return run_cmd(bazel_cmd, log_all=True, simple=True, log_output=True) From 7923a47b9015f6f98d29a89aa9aa124c475e032a Mon Sep 17 00:00:00 2001 From: casparl Date: Tue, 18 Oct 2022 17:17:05 +0200 Subject: [PATCH 48/54] Changed function description for sanity check. Removed 'bin', since its presence is already indirectly checked through the files in the sanity_check --- easybuild/easyblocks/m/mamba.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/m/mamba.py b/easybuild/easyblocks/m/mamba.py index beebea0245..bba972ae1a 100644 --- a/easybuild/easyblocks/m/mamba.py +++ b/easybuild/easyblocks/m/mamba.py @@ -39,11 +39,11 @@ class EB_Mamba(EB_Anaconda): def sanity_check_step(self): """ - Custom sanity check for Anaconda and Miniconda + Custom sanity check for Mamba """ custom_paths = { 'files': [os.path.join('bin', x) for x in ['2to3', 'conda', 'pydoc', 'python', 'mamba']], - 'dirs': ['bin', 'etc', 'lib', 'pkgs'], + 'dirs': ['etc', 'lib', 'pkgs'], } # Directly call EB_Anaconda's super, as this sanity_check_step should _overwrite_ Anaconda's (not call it) super(EB_Anaconda, self).sanity_check_step(custom_paths=custom_paths) From 71e057678832dc91c76d4cd59d41f30a9c0f9f9d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 19 Oct 2022 12:48:59 +0200 Subject: [PATCH 49/54] don't check for pip command in EasyBuildMeta easyblcok + pass path to python command to use to determine pip version --- easybuild/easyblocks/e/easybuildmeta.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/e/easybuildmeta.py b/easybuild/easyblocks/e/easybuildmeta.py index abb7b89837..b1e9970d00 100644 --- a/easybuild/easyblocks/e/easybuildmeta.py +++ b/easybuild/easyblocks/e/easybuildmeta.py @@ -35,7 +35,7 @@ from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_pip_version from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import apply_regex_substitutions, change_dir, read_file, which +from easybuild.tools.filetools import apply_regex_substitutions, change_dir, read_file from easybuild.tools.modules import get_software_root_env_var_name from easybuild.tools.py2vs3 import OrderedDict from easybuild.tools.utilities import flatten @@ -71,14 +71,13 @@ def __init__(self, *args, **kwargs): # - pip is available, and recent enough (>= 21.0); # - use_pip is not specified; pyver = sys.version.split(' ')[0] - pip = which('pip') - self.log.info("Python version: %s - pip: %s", pyver, pip) - if sys.version_info >= (3, 6) and pip and self.cfg['use_pip'] is None: + self.log.info("Python version: %s", pyver) + if sys.version_info >= (3, 6) and self.cfg['use_pip'] is None: # try to determine pip version, ignore any failures that occur while doing so; # problems may occur due changes in environment ($PYTHONPATH, etc.) pip_version = None try: - pip_version = det_pip_version() + pip_version = det_pip_version(python_cmd=sys.executable) self.log.info("Found Python v%s + pip: %s", pyver, pip_version) except Exception as err: self.log.warning("Failed to determine pip version: %s", err) From 9f248200824c4750416156b4d2152d3e3790b7be Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 20 Oct 2022 10:35:17 +0200 Subject: [PATCH 50/54] make numexpr easyblock aware of toolchain with GCC + imkl --- easybuild/easyblocks/n/numexpr.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/n/numexpr.py b/easybuild/easyblocks/n/numexpr.py index 00fe5c89a7..cce57b69ea 100644 --- a/easybuild/easyblocks/n/numexpr.py +++ b/easybuild/easyblocks/n/numexpr.py @@ -28,7 +28,9 @@ import os from distutils.version import LooseVersion +import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.pythonpackage import PythonPackage +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import write_file from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.systemtools import get_cpu_features @@ -78,12 +80,20 @@ def configure_step(self): mkl_ver = get_software_version('imkl') + comp_fam = self.toolchain.comp_family() + self.log.info("Using toolchain with compiler family %s", comp_fam) + if LooseVersion(mkl_ver) >= LooseVersion('2021'): mkl_lib_dirs = [ os.path.join(self.imkl_root, 'mkl', 'latest', 'lib', 'intel64'), ] mkl_include_dirs = os.path.join(self.imkl_root, 'mkl', 'latest', 'include') - mkl_libs = ['mkl_intel_lp64', 'mkl_intel_thread', 'mkl_core', 'iomp5'] + if comp_fam == toolchain.INTELCOMP: + mkl_libs = ['mkl_intel_lp64', 'mkl_intel_thread', 'mkl_core', 'iomp5'] + elif comp_fam == toolchain.GCC: + mkl_libs = ['mkl_intel_lp64', 'mkl_gnu_thread', 'mkl_core', 'gomp'] + else: + raise EasyBuildError("Unknown compiler family, don't know how to link MKL libraries: %s", comp_fam) else: mkl_lib_dirs = [ os.path.join(self.imkl_root, 'mkl', 'lib', 'intel64'), @@ -103,7 +113,9 @@ def configure_step(self): else: site_cfg_lines.append("mkl_libs = %s" % ', '.join(mkl_libs)) - write_file('site.cfg', '\n'.join(site_cfg_lines)) + site_cfg_txt = '\n'.join(site_cfg_lines) + write_file('site.cfg', site_cfg_txt) + self.log.info("site.cfg used for numexpr:\n" + site_cfg_txt) def sanity_check_step(self): """Custom sanity check for numexpr.""" From d8aa9420be572ab4df2c5993c5a3cdf370623404 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Thu, 20 Oct 2022 10:37:49 +0100 Subject: [PATCH 51/54] Add sanity check commands to netCDF --- easybuild/easyblocks/n/netcdf.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/n/netcdf.py b/easybuild/easyblocks/n/netcdf.py index 879136495b..47f5645ea4 100644 --- a/easybuild/easyblocks/n/netcdf.py +++ b/easybuild/easyblocks/n/netcdf.py @@ -141,7 +141,9 @@ def sanity_check_step(self): 'dirs': [] } - super(EB_netCDF, self).sanity_check_step(custom_paths=custom_paths) + custom_commands = ["nc-config --help", "ncgen -h"] + + super(EB_netCDF, self).sanity_check_step(custom_commands=custom_commands, custom_paths=custom_paths) def set_netcdf_env_vars(log): From 2aecf21ebbf5737c72a2b02ae58db5cf78724d23 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 20 Oct 2022 11:40:39 +0200 Subject: [PATCH 52/54] trivial code refactoring in sanity_check_step of netCDF eaysblock --- easybuild/easyblocks/n/netcdf.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/n/netcdf.py b/easybuild/easyblocks/n/netcdf.py index 47f5645ea4..83289a936e 100644 --- a/easybuild/easyblocks/n/netcdf.py +++ b/easybuild/easyblocks/n/netcdf.py @@ -141,7 +141,10 @@ def sanity_check_step(self): 'dirs': [] } - custom_commands = ["nc-config --help", "ncgen -h"] + custom_commands = [ + "nc-config --help", + "ncgen -h", + ] super(EB_netCDF, self).sanity_check_step(custom_commands=custom_commands, custom_paths=custom_paths) From 53f742829eaa56cfe1496850a3b3f66de45c2c12 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Fri, 21 Oct 2022 12:15:18 +0800 Subject: [PATCH 53/54] prepare release notes for EasyBuild v4.6.2 + bump version to 4.6.2 --- RELEASE_NOTES | 29 ++++++++++++++++++++++++++++- easybuild/easyblocks/__init__.py | 2 +- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 7e9f4fd022..1941781e8d 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -3,7 +3,34 @@ For more detailed information, please see the git log. These release notes can also be consulted at http://easybuild.readthedocs.org/en/latest/Release_notes.html. -The latest version of easybuild-easyblocks provides 246 software-specific easyblocks and 37 generic easyblocks. +The latest version of easybuild-easyblocks provides 248 software-specific easyblocks and 37 generic easyblocks. + + +v4.6.2 (October 21st 2022) +-------------------------- + +update/bugfix release + +- 2 new software-specific easyblock: + - CUDA compatibility libraries (#2764) and mamba (#2808) +- minor enhancements and updates, including: + - update OpenFOAM easyblock to support OpenFOAM 10 + clean up variant/version checks (#2766) + - added support for ESMPy in ESMF (#2789) + - enhance OpenBLAS easyblock to support running LAPACK test suite + checking how many tests fail (#2801) + - make numexpr easyblock aware of toolchain with GCC + imkl (#2810) + - add sanity check commands for netCDF (#2811) +- various bug fixes, including: + - handle problems copying symlink that points to CUDA folder that is not created for non CUDA builds of SuiteSparse (#2790) + - don't install docs (to avoid trouble with Java) + add Rocky support for ABAQUS (#2792) + - Correctly count the number of failing tests (not failing test suites) in PyTorch builds (#2794) + - fix docstring for PyTorch easyblock (#2795) + - handle iterative builds with MakeCP EasyBlock (#2798) + - correctly count errors/failures in PyTorch test summary (#2803) + - accept both None and empty value for optarch to let OpenCV detect host CPU (#2804) + - enhance EasyBuildMeta easyblock: auto-enable installing with pip + fix setup.py of easyconfigs package so installation with setuptools >= 61.0 works (#2805) + - use `python -m pip` instead of `pip` in PythonPackage easyblock (#2807) +- other changes: + - make the test output from PythonPackage less verbose by disabling default search for error patterns done by run_cmd (2797) v4.6.1 (September 12th 2022) diff --git a/easybuild/easyblocks/__init__.py b/easybuild/easyblocks/__init__.py index c546b0843b..a1ef9ee8ef 100644 --- a/easybuild/easyblocks/__init__.py +++ b/easybuild/easyblocks/__init__.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.6.2.dev0') +VERSION = LooseVersion('4.6.2') UNKNOWN = 'UNKNOWN' From ea557bd10227520885dcdd237ab3d11347637a36 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 21 Oct 2022 06:33:02 +0200 Subject: [PATCH 54/54] minor tweak release notes for EasyBuild v4.6.2 --- RELEASE_NOTES | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 1941781e8d..77347cb907 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -22,10 +22,9 @@ update/bugfix release - various bug fixes, including: - handle problems copying symlink that points to CUDA folder that is not created for non CUDA builds of SuiteSparse (#2790) - don't install docs (to avoid trouble with Java) + add Rocky support for ABAQUS (#2792) - - Correctly count the number of failing tests (not failing test suites) in PyTorch builds (#2794) + - correctly count the number of failing tests (not failing test suites) in PyTorch builds (#2794, #2803) - fix docstring for PyTorch easyblock (#2795) - - handle iterative builds with MakeCP EasyBlock (#2798) - - correctly count errors/failures in PyTorch test summary (#2803) + - handle iterative builds with MakeCp easyblock (#2798) - accept both None and empty value for optarch to let OpenCV detect host CPU (#2804) - enhance EasyBuildMeta easyblock: auto-enable installing with pip + fix setup.py of easyconfigs package so installation with setuptools >= 61.0 works (#2805) - use `python -m pip` instead of `pip` in PythonPackage easyblock (#2807)