diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 7e9f4fd022..77347cb907 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -3,7 +3,33 @@ For more detailed information, please see the git log. These release notes can also be consulted at http://easybuild.readthedocs.org/en/latest/Release_notes.html. -The latest version of easybuild-easyblocks provides 246 software-specific easyblocks and 37 generic easyblocks. +The latest version of easybuild-easyblocks provides 248 software-specific easyblocks and 37 generic easyblocks. + + +v4.6.2 (October 21st 2022) +-------------------------- + +update/bugfix release + +- 2 new software-specific easyblock: + - CUDA compatibility libraries (#2764) and mamba (#2808) +- minor enhancements and updates, including: + - update OpenFOAM easyblock to support OpenFOAM 10 + clean up variant/version checks (#2766) + - added support for ESMPy in ESMF (#2789) + - enhance OpenBLAS easyblock to support running LAPACK test suite + checking how many tests fail (#2801) + - make numexpr easyblock aware of toolchain with GCC + imkl (#2810) + - add sanity check commands for netCDF (#2811) +- various bug fixes, including: + - handle problems copying symlink that points to CUDA folder that is not created for non CUDA builds of SuiteSparse (#2790) + - don't install docs (to avoid trouble with Java) + add Rocky support for ABAQUS (#2792) + - correctly count the number of failing tests (not failing test suites) in PyTorch builds (#2794, #2803) + - fix docstring for PyTorch easyblock (#2795) + - handle iterative builds with MakeCp easyblock (#2798) + - accept both None and empty value for optarch to let OpenCV detect host CPU (#2804) + - enhance EasyBuildMeta easyblock: auto-enable installing with pip + fix setup.py of easyconfigs package so installation with setuptools >= 61.0 works (#2805) + - use `python -m pip` instead of `pip` in PythonPackage easyblock (#2807) +- other changes: + - make the test output from PythonPackage less verbose by disabling default search for error patterns done by run_cmd (2797) v4.6.1 (September 12th 2022) diff --git a/easybuild/easyblocks/__init__.py b/easybuild/easyblocks/__init__.py index 7aff8318b6..a1ef9ee8ef 100644 --- a/easybuild/easyblocks/__init__.py +++ b/easybuild/easyblocks/__init__.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.6.1') +VERSION = LooseVersion('4.6.2') UNKNOWN = 'UNKNOWN' diff --git a/easybuild/easyblocks/a/abaqus.py b/easybuild/easyblocks/a/abaqus.py index 91ba89d56d..56d15b93be 100644 --- a/easybuild/easyblocks/a/abaqus.py +++ b/easybuild/easyblocks/a/abaqus.py @@ -44,6 +44,7 @@ from easybuild.tools.filetools import change_dir, symlink, write_file from easybuild.tools.modules import get_software_root from easybuild.tools.run import run_cmd_qa +from easybuild.tools.systemtools import get_os_name from easybuild.tools.py2vs3 import OrderedDict @@ -78,6 +79,9 @@ def extract_step(self): def configure_step(self): """Configure ABAQUS installation.""" if LooseVersion(self.version) >= LooseVersion('2016'): + # Rocky Linux isn't recognized; faking it as RHEL + if get_os_name() in ['Rocky Linux', 'AlmaLinux']: + setvar('DISTRIB_ID', 'RedHatEnterpriseServer') # skip checking of Linux version setvar('DSY_Force_OS', 'linux_a64') # skip checking of license server @@ -126,10 +130,9 @@ def install_step(self): # rather than a regular dictionary (where there's no guarantee on key order in general) std_qa = OrderedDict() - # Enable Extended Product Documentation - std_qa[selectionstr % (r"\[ \]", "Extended Product Documentation")] = "%(nr)s" - # Enable Abaqus CAE (docs) - std_qa[selectionstr % (r"\[ \]", "Abaqus CAE")] = "%(nr)s" + # Disable Extended Product Documentation because it has a troublesome Java dependency + std_qa[selectionstr % (r"\[*\]", "Extended Product Documentation")] = "%(nr)s" + installed_docs = False # hard disabled, previous support was actually incomplete # enable all ABAQUS components std_qa[selectionstr % (r"\[ \]", "Abaqus.*")] = "%(nr)s" @@ -146,9 +149,9 @@ def install_step(self): # Disable/enable Tosca if self.cfg['with_tosca']: - std_qa[selectionstr % (r"\[\ \]", "Tosca")] = "%(nr)s" + std_qa[selectionstr % (r"\[\ \]", "Tosca.*")] = "%(nr)s" else: - std_qa[selectionstr % (r"\[\*\]", "Tosca")] = "%(nr)s" + std_qa[selectionstr % (r"\[\*\]", "Tosca.*")] = "%(nr)s" # disable CloudView std_qa[r"(?P[0-9]+) \[X\] Search using CloudView\nEnter selection:"] = '%(cloudview)s\n\n' @@ -170,7 +173,7 @@ def install_step(self): cae_subdir = os.path.join(self.installdir, 'cae') sim_subdir = os.path.join(self.installdir, 'sim') std_qa[r"Default.*SIMULIA/EstProducts.*:"] = cae_subdir - std_qa[r"SIMULIA[0-9]*doc.*:"] = os.path.join(self.installdir, 'doc') + std_qa[r"SIMULIA[0-9]*doc.*:"] = os.path.join(self.installdir, 'doc') # if docs are installed std_qa[r"SimulationServices.*:"] = sim_subdir std_qa[r"Choose the CODE installation directory.*:\n.*\n\n.*:"] = sim_subdir std_qa[r"SIMULIA/CAE.*:"] = cae_subdir @@ -197,13 +200,13 @@ def install_step(self): std_qa[r"Please choose an action:"] = '1' - if LooseVersion(self.version) >= LooseVersion('2022'): + if LooseVersion(self.version) >= LooseVersion('2022') and installed_docs: java_root = get_software_root('Java') if java_root: std_qa[r"Please enter .*Java Runtime Environment.* path.(\n.*)+Default \[\]:"] = java_root std_qa[r"Please enter .*Java Runtime Environment.* path.(\n.*)+Default \[.+\]:"] = '' else: - raise EasyBuildError("Java is a required dependency for ABAQUS versions >= 2022, but it is missing") + raise EasyBuildError("Java is required for ABAQUS docs versions >= 2022, but it is missing") # Continue std_qa[nextstr] = '' @@ -330,7 +333,7 @@ def sanity_check_step(self): custom_commands = [] if LooseVersion(self.version) >= LooseVersion('2016'): - custom_paths['dirs'].extend(['cae', 'Commands', 'doc']) + custom_paths['dirs'].extend(['cae', 'Commands']) if LooseVersion(self.version) < LooseVersion('2020'): custom_paths['dirs'].extend(['sim']) # 'all' also check license server, but lmstat is usually not available diff --git a/easybuild/easyblocks/c/cudacompat.py b/easybuild/easyblocks/c/cudacompat.py new file mode 100644 index 0000000000..07e7bdb175 --- /dev/null +++ b/easybuild/easyblocks/c/cudacompat.py @@ -0,0 +1,266 @@ +## +# Copyright 2012-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for CUDA compat libraries, implemented as an easyblock + +Ref: https://docs.nvidia.com/deploy/cuda-compatibility/index.html#manually-installing-from-runfile + +@author: Alexander Grund (TU Dresden) +""" + +import os +from distutils.version import LooseVersion + +from easybuild.easyblocks.generic.binary import Binary +from easybuild.framework.easyconfig import CUSTOM, MANDATORY +from easybuild.tools.build_log import EasyBuildError, print_warning +from easybuild.tools.config import build_option, IGNORE +from easybuild.tools.filetools import copy_file, find_glob_pattern, mkdir, symlink, which +from easybuild.tools.run import run_cmd + + +class EB_CUDAcompat(Binary): + """ + Support for installing CUDA compat libraries. + """ + + @staticmethod + def extra_options(): + """Add variable for driver version""" + extra_vars = Binary.extra_options() + extra_vars.update({ + 'compatible_driver_versions': [None, "Minimum (system) CUDA driver versions which are compatible", CUSTOM], + 'nv_version': [None, "Version of the driver package", MANDATORY], + }) + # We don't need the extract and install step from the Binary EasyBlock + del extra_vars['extract_sources'] + del extra_vars['install_cmd'] + # And also no need to modify the PATH + del extra_vars['prepend_to_path'] + return extra_vars + + def __init__(self, *args, **kwargs): + """Initialize custom class variables for CUDACompat.""" + super(EB_CUDAcompat, self).__init__(*args, **kwargs) + self._has_nvidia_smi = None + + @property + def has_nvidia_smi(self): + """Return whether the system has nvidia-smi and print a warning once if not""" + if self._has_nvidia_smi is None: + self._has_nvidia_smi = which('nvidia-smi', on_error=IGNORE) is not None + if not self._has_nvidia_smi: + print_warning('Could not find nvidia-smi. Assuming a system without GPUs and skipping checks!') + return self._has_nvidia_smi + + def _run_nvidia_smi(self, args): + """ + Run nvidia-smi with the given argument(s) and return the output. + + Also does sensible logging. + Raises RuntimeError on failure. + """ + if not self.has_nvidia_smi: + raise RuntimeError('Could not find nvidia-smi.') + cmd = 'nvidia-smi ' + args + out, ec = run_cmd(cmd, log_ok=False, log_all=False, regexp=False) + if ec != 0: + raise RuntimeError("`%s` returned exit code %s with output:\n%s" % (cmd, ec, out)) + else: + self.log.info('`%s` succeeded with output:\n%s' % (cmd, out)) + return out.strip().split('\n') + + def prepare_step(self, *args, **kwargs): + """Parse and check the compatible_driver_versions value of the EasyConfig""" + compatible_driver_versions = self.cfg.get('compatible_driver_versions') + if compatible_driver_versions: + try: + # Create a dictionary with the major version as the keys + self.compatible_driver_version_map = { + int(v.split('.', 1)[0]): v + for v in compatible_driver_versions + } + except ValueError: + raise EasyBuildError("Invalid format of compatible_driver_versions. " + "Expected numeric major versions, got '%s'", compatible_driver_versions) + else: + self.compatible_driver_version_map = None + if 'LD_LIBRARY_PATH' in (build_option('filter_env_vars') or []): + raise EasyBuildError("This module relies on setting $LD_LIBRARY_PATH, " + "so you need to remove this variable from --filter-env-vars") + super(EB_CUDAcompat, self).prepare_step(*args, **kwargs) + + def fetch_step(self, *args, **kwargs): + """Check for EULA acceptance prior to getting sources.""" + # EULA for NVIDIA driver must be accepted via --accept-eula-for EasyBuild configuration option, + # or via 'accept_eula = True' in easyconfig file + self.check_accepted_eula( + name='NVIDIA-driver', + more_info='https://www.nvidia.com/content/DriverDownload-March2009/licence.php?lang=us' + ) + return super(EB_CUDAcompat, self).fetch_step(*args, **kwargs) + + def extract_step(self): + """Extract the files without running the installer.""" + execpath = self.src[0]['path'] + tmpdir = os.path.join(self.builddir, 'tmp') + targetdir = os.path.join(self.builddir, 'extracted') + run_cmd("/bin/sh " + execpath + " --extract-only --tmpdir='%s' --target '%s'" % (tmpdir, targetdir)) + self.src[0]['finalpath'] = targetdir + + def test_step(self): + """ + Check for a compatible driver version if the EC has that information. + + This can be skipped with `--skip-test-step`. + """ + + if self.compatible_driver_version_map and self.has_nvidia_smi: + try: + out_lines = self._run_nvidia_smi('--query-gpu=driver_version --format=csv,noheader') + if not out_lines or not out_lines[0]: + raise RuntimeError('nvidia-smi did not find any GPUs on the system') + driver_version = out_lines[0] + version_parts = driver_version.split('.') + if len(version_parts) < 3 or any(not v.isdigit() for v in version_parts): + raise RuntimeError("Expected the version to be in format x.y.z (all numeric) " + "but got '%s'" % driver_version) + except RuntimeError as err: + self.log.warning("Failed to get the CUDA driver version: %s", err) + driver_version = None + + if not driver_version: + print_warning('Failed to determine the CUDA driver version, so skipping the compatibility check!') + else: + driver_version_major = int(driver_version.split('.', 1)[0]) + compatible_driver_versions = ', '.join(sorted(self.compatible_driver_version_map.values())) + try: + min_required_version = self.compatible_driver_version_map[driver_version_major] + except KeyError: + raise EasyBuildError('The installed CUDA driver %s is not a supported branch/major version for ' + '%s %s. Supported drivers: %s', + driver_version, self.name, self.version, compatible_driver_versions) + if LooseVersion(driver_version) < min_required_version: + raise EasyBuildError('The installed CUDA driver %s is to old for %s %s, ' + 'need at least %s. Supported drivers: %s', + driver_version, self.name, self.version, + min_required_version, compatible_driver_versions) + else: + self.log.info('The installed CUDA driver %s appears to be supported.', driver_version) + + return super(EB_CUDAcompat, self).test_step() + + def install_step(self): + """Install CUDA compat libraries by copying library files and creating the symlinks.""" + libdir = os.path.join(self.installdir, 'lib') + mkdir(libdir) + + # From https://docs.nvidia.com/deploy/cuda-compatibility/index.html#installing-from-network-repo: + # The cuda-compat package consists of the following files: + # - libcuda.so.* - the CUDA Driver + # - libnvidia-nvvm.so.* - JIT LTO ( CUDA 11.5 and later only) + # - libnvidia-ptxjitcompiler.so.* - the JIT (just-in-time) compiler for PTX files + + library_globs = [ + 'libcuda.so.*', + 'libnvidia-ptxjitcompiler.so.*', + ] + if LooseVersion(self.version) >= '11.5': + library_globs.append('libnvidia-nvvm.so.*') + + startdir = self.cfg['start_dir'] + nv_version = self.cfg['nv_version'] + for library_glob in library_globs: + library_path = find_glob_pattern(os.path.join(startdir, library_glob)) + library = os.path.basename(library_path) + # Sanity check the version + if library_glob == 'libcuda.so.*': + library_version = library.split('.', 2)[2] + if library_version != nv_version: + raise EasyBuildError('Expected driver version %s (from nv_version) but found %s ' + '(determined from file %s)', nv_version, library_version, library_path) + + copy_file(library_path, os.path.join(libdir, library)) + if library.endswith('.' + nv_version): + # E.g. libcuda.so.510.73.08 -> libcuda.so.1 + versioned_symlink = library[:-len(nv_version)] + '1' + else: + # E.g. libnvidia-nvvm.so.4.0.0 -> libnvidia-nvvm.so.4 + versioned_symlink = library.rsplit('.', 2)[0] + symlink(library, os.path.join(libdir, versioned_symlink), use_abspath_source=False) + # E.g. libcuda.so.1 -> libcuda.so + unversioned_symlink = versioned_symlink.rsplit('.', 1)[0] + symlink(versioned_symlink, os.path.join(libdir, unversioned_symlink), use_abspath_source=False) + + def make_module_req_guess(self): + """Don't try to guess anything.""" + return dict() + + def make_module_extra(self): + """Skip the changes from the Binary EasyBlock and (only) set LD_LIBRARY_PATH.""" + + txt = super(Binary, self).make_module_extra() + txt += self.module_generator.prepend_paths('LD_LIBRARY_PATH', 'lib') + return txt + + def sanity_check_step(self): + """Check for core files (unversioned libs, symlinks)""" + libraries = [ + 'libcuda.so', + 'libnvidia-ptxjitcompiler.so', + ] + if LooseVersion(self.version) >= '11.5': + libraries.append('libnvidia-nvvm.so') + custom_paths = { + 'files': [os.path.join(self.installdir, 'lib', x) for x in libraries], + 'dirs': ['lib', 'lib64'], + } + super(EB_CUDAcompat, self).sanity_check_step(custom_paths=custom_paths) + + if self.has_nvidia_smi: + fake_mod_data = None + + # skip loading of fake module when using --sanity-check-only, load real module instead + if build_option('sanity_check_only'): + self.load_module() + elif not self.dry_run: + fake_mod_data = self.load_fake_module(purge=True, verbose=True) + + try: + out_lines = self._run_nvidia_smi('--query --display=COMPUTE') + + if fake_mod_data: + self.clean_up_fake_module(fake_mod_data) + + cuda_version = next((line.rsplit(' ', 1)[1] for line in out_lines if line.startswith('CUDA')), None) + if not cuda_version: + raise RuntimeError('Failed to find CUDA version!') + self.log.info('CUDA version (as per nvidia-smi) after loading the module: ' + cuda_version) + if LooseVersion(cuda_version) != self.version: + raise RuntimeError('Reported CUDA version %s is not %s' % (cuda_version, self.version)) + except RuntimeError as err: + if fake_mod_data: + self.clean_up_fake_module(fake_mod_data) + raise EasyBuildError('Version check via nvidia-smi after loading the module failed: %s', err) diff --git a/easybuild/easyblocks/d/dm_reverb.py b/easybuild/easyblocks/d/dm_reverb.py index 18012d98bb..c847aea331 100644 --- a/easybuild/easyblocks/d/dm_reverb.py +++ b/easybuild/easyblocks/d/dm_reverb.py @@ -94,7 +94,8 @@ def build_step(self, *args, **kwargs): # print full compilation commands bazel_build_opts += " --subcommands" - bazel_cmd = "bazel %s build %s %s" % (bazel_opts, bazel_build_opts, bazel_build_pkg) + bazel_cmd = "%s bazel %s build %s %s" % (self.cfg['prebuildopts'], bazel_opts, bazel_build_opts, + bazel_build_pkg) return run_cmd(bazel_cmd, log_all=True, simple=True, log_output=True) @@ -121,6 +122,7 @@ def install_step(self, *args, **kwargs): 'installopts': self.cfg['installopts'], 'loc': whl_path, 'prefix': self.installdir, + 'python': self.python_cmd, } return super(EB_dm_minus_reverb, self).install_step(*args, **kwargs) diff --git a/easybuild/easyblocks/e/easybuildmeta.py b/easybuild/easyblocks/e/easybuildmeta.py index 5949a181f8..b1e9970d00 100644 --- a/easybuild/easyblocks/e/easybuildmeta.py +++ b/easybuild/easyblocks/e/easybuildmeta.py @@ -33,9 +33,9 @@ import sys from distutils.version import LooseVersion -from easybuild.easyblocks.generic.pythonpackage import PythonPackage +from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_pip_version from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import read_file +from easybuild.tools.filetools import apply_regex_substitutions, change_dir, read_file from easybuild.tools.modules import get_software_root_env_var_name from easybuild.tools.py2vs3 import OrderedDict from easybuild.tools.utilities import flatten @@ -66,6 +66,27 @@ def __init__(self, *args, **kwargs): # consider setuptools first, in case it is listed as a sources self.easybuild_pkgs.insert(0, 'setuptools') + # opt-in to using pip for recent version of EasyBuild, if: + # - EasyBuild is being installed for Python >= 3.6; + # - pip is available, and recent enough (>= 21.0); + # - use_pip is not specified; + pyver = sys.version.split(' ')[0] + self.log.info("Python version: %s", pyver) + if sys.version_info >= (3, 6) and self.cfg['use_pip'] is None: + # try to determine pip version, ignore any failures that occur while doing so; + # problems may occur due changes in environment ($PYTHONPATH, etc.) + pip_version = None + try: + pip_version = det_pip_version(python_cmd=sys.executable) + self.log.info("Found Python v%s + pip: %s", pyver, pip_version) + except Exception as err: + self.log.warning("Failed to determine pip version: %s", err) + + if pip_version and LooseVersion(pip_version) >= LooseVersion('21.0'): + self.log.info("Auto-enabling use of pip to install EasyBuild!") + self.cfg['use_pip'] = True + self.determine_install_command() + # Override this function since we want to respect the user choice for the python installation to use # (which can be influenced by EB_PYTHON and EB_INSTALLPYTHON) def prepare_python(self): @@ -92,6 +113,21 @@ def build_step(self): """No building for EasyBuild packages.""" pass + def fix_easyconfigs_setup_py_setuptools61(self): + """ + Patch setup.py of easybuild-easyconfigs package if needed to make sure that installation works + for recent setuptools versions (>= 61.0). + """ + # cfr. https://github.com/easybuilders/easybuild-easyconfigs/pull/15206 + cwd = os.getcwd() + regex = re.compile(r'packages=\[\]') + setup_py_txt = read_file('setup.py') + if regex.search(setup_py_txt) is None: + self.log.info("setup.py at %s needs to be fixed to install with setuptools >= 61.0", cwd) + apply_regex_substitutions('setup.py', [(r'^setup\(', 'setup(packages=[],')]) + else: + self.log.info("setup.py at %s does not need to be fixed to install with setuptools >= 61.0", cwd) + def install_step(self): """Install EasyBuild packages one by one.""" try: @@ -108,7 +144,11 @@ def install_step(self): else: self.log.info("Installing package %s", pkg) - os.chdir(os.path.join(self.builddir, seldirs[0])) + change_dir(os.path.join(self.builddir, seldirs[0])) + + if pkg == 'easybuild-easyconfigs': + self.fix_easyconfigs_setup_py_setuptools61() + super(EB_EasyBuildMeta, self).install_step() except OSError as err: diff --git a/easybuild/easyblocks/e/esmf.py b/easybuild/easyblocks/e/esmf.py index 9af0a257a1..cc3986ca45 100644 --- a/easybuild/easyblocks/e/esmf.py +++ b/easybuild/easyblocks/e/esmf.py @@ -27,6 +27,7 @@ @author: Kenneth Hoste (Ghent University) @author: Damian Alvarez (Forschungszentrum Juelich GmbH) +@author: Maxime Boissonneault (Digital Research Alliance of Canada) """ import os from distutils.version import LooseVersion @@ -35,7 +36,7 @@ import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.modules import get_software_root +from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import get_shared_lib_ext @@ -114,6 +115,39 @@ def configure_step(self): cmd = "make info" run_cmd(cmd, log_all=True, simple=True, log_ok=True) + def install_step(self): + # first, install the software + super(EB_ESMF, self).install_step() + + python = get_software_version('Python') + if python: + # then, install the python bindings + py_subdir = os.path.join(self.builddir, 'esmf-ESMF_%s' % '_'.join(self.version.split('.')), + 'src', 'addon', 'ESMPy') + try: + os.chdir(py_subdir) + except OSError as err: + raise EasyBuildError("Failed to move to: %s", err) + + cmd = "python setup.py build --ESMFMKFILE=%s/lib/esmf.mk " % self.installdir + cmd += " && python setup.py install --prefix=%s" % self.installdir + run_cmd(cmd, log_all=True, simple=True, log_ok=True) + + def make_module_extra(self): + """Add install path to PYTHONPATH or EBPYTHONPREFIXES""" + txt = super(EB_ESMF, self).make_module_extra() + + if self.cfg['multi_deps'] and 'Python' in self.cfg['multi_deps']: + txt += self.module_generator.prepend_paths('EBPYTHONPREFIXES', '') + else: + python = get_software_version('Python') + if python: + pyshortver = '.'.join(get_software_version('Python').split('.')[:2]) + pythonpath = os.path.join('lib', 'python%s' % pyshortver, 'site-packages') + txt += self.module_generator.prepend_paths('PYTHONPATH', [pythonpath]) + + return txt + def sanity_check_step(self): """Custom sanity check for ESMF.""" @@ -130,4 +164,8 @@ def sanity_check_step(self): 'dirs': ['include', 'mod'], } - super(EB_ESMF, self).sanity_check_step(custom_paths=custom_paths) + custom_commands = [] + if get_software_root('Python'): + custom_commands += ["python -c 'import ESMF'"] + + super(EB_ESMF, self).sanity_check_step(custom_commands=custom_commands, custom_paths=custom_paths) diff --git a/easybuild/easyblocks/generic/makecp.py b/easybuild/easyblocks/generic/makecp.py index f8603445b7..02b2ad12d0 100644 --- a/easybuild/easyblocks/generic/makecp.py +++ b/easybuild/easyblocks/generic/makecp.py @@ -26,6 +26,7 @@ @author: George Tsouloupas (The Cyprus Institute) @author: Fotis Georgatos (Uni.Lu, NTUA) @author: Kenneth Hoste (Ghent University) +@author: Maxime Boissonneault (Digital Research Alliance of Canada, Universite Laval) """ import os import glob @@ -70,6 +71,10 @@ def install_step(self): files_to_copy = self.cfg.get('files_to_copy') or [] self.log.debug("Starting install_step with files_to_copy: %s", files_to_copy) + + # if this is an iterative build directories will be copied multiple times + dirs_exist_ok = True if self.iter_opts else False + for fil in files_to_copy: if isinstance(fil, tuple): # ([src1, src2], targetdir) @@ -128,6 +133,6 @@ def install_step(self): elif os.path.isdir(filepath): self.log.debug("Copying directory %s to %s", filepath, target) fulltarget = os.path.join(target, os.path.basename(filepath)) - copy_dir(filepath, fulltarget, symlinks=self.cfg['keepsymlinks']) + copy_dir(filepath, fulltarget, symlinks=self.cfg['keepsymlinks'], dirs_exist_ok=dirs_exist_ok) else: raise EasyBuildError("Can't copy non-existing path %s to %s", filepath, target) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index f84b601ef0..536574c743 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -60,7 +60,7 @@ # not 'easy_install' deliberately, to avoid that pkg installations listed in easy-install.pth get preference # '.' is required at the end when using easy_install/pip in unpacked source dir EASY_INSTALL_TARGET = "easy_install" -PIP_INSTALL_CMD = "pip install --prefix=%(prefix)s %(installopts)s %(loc)s" +PIP_INSTALL_CMD = "%(python)s -m pip install --prefix=%(prefix)s %(installopts)s %(loc)s" SETUP_PY_INSTALL_CMD = "%(python)s setup.py %(install_target)s --prefix=%(prefix)s %(installopts)s" UNKNOWN = 'UNKNOWN' @@ -204,14 +204,14 @@ def get_pylibdirs(python_cmd): return all_pylibdirs -def det_pip_version(): - """Determine version of currently active 'pip' command.""" +def det_pip_version(python_cmd='python'): + """Determine version of currently active 'pip' module.""" pip_version = None log = fancylogger.getLogger('det_pip_version', fname=False) log.info("Determining pip version...") - out, _ = run_cmd("pip --version", verbose=False, simple=False, trace=False) + out, _ = run_cmd("%s -m pip --version" % python_cmd, verbose=False, simple=False, trace=False) pip_version_regex = re.compile('^pip ([0-9.]+)') res = pip_version_regex.search(out) @@ -247,8 +247,8 @@ def extra_options(extra_vars=None): "the pip version check. Enabled by default when pip_ignore_installed=True", CUSTOM], 'req_py_majver': [None, "Required major Python version (only relevant when using system Python)", CUSTOM], 'req_py_minver': [None, "Required minor Python version (only relevant when using system Python)", CUSTOM], - 'sanity_pip_check': [False, "Run 'pip check' to ensure all required Python packages are installed " - "and check for any package with an invalid (0.0.0) version.", CUSTOM], + 'sanity_pip_check': [False, "Run 'python -m pip check' to ensure all required Python packages are " + "installed and check for any package with an invalid (0.0.0) version.", CUSTOM], 'runtest': [True, "Run unit tests.", CUSTOM], # overrides default 'unpack_sources': [None, "Unpack sources prior to build/install. Defaults to 'True' except for whl files", CUSTOM], @@ -261,8 +261,8 @@ def extra_options(extra_vars=None): # see https://packaging.python.org/tutorials/installing-packages/#installing-setuptools-extras 'use_pip_extras': [None, "String with comma-separated list of 'extras' to install via pip", CUSTOM], 'use_pip_for_deps': [False, "Install dependencies using '%s'" % PIP_INSTALL_CMD, CUSTOM], - 'use_pip_requirement': [False, "Install using 'pip install --requirement'. The sources is expected " + - "to be the requirements file.", CUSTOM], + 'use_pip_requirement': [False, "Install using 'python -m pip install --requirement'. The sources is " + + "expected to be the requirements file.", CUSTOM], 'zipped_egg': [False, "Install as a zipped eggs", CUSTOM], }) # Use PYPI_SOURCE as the default for source_urls. @@ -318,6 +318,12 @@ def __init__(self, *args, **kwargs): # determine install command self.use_setup_py = False + self.determine_install_command() + + def determine_install_command(self): + """ + Determine install command to use. + """ if self.cfg.get('use_pip', False) or self.cfg.get('use_pip_editable', False): self.install_cmd = PIP_INSTALL_CMD @@ -361,7 +367,7 @@ def __init__(self, *args, **kwargs): else: raise EasyBuildError("Installing zipped eggs requires using easy_install or pip") - self.log.debug("Using '%s' as install command", self.install_cmd) + self.log.info("Using '%s' as install command", self.install_cmd) def set_pylibdirs(self): """Set Python lib directory-related class variables.""" @@ -468,7 +474,7 @@ def compose_install_command(self, prefix, extrapath=None, installopts=None): using_pip = self.install_cmd.startswith(PIP_INSTALL_CMD) if using_pip: - pip_version = det_pip_version() + pip_version = det_pip_version(python_cmd=self.python_cmd) if pip_version: # pip 8.x or newer required, because of --prefix option being used if LooseVersion(pip_version) >= LooseVersion('8.0'): @@ -684,8 +690,8 @@ def test_step(self, return_output_ec=False): ]) if return_output_ec: - (out, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False) - # need to log seperately, since log_all and log_ok need to be false to retreive out and ec + (out, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, regexp=False) + # need to log seperately, since log_all and log_ok need to be false to retrieve out and ec self.log.info("cmd '%s' exited with exit code %s and output:\n%s", cmd, ec, out) else: run_cmd(cmd, log_all=True, simple=True) @@ -834,8 +840,11 @@ def sanity_check_step(self, *args, **kwargs): kwargs.update({'exts_filter': exts_filter}) if self.cfg.get('sanity_pip_check', False): - pip_version = det_pip_version() + pip_version = det_pip_version(python_cmd=python_cmd) + if pip_version: + pip_check_command = "%s -m pip check" % python_cmd + if LooseVersion(pip_version) >= LooseVersion('9.0.0'): if not self.is_extension: @@ -846,11 +855,11 @@ def sanity_check_step(self, *args, **kwargs): pip_check_errors = [] - pip_check_msg, ec = run_cmd("pip check", log_ok=False) + pip_check_msg, ec = run_cmd(pip_check_command, log_ok=False) if ec: - pip_check_errors.append('`pip check` failed:\n%s' % pip_check_msg) + pip_check_errors.append('`%s` failed:\n%s' % (pip_check_command, pip_check_msg)) else: - self.log.info('`pip check` completed successfully') + self.log.info('`%s` completed successfully' % pip_check_command) # Also check for a common issue where the package version shows up as 0.0.0 often caused # by using setup.py as the installation method for a package which is released as a generic wheel @@ -869,8 +878,8 @@ def sanity_check_step(self, *args, **kwargs): version = next(pkg['version'] for pkg in pkgs if pkg['name'] == unversioned_package) except StopIteration: msg = ('Package %s in unversioned_packages was not found in the installed packages. ' - 'Check that the name from `pip list` is used which may be different than the ' - 'module name.' % unversioned_package) + 'Check that the name from `python -m pip list` is used which may be different ' + 'than the module name.' % unversioned_package) else: msg = ('Package %s in unversioned_packages has a version of %s which is valid. ' 'Please remove it from unversioned_packages.' % (unversioned_package, version)) @@ -894,7 +903,8 @@ def sanity_check_step(self, *args, **kwargs): if pip_check_errors: raise EasyBuildError('\n'.join(pip_check_errors)) else: - raise EasyBuildError("pip >= 9.0.0 is required for running 'pip check', found %s", pip_version) + raise EasyBuildError("pip >= 9.0.0 is required for running '%s', found %s", (pip_check_command, + pip_version)) else: raise EasyBuildError("Failed to determine pip version!") diff --git a/easybuild/easyblocks/m/mamba.py b/easybuild/easyblocks/m/mamba.py new file mode 100644 index 0000000000..bba972ae1a --- /dev/null +++ b/easybuild/easyblocks/m/mamba.py @@ -0,0 +1,49 @@ +## +# Copyright 2009-2022 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for building and installing Mamba, implemented as an easyblock + +@author: Caspar van Leeuwen (SURF) +@author: Kenneth Hoste (HPC-UGent) +""" + +import os + +from easybuild.easyblocks.a.anaconda import EB_Anaconda + + +class EB_Mamba(EB_Anaconda): + """Support for building/installing Mamba.""" + + def sanity_check_step(self): + """ + Custom sanity check for Mamba + """ + custom_paths = { + 'files': [os.path.join('bin', x) for x in ['2to3', 'conda', 'pydoc', 'python', 'mamba']], + 'dirs': ['etc', 'lib', 'pkgs'], + } + # Directly call EB_Anaconda's super, as this sanity_check_step should _overwrite_ Anaconda's (not call it) + super(EB_Anaconda, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/n/netcdf.py b/easybuild/easyblocks/n/netcdf.py index 879136495b..83289a936e 100644 --- a/easybuild/easyblocks/n/netcdf.py +++ b/easybuild/easyblocks/n/netcdf.py @@ -141,7 +141,12 @@ def sanity_check_step(self): 'dirs': [] } - super(EB_netCDF, self).sanity_check_step(custom_paths=custom_paths) + custom_commands = [ + "nc-config --help", + "ncgen -h", + ] + + super(EB_netCDF, self).sanity_check_step(custom_commands=custom_commands, custom_paths=custom_paths) def set_netcdf_env_vars(log): diff --git a/easybuild/easyblocks/n/numexpr.py b/easybuild/easyblocks/n/numexpr.py index 00fe5c89a7..cce57b69ea 100644 --- a/easybuild/easyblocks/n/numexpr.py +++ b/easybuild/easyblocks/n/numexpr.py @@ -28,7 +28,9 @@ import os from distutils.version import LooseVersion +import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.pythonpackage import PythonPackage +from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import write_file from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.systemtools import get_cpu_features @@ -78,12 +80,20 @@ def configure_step(self): mkl_ver = get_software_version('imkl') + comp_fam = self.toolchain.comp_family() + self.log.info("Using toolchain with compiler family %s", comp_fam) + if LooseVersion(mkl_ver) >= LooseVersion('2021'): mkl_lib_dirs = [ os.path.join(self.imkl_root, 'mkl', 'latest', 'lib', 'intel64'), ] mkl_include_dirs = os.path.join(self.imkl_root, 'mkl', 'latest', 'include') - mkl_libs = ['mkl_intel_lp64', 'mkl_intel_thread', 'mkl_core', 'iomp5'] + if comp_fam == toolchain.INTELCOMP: + mkl_libs = ['mkl_intel_lp64', 'mkl_intel_thread', 'mkl_core', 'iomp5'] + elif comp_fam == toolchain.GCC: + mkl_libs = ['mkl_intel_lp64', 'mkl_gnu_thread', 'mkl_core', 'gomp'] + else: + raise EasyBuildError("Unknown compiler family, don't know how to link MKL libraries: %s", comp_fam) else: mkl_lib_dirs = [ os.path.join(self.imkl_root, 'mkl', 'lib', 'intel64'), @@ -103,7 +113,9 @@ def configure_step(self): else: site_cfg_lines.append("mkl_libs = %s" % ', '.join(mkl_libs)) - write_file('site.cfg', '\n'.join(site_cfg_lines)) + site_cfg_txt = '\n'.join(site_cfg_lines) + write_file('site.cfg', site_cfg_txt) + self.log.info("site.cfg used for numexpr:\n" + site_cfg_txt) def sanity_check_step(self): """Custom sanity check for numexpr.""" diff --git a/easybuild/easyblocks/o/openblas.py b/easybuild/easyblocks/o/openblas.py index 4973fa0df1..9ce5034ba0 100644 --- a/easybuild/easyblocks/o/openblas.py +++ b/easybuild/easyblocks/o/openblas.py @@ -3,21 +3,39 @@ @author: Andrew Edmondson (University of Birmingham) @author: Alex Domingo (Vrije Universiteit Brussel) +@author: Kenneth Hoste (Ghent University) """ import os +import re from distutils.version import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake +from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.systemtools import POWER, get_cpu_architecture, get_shared_lib_ext -from easybuild.tools.build_log import print_warning +from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import ERROR from easybuild.tools.run import run_cmd, check_log_for_errors +LAPACK_TEST_TARGET = 'lapack-test' TARGET = 'TARGET' class EB_OpenBLAS(ConfigureMake): """Support for building/installing OpenBLAS.""" + @staticmethod + def extra_options(): + """Custom easyconfig parameters for OpenBLAS easyblock.""" + extra_vars = { + 'max_failing_lapack_tests_num_errors': [0, "Maximum number of LAPACK tests failing " + "due to numerical errors", CUSTOM], + 'max_failing_lapack_tests_other_errors': [0, "Maximum number of LAPACK tests failing " + "due to non-numerical errors", CUSTOM], + 'run_lapack_tests': [False, "Run LAPACK tests during test step, " + "and check whether failing tests exceeds threshold", CUSTOM], + } + + return ConfigureMake.extra_options(extra_vars) + def configure_step(self): """ set up some options - but no configure command to run""" @@ -72,10 +90,47 @@ def build_step(self): cmd = ' '.join([self.cfg['prebuildopts'], makecmd, ' '.join(build_parts), self.cfg['buildopts']]) run_cmd(cmd, log_all=True, simple=True) + def check_lapack_test_results(self, test_output): + """Check output of OpenBLAS' LAPACK test suite ('make lapack-test').""" + + # example: + # --> LAPACK TESTING SUMMARY <-- + # SUMMARY nb test run numerical error other error + # ================ =========== ================= ================ + # ... + # --> ALL PRECISIONS 4116982 4172 (0.101%) 0 (0.000%) + test_summary_pattern = r'\s+'.join([ + r"^--> ALL PRECISIONS", + r"(?P[0-9]+)", + r"(?P[0-9]+)\s+\([0-9.]+\%\)", + r"(?P[0-9]+)\s+\([0-9.]+\%\)", + ]) + regex = re.compile(test_summary_pattern, re.M) + res = regex.search(test_output) + if res: + (tot_cnt, fail_cnt_num_errors, fail_cnt_other_errors) = [int(x) for x in res.groups()] + msg = "%d LAPACK tests run - %d failed due to numerical errors - %d failed due to other errors" + self.log.info(msg, tot_cnt, fail_cnt_num_errors, fail_cnt_other_errors) + + if fail_cnt_other_errors > self.cfg['max_failing_lapack_tests_other_errors']: + raise EasyBuildError("Too many LAPACK tests failed due to non-numerical errors: %d (> %d)", + fail_cnt_other_errors, self.cfg['max_failing_lapack_tests_other_errors']) + + if fail_cnt_num_errors > self.cfg['max_failing_lapack_tests_num_errors']: + raise EasyBuildError("Too many LAPACK tests failed due to numerical errors: %d (> %d)", + fail_cnt_num_errors, self.cfg['max_failing_lapack_tests_num_errors']) + else: + raise EasyBuildError("Failed to find test summary using pattern '%s' in test output: %s", + test_summary_pattern, test_output) + def test_step(self): """ Mandatory test step plus optional runtest""" run_tests = ['tests'] + + if self.cfg['run_lapack_tests']: + run_tests += [LAPACK_TEST_TARGET] + if self.cfg['runtest']: run_tests += [self.cfg['runtest']] @@ -86,6 +141,10 @@ def test_step(self): # Raise an error if any test failed check_log_for_errors(out, [('FATAL ERROR', ERROR)]) + # check number of failing LAPACK tests more closely + if runtest == LAPACK_TEST_TARGET: + self.check_lapack_test_results(out) + def sanity_check_step(self): """ Custom sanity check for OpenBLAS """ custom_paths = { diff --git a/easybuild/easyblocks/o/opencv.py b/easybuild/easyblocks/o/opencv.py index 7d18d0906e..035fd751a3 100644 --- a/easybuild/easyblocks/o/opencv.py +++ b/easybuild/easyblocks/o/opencv.py @@ -165,7 +165,7 @@ def configure_step(self): # see https://github.com/opencv/opencv/wiki/CPU-optimizations-build-options if self.toolchain.options.get('optarch') and 'CPU_BASELINE' not in self.cfg['configopts']: optarch = build_option('optarch') - if optarch is None: + if not optarch: # optimize for host arch (let OpenCV detect it) self.cfg.update('configopts', '-DCPU_BASELINE=DETECT') elif optarch == OPTARCH_GENERIC: diff --git a/easybuild/easyblocks/o/openfoam.py b/easybuild/easyblocks/o/openfoam.py index bbdfc4e29e..82327581b0 100644 --- a/easybuild/easyblocks/o/openfoam.py +++ b/easybuild/easyblocks/o/openfoam.py @@ -78,7 +78,11 @@ def __init__(self, *args, **kwargs): self.looseversion = LooseVersion(clean_version) - if 'extend' in self.name.lower(): + self.is_extend = 'extend' in self.name.lower() + self.is_dot_com = self.looseversion >= LooseVersion('1606') + self.is_dot_org = self.looseversion <= LooseVersion('100') + + if self.is_extend: if self.looseversion >= LooseVersion('3.0'): self.openfoamdir = 'foam-extend-%s' % self.version else: @@ -153,7 +157,7 @@ def configure_step(self): extra_flags = '-fuse-ld=bfd' # older versions of OpenFOAM-Extend require -fpermissive - if 'extend' in self.name.lower() and self.looseversion < LooseVersion('2.0'): + if self.is_extend and self.looseversion < LooseVersion('2.0'): extra_flags += ' -fpermissive' if self.looseversion < LooseVersion('3.0'): @@ -174,13 +178,13 @@ def configure_step(self): regex_subs = [(r"^(setenv|export) WM_THIRD_PARTY_USE_.*[ =].*$", r"# \g<0>")] # this does not work for OpenFOAM Extend lower than 2.0 - if 'extend' not in self.name.lower() or self.looseversion >= LooseVersion('2.0'): + if not self.is_extend or self.looseversion >= LooseVersion('2.0'): key = "WM_PROJECT_VERSION" regex_subs += [(r"^(setenv|export) %s=.*$" % key, r"export %s=%s #\g<0>" % (key, self.version))] WM_env_var = ['WM_COMPILER', 'WM_COMPILE_OPTION', 'WM_MPLIB', 'WM_THIRD_PARTY_DIR'] # OpenFOAM >= 3.0.0 can use 64 bit integers - if 'extend' not in self.name.lower() and self.looseversion >= LooseVersion('3.0'): + if not self.is_extend and self.looseversion >= LooseVersion('3.0'): WM_env_var.append('WM_LABEL_SIZE') for env_var in WM_env_var: regex_subs.append((r"^(setenv|export) (?P%s)[ =](?P.*)$" % env_var, @@ -257,14 +261,14 @@ def configure_step(self): env.setvar("WM_NCOMPPROCS", str(self.cfg['parallel'])) # OpenFOAM >= 3.0.0 can use 64 bit integers - if 'extend' not in self.name.lower() and self.looseversion >= LooseVersion('3.0'): + if not self.is_extend and self.looseversion >= LooseVersion('3.0'): if self.toolchain.options['i8']: env.setvar("WM_LABEL_SIZE", '64') else: env.setvar("WM_LABEL_SIZE", '32') # make sure lib/include dirs for dependencies are found - openfoam_extend_v3 = 'extend' in self.name.lower() and self.looseversion >= LooseVersion('3.0') + openfoam_extend_v3 = self.is_extend and self.looseversion >= LooseVersion('3.0') if self.looseversion < LooseVersion("2") or openfoam_extend_v3: self.log.debug("List of deps: %s" % self.cfg.dependencies()) for dep in self.cfg.dependencies(): @@ -297,7 +301,7 @@ def build_step(self): setup_cmake_env(self.toolchain) precmd = "source %s" % os.path.join(self.builddir, self.openfoamdir, "etc", "bashrc") - if 'extend' not in self.name.lower() and self.looseversion >= LooseVersion('4.0'): + if not self.is_extend and self.looseversion >= LooseVersion('4.0'): if self.looseversion >= LooseVersion('2006'): cleancmd = "cd $WM_PROJECT_DIR && wclean -platform -all && cd -" else: @@ -312,7 +316,7 @@ def build_step(self): 'prebuildopts': self.cfg['prebuildopts'], 'makecmd': os.path.join(self.builddir, self.openfoamdir, '%s'), } - if 'extend' in self.name.lower() and self.looseversion >= LooseVersion('3.0'): + if self.is_extend and self.looseversion >= LooseVersion('3.0'): qa = { "Proceed without compiling ParaView [Y/n]": 'Y', "Proceed without compiling cudaSolvers? [Y/n]": 'Y', @@ -340,7 +344,7 @@ def det_psubdir(self): """Determine the platform-specific installation directory for OpenFOAM.""" # OpenFOAM >= 3.0.0 can use 64 bit integers # same goes for OpenFOAM-Extend >= 4.1 - if 'extend' in self.name.lower(): + if self.is_extend: set_int_size = self.looseversion >= LooseVersion('4.1') else: set_int_size = self.looseversion >= LooseVersion('3.0') @@ -357,7 +361,7 @@ def det_psubdir(self): arch = get_cpu_architecture() if arch == AARCH64: # Variants have different abbreviations for ARM64... - if self.looseversion < LooseVersion("100"): + if self.is_dot_org: archpart = 'Arm64' else: archpart = 'ARM64' @@ -386,7 +390,7 @@ def install_step(self): # to make sure they take precedence over the libraries in the dummy subdirectory shlib_ext = get_shared_lib_ext() psubdir = self.det_psubdir() - openfoam_extend_v3 = 'extend' in self.name.lower() and self.looseversion >= LooseVersion('3.0') + openfoam_extend_v3 = self.is_extend and self.looseversion >= LooseVersion('3.0') if openfoam_extend_v3 or self.looseversion < LooseVersion("2"): libdir = os.path.join(self.installdir, self.openfoamdir, "lib", psubdir) else: @@ -410,7 +414,7 @@ def sanity_check_step(self): shlib_ext = get_shared_lib_ext() psubdir = self.det_psubdir() - openfoam_extend_v3 = 'extend' in self.name.lower() and self.looseversion >= LooseVersion('3.0') + openfoam_extend_v3 = self.is_extend and self.looseversion >= LooseVersion('3.0') if openfoam_extend_v3 or self.looseversion < LooseVersion("2"): toolsdir = os.path.join(self.openfoamdir, "applications", "bin", psubdir) libsdir = os.path.join(self.openfoamdir, "lib", psubdir) @@ -422,26 +426,36 @@ def sanity_check_step(self): # some randomly selected binaries # if one of these is missing, it's very likely something went wrong + tools = ["boundaryFoam", "engineFoam", "buoyantSimpleFoam", "buoyantBoussinesqSimpleFoam", "sonicFoam"] + tools += ["surfaceAdd", "surfaceFind", "surfaceSmooth"] + tools += ["blockMesh", "checkMesh", "deformedGeom", "engineSwirl", "modifyMesh", "refineMesh"] + + # surfaceSmooth is replaced by surfaceLambdaMuSmooth is OpenFOAM v2.3.0 + if not self.is_extend and self.looseversion >= LooseVersion("2.3.0"): + tools.remove("surfaceSmooth") + tools.append("surfaceLambdaMuSmooth") + # sonicFoam and buoyantBoussineqSimpleFoam deprecated in version 7+ + if self.is_dot_org and self.looseversion >= LooseVersion('7'): + tools.remove("buoyantBoussinesqSimpleFoam") + tools.remove("sonicFoam") + # buoyantSimpleFoam replaced by buoyantFoam in versions 10+ + if self.is_dot_org and self.looseversion >= LooseVersion("10"): + tools.remove("buoyantSimpleFoam") + tools.append("buoyantFoam") + # engineFoam replaced by reactingFoam in versions 10+ + if self.is_dot_org and self.looseversion >= LooseVersion("10"): + tools.remove("engineFoam") + tools.append("reactingFoam") + bins = [os.path.join(self.openfoamdir, "bin", x) for x in ["paraFoam"]] + \ - [os.path.join(toolsdir, "buoyantSimpleFoam")] + \ - [os.path.join(toolsdir, "%sFoam" % x) for x in ["boundary", "engine"]] + \ - [os.path.join(toolsdir, "surface%s" % x) for x in ["Add", "Find", "Smooth"]] + \ - [os.path.join(toolsdir, x) for x in ['blockMesh', 'checkMesh', 'deformedGeom', 'engineSwirl', - 'modifyMesh', 'refineMesh']] + [os.path.join(toolsdir, x) for x in tools] # test setting up the OpenFOAM environment in bash shell load_openfoam_env = "source $FOAM_BASH" custom_commands = [load_openfoam_env] - # only include Boussinesq and sonic since for OpenFOAM < 7, since those solvers have been deprecated - if self.looseversion < LooseVersion('7'): - bins.extend([ - os.path.join(toolsdir, "buoyantBoussinesqSimpleFoam"), - os.path.join(toolsdir, "sonicFoam") - ]) - # check for the Pstream and scotchDecomp libraries, there must be a dummy one and an mpi one - if 'extend' in self.name.lower(): + if self.is_extend: libs = [os.path.join(libsdir, "libscotchDecomp.%s" % shlib_ext), os.path.join(libsdir, "libmetisDecomp.%s" % shlib_ext)] if self.looseversion < LooseVersion('3.2'): @@ -452,7 +466,7 @@ def sanity_check_step(self): libs.extend([os.path.join(libsdir, "libparMetisDecomp.%s" % shlib_ext)]) else: # OpenFOAM v2012 puts mpi into eb-mpi - if self.looseversion >= LooseVersion("2012"): + if self.is_dot_com and self.looseversion >= LooseVersion("2012"): mpilibssubdir = "eb-mpi" else: mpilibssubdir = "mpi" @@ -463,12 +477,7 @@ def sanity_check_step(self): [os.path.join(libsdir, "libscotchDecomp.%s" % shlib_ext)] + \ [os.path.join(libsdir, "dummy", "libscotchDecomp.%s" % shlib_ext)] - if 'extend' not in self.name.lower() and self.looseversion >= LooseVersion("2.3.0"): - # surfaceSmooth is replaced by surfaceLambdaMuSmooth is OpenFOAM v2.3.0 - bins.remove(os.path.join(toolsdir, "surfaceSmooth")) - bins.append(os.path.join(toolsdir, "surfaceLambdaMuSmooth")) - - if 'extend' not in self.name.lower() and self.looseversion >= LooseVersion("2.4.0"): + if not self.is_extend and self.looseversion >= LooseVersion("2.4.0"): # also check for foamMonitor for OpenFOAM versions other than OpenFOAM-Extend bins.append(os.path.join(self.openfoamdir, 'bin', 'foamMonitor')) @@ -484,7 +493,7 @@ def sanity_check_step(self): # run motorBike tutorial case to ensure the installation is functional (if it's available); # only for recent (>= v6.0) versions of openfoam.org variant - if self.looseversion >= LooseVersion('6') and self.looseversion < LooseVersion('100'): + if self.is_dot_org and self.looseversion >= LooseVersion('6'): openfoamdir_path = os.path.join(self.installdir, self.openfoamdir) motorbike_path = os.path.join(openfoamdir_path, 'tutorials', 'incompressible', 'simpleFoam', 'motorBike') if os.path.exists(motorbike_path): @@ -537,7 +546,7 @@ def make_module_extra(self, altroot=None, altversion=None): ] # OpenFOAM >= 3.0.0 can use 64 bit integers - if 'extend' not in self.name.lower() and self.looseversion >= LooseVersion('3.0'): + if not self.is_extend and self.looseversion >= LooseVersion('3.0'): if self.toolchain.options['i8']: env_vars += [('WM_LABEL_SIZE', '64')] else: diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index 36aa710cb2..42ed999e85 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -40,11 +40,10 @@ from easybuild.tools.filetools import symlink, apply_regex_substitutions from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.systemtools import POWER, get_cpu_architecture -from easybuild.tools.utilities import nub class EB_PyTorch(PythonPackage): - """Support for building/installing TensorFlow.""" + """Support for building/installing PyTorch.""" @staticmethod def extra_options(): @@ -264,17 +263,68 @@ def test_step(self): (tests_out, tests_ec) = super(EB_PyTorch, self).test_step(return_output_ec=True) ran_tests_hits = re.findall(r"^Ran (?P[0-9]+) tests in", tests_out, re.M) - test_cnt = sum(int(hit) for hit in ran_tests_hits) - - failed_tests = nub(re.findall(r"^(?P.*) failed!(?: Received signal: \w+)?\s*$", tests_out, re.M)) - failed_test_cnt = len(failed_tests) - - if failed_test_cnt: + test_cnt = 0 + for hit in ran_tests_hits: + test_cnt += int(hit) + + # Get matches to create clear summary report, greps for patterns like: + # FAILED (errors=10, skipped=190, expected failures=6) + # test_fx failed! + regex = r"^Ran (?P[0-9]+) tests.*$\n\nFAILED \((?P.*)\)$\n(?:^(?:(?!failed!).)*$\n)*(?P.*) failed!$" # noqa: E501 + summary_matches = re.findall(regex, tests_out, re.M) + + # Get matches to create clear summary report, greps for patterns like: + # ===================== 2 failed, 128 passed, 2 skipped, 2 warnings in 3.43s ===================== + regex = r"^=+ (?P.*) in [0-9]+\.*[0-9]*[a-zA-Z]* =+$\n(?P.*) failed!$" + summary_matches_pattern2 = re.findall(regex, tests_out, re.M) + + # Count failures and errors + def get_count_for_pattern(regex, text): + match = re.findall(regex, text, re.M) + if len(match) == 0: + return 0 + elif len(match) == 1: + return int(match[0]) + elif len(match) > 1: + # Shouldn't happen, but means something went wrong with the regular expressions. + # Throw warning, as the build might be fine, no need to error on this. + warn_msg = "Error in counting the number of test failures in the output of the PyTorch test suite.\n" + warn_msg += "Please check the EasyBuild log to verify the number of failures (if any) was acceptable." + print_warning(warn_msg) + + failure_cnt = 0 + error_cnt = 0 + # Loop over first pattern to count failures/errors: + for summary in summary_matches: + failures = get_count_for_pattern(r"^.*(?[0-9]+).*$", summary[1]) + failure_cnt += failures + errs = get_count_for_pattern(r"^.*errors=(?P[0-9]+).*$", summary[1]) + error_cnt += errs + + # Loop over the second pattern to count failures/errors + for summary in summary_matches_pattern2: + failures = get_count_for_pattern(r"^.*[^0-9](?P[0-9]+) failed.*$", summary[0]) + failure_cnt += failures + errs = get_count_for_pattern(r"^.*[^0-9](?P[0-9]+) error.*$", summary[0]) + error_cnt += errs + + # Calculate total number of unsuccesful tests + failed_test_cnt = failure_cnt + error_cnt + + if failed_test_cnt > 0: max_failed_tests = self.cfg['max_failed_tests'] - test_or_tests = 'tests' if failed_test_cnt > 1 else 'test' - msg = "%d %s (out of %d) failed:\n" % (failed_test_cnt, test_or_tests, test_cnt) - msg += '\n'.join('* %s' % t for t in sorted(failed_tests)) + failure_or_failures = 'failures' if failure_cnt > 1 else 'failure' + error_or_errors = 'errors' if error_cnt > 1 else 'error' + msg = "%d test %s, %d test %s (out of %d):\n" % ( + failure_cnt, failure_or_failures, error_cnt, error_or_errors, test_cnt + ) + for summary in summary_matches_pattern2: + msg += "{test_suite} ({failure_summary})\n".format(test_suite=summary[1], failure_summary=summary[0]) + for summary in summary_matches: + msg += "{test_suite} ({total} total tests, {failure_summary})\n".format( + test_suite=summary[2], total=summary[0], failure_summary=summary[1] + ) if max_failed_tests == 0: raise EasyBuildError(msg) @@ -287,8 +337,12 @@ def test_step(self): "are known to be flaky, or do not affect your intended usage of PyTorch.", "In case of doubt, reach out to the EasyBuild community (via GitHub, Slack, or mailing list).", ]) + # Print to console, the user should really be aware that we are accepting failing tests here... print_warning(msg) + # Also log this warning in the file log + self.log.warning(msg) + if failed_test_cnt > max_failed_tests: raise EasyBuildError("Too many failed tests (%d), maximum allowed is %d", failed_test_cnt, max_failed_tests) diff --git a/easybuild/easyblocks/s/suitesparse.py b/easybuild/easyblocks/s/suitesparse.py index 807fb835eb..2cff1575ce 100644 --- a/easybuild/easyblocks/s/suitesparse.py +++ b/easybuild/easyblocks/s/suitesparse.py @@ -42,7 +42,7 @@ from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import mkdir, write_file, adjust_permissions +from easybuild.tools.filetools import mkdir, write_file, adjust_permissions, copy_dir from easybuild.tools.modules import get_software_root from easybuild.tools.modules import get_software_libdir from easybuild.tools.systemtools import get_shared_lib_ext @@ -150,7 +150,10 @@ def install_step(self): dst = os.path.join(self.installdir, x) try: if os.path.isdir(src): - shutil.copytree(src, dst) + # symlink points to CUDA folder that is + # not created for non GPU nodes. shutil + # throws an error in this case. + copy_dir(src, dst, symlinks=True) # symlink # - dst/Lib to dst/lib # - dst/Include to dst/include diff --git a/easybuild/easyblocks/t/tensorrt.py b/easybuild/easyblocks/t/tensorrt.py index 3d5489b13c..e438658803 100644 --- a/easybuild/easyblocks/t/tensorrt.py +++ b/easybuild/easyblocks/t/tensorrt.py @@ -108,6 +108,7 @@ def extensions_step(self): 'installopts': self.cfg['installopts'], 'loc': whl_paths[0], 'prefix': self.installdir, + 'python': self.python_cmd, } # Use --no-deps to prevent pip from downloading & installing