From 6e9dfb929599bf30ff3d7d6907627647b4c948be Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 10 Jan 2019 13:17:54 +0100 Subject: [PATCH 01/29] add custom easyblock for STAR-CCM+ --- easybuild/easyblocks/s/star_ccm.py | 114 +++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 easybuild/easyblocks/s/star_ccm.py diff --git a/easybuild/easyblocks/s/star_ccm.py b/easybuild/easyblocks/s/star_ccm.py new file mode 100644 index 0000000000..4d847ad9e2 --- /dev/null +++ b/easybuild/easyblocks/s/star_ccm.py @@ -0,0 +1,114 @@ +## +# Copyright 2018-2019 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for building and installing STAR-CCM+, implemented as an easyblock +""" +import glob +import os +import tempfile + +import easybuild.tools.environment as env +from easybuild.framework.easyblock import EasyBlock +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import change_dir +from easybuild.tools.run import run_cmd + + +class EB_STAR_minus_CCM_plus_(EasyBlock): + """Support for building/installing STAR-CCM+.""" + + def __init__(self, *args, **kwargs): + """Initialise STAR-CCM+ easyblock.""" + super(EB_STAR_minus_CCM_plus_, self).__init__(*args, **kwargs) + self.starccm_subdir = None + self.starview_subdir = None + + def configure_step(self): + """No configuration procedure for STAR-CCM+.""" + pass + + def build_step(self): + """No build procedure for STAR-CCM+.""" + pass + + def install_step(self): + """Custom install procedure for STAR-CCM+.""" + cands = glob.glob("./STAR-CCM+%s_*.sh" % self.version) + if len(cands) == 1: + install_script = cands[0] + + env.setvar('CHECK_DISK_SPACE', 'OFF') + env.setvar('IATEMPDIR', tempfile.mkdtemp()) + + cmd = ' '.join([ + install_script, + "-i silent", + "-DINSTALLDIR=%s" % self.installdir, + "-DINSTALLFLEX=false", + "-DADDSYSTEMPATH=false", + ]) + run_cmd(cmd, log_all=True, simple=True) + else: + raise EasyBuildError("Found no or multiple possible matches for STAR-CCM+ installation script: %s", cands) + + def find_starccm_subdirs(self): + """Determine subdirectory of install directory in which STAR-CCM+ was installed.""" + cwd = change_dir(self.installdir) + cands = glob.glob(os.path.join(self.version + '*', 'STAR-CCM+%s*' % self.version)) + if len(cands) == 1: + self.starccm_subdir = cands[0] + self.log.info("Found STAR-CCM+ subdirectory: %s", self.starccm_subdir) + self.starview_subdir = os.path.join(os.path.dirname(self.starccm_subdir), 'STAR-View+%s' % self.version) + else: + raise EasyBuildError("Failed to determine the STAR-CCM+ subdirectory in %s: %s", self.installdir, cands) + + change_dir(cwd) + + def sanity_check_step(self): + """Custom sanity check for STAR-CCM+.""" + if self.starccm_subdir is None or self.starview_subdir is None: + self.find_starccm_subdirs() + + custom_paths = { + 'files': [os.path.join(self.installdir, self.starccm_subdir, 'star', 'bin', 'starccm+'), + os.path.join(self.installdir, self.starview_subdir, 'bin', 'starview+')], + 'dirs': [], + } + super(EB_STAR_minus_CCM_plus_, self).sanity_check_step(custom_paths=custom_paths) + + def make_module_extra(self): + """Extra statements specific to STAR-CCM+ to include in generated module file.""" + if self.starccm_subdir is None or self.starview_subdir is None: + self.find_starccm_subdirs() + + txt = super(EB_STAR_minus_CCM_plus_, self).make_module_extra() + + bin_dirs = [ + os.path.join(self.starccm_subdir, 'star', 'bin'), + os.path.join(self.starview_subdir, 'bin'), + ] + txt += self.module_generator.prepend_paths('PATH', bin_dirs) + + return txt From 2a4d1692edaef5550833514144cdb5d5805ff2b0 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Sat, 26 Feb 2022 15:52:20 +0100 Subject: [PATCH 02/29] update Siesta EasyBlock to support GCC 10+ --- easybuild/easyblocks/s/siesta.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/s/siesta.py b/easybuild/easyblocks/s/siesta.py index 4616d27fa9..4a302f1c2c 100644 --- a/easybuild/easyblocks/s/siesta.py +++ b/easybuild/easyblocks/s/siesta.py @@ -108,6 +108,12 @@ def configure_step(self): (r"^(FPPFLAGS_free_F90\s*=.*)$", r"\1 -ffree-line-length-none"), ] + gfortran_flags = '' + gcc_version = get_software_version('GCCcore') or get_software_version('GCC') + if LooseVersion(gcc_version) >= LooseVersion('10.0'): + # -fallow-argument-mismatch is required when compiling with GCC 10.x & more recent + gfortran_flags = '-fallow-argument-mismatch' + netcdff_loc = get_software_root('netCDF-Fortran') if netcdff_loc: # Needed for gfortran at least @@ -181,7 +187,7 @@ def configure_step(self): regex_subs.extend([ (r"^(LIBS\s*=).*$", r"\1 %s" % complibs), # Needed for a couple of the utils - (r"^(FFLAGS\s*=\s*).*$", r"\1 -fPIC %s" % os.environ['FCFLAGS']), + (r"^(FFLAGS\s*=\s*).*$", r"\1 -fPIC %s %s" % (os.environ['FCFLAGS'], gfortran_flags)), ]) regex_newlines.append((r"^(COMP_LIBS\s*=.*)$", r"\1\nWXML = libwxml.a")) From b7957001b4c462d61332dc6aee98c2a3497aff12 Mon Sep 17 00:00:00 2001 From: Christoph Siegert Date: Tue, 7 Jun 2022 10:07:48 +0200 Subject: [PATCH 03/29] enable shared lib for Libint 2.7+ --- easybuild/easyblocks/l/libint.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/easyblocks/l/libint.py b/easybuild/easyblocks/l/libint.py index acfaa2d1f8..f2e6fa4ba8 100644 --- a/easybuild/easyblocks/l/libint.py +++ b/easybuild/easyblocks/l/libint.py @@ -119,6 +119,9 @@ def configure_step(self): if self.cfg['with_fortran']: self.cfg.update('configopts', '-DENABLE_FORTRAN=ON') + # also build shared libraries (not enabled by default) + self.cfg.update('configopts', "-DLIBINT2_BUILD_SHARED_AND_STATIC_LIBS=ON") + # specify current directory as source directory (that contains CMakeLists.txt), # since that's the path to the unpacked source tarball for Libint library (created by 'make export') super(EB_Libint, self).configure_step(srcdir=os.getcwd()) From a0d8d62ae1eff923124c0b34723f66b97656143c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 7 Jun 2022 22:31:53 +0200 Subject: [PATCH 04/29] bump version to 4.5.6dev --- easybuild/easyblocks/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/__init__.py b/easybuild/easyblocks/__init__.py index 9e30834527..d8d9cbff8d 100644 --- a/easybuild/easyblocks/__init__.py +++ b/easybuild/easyblocks/__init__.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.5.5') +VERSION = LooseVersion('4.5.6.dev0') UNKNOWN = 'UNKNOWN' From 75ae3af71fe12a4b305e076456e9d853b4cec3e1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 9 Jun 2022 09:31:18 +0200 Subject: [PATCH 05/29] enhance PythonPackage.test_step to allow returning output + exit code of test command (and not fail on non-zero exit) --- easybuild/easyblocks/generic/pythonpackage.py | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index ba4ae7db29..cd793aa18a 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -639,8 +639,12 @@ def build_step(self): # We consider the build and install output together as downloads likely happen here if this is run self.install_cmd_output += out - def test_step(self): - """Test the built Python package.""" + def test_step(self, return_output_ec=False): + """ + Test the built Python package. + + :param return_output: return output and exit code of test command + """ if isinstance(self.cfg['runtest'], string_type): self.testcmd = self.cfg['runtest'] @@ -649,6 +653,8 @@ def test_step(self): extrapath = "" testinstalldir = None + out, ec = (None, None) + if self.testinstall: # install in test directory and export PYTHONPATH @@ -670,12 +676,24 @@ def test_step(self): if self.testcmd: testcmd = self.testcmd % {'python': self.python_cmd} - cmd = ' '.join([extrapath, self.cfg['pretestopts'], testcmd, self.cfg['testopts']]) - run_cmd(cmd, log_all=True, simple=True) + cmd = ' '.join([ + extrapath, + self.cfg['pretestopts'], + testcmd, + self.cfg['testopts'], + ]) + + if return_output_ec: + (out, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False) + else: + run_cmd(cmd, log_all=True, simple=True) if testinstalldir: remove_dir(testinstalldir) + if return_output_ec: + return (out, ec) + def install_step(self): """Install Python package to a custom path using setup.py""" From da23e044a19f02eb3c3888759c8ad5512b600c1e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 9 Jun 2022 09:31:58 +0200 Subject: [PATCH 06/29] allow some PyTorch tests to fail + print warning if one or more tests fail --- easybuild/easyblocks/p/pytorch.py | 39 +++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index 677a558e7d..0dce898e9b 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -31,15 +31,16 @@ import os import re import tempfile +import easybuild.tools.environment as env from distutils.version import LooseVersion from easybuild.easyblocks.generic.pythonpackage import PythonPackage from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option -import easybuild.tools.environment as env +from easybuild.tools.filetools import symlink, apply_regex_substitutions from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.systemtools import POWER, get_cpu_architecture -from easybuild.tools.filetools import symlink, apply_regex_substitutions +from easybuild.tools.utilities import nub class EB_PyTorch(PythonPackage): @@ -49,9 +50,10 @@ class EB_PyTorch(PythonPackage): def extra_options(): extra_vars = PythonPackage.extra_options() extra_vars.update({ - 'excluded_tests': [{}, 'Mapping of architecture strings to list of tests to be excluded', CUSTOM], - 'custom_opts': [[], 'List of options for the build/install command. Can be used to change the defaults ' + - 'set by the PyTorch EasyBlock, for example ["USE_MKLDNN=0"].', CUSTOM], + 'custom_opts': [[], "List of options for the build/install command. Can be used to change the defaults " + + "set by the PyTorch EasyBlock, for example ['USE_MKLDNN=0'].", CUSTOM], + 'excluded_tests': [{}, "Mapping of architecture strings to list of tests to be excluded", CUSTOM], + 'max_failed_tests': [10, "Maximum number of failing tests", CUSTOM], }) extra_vars['download_dep_fail'][0] = True extra_vars['sanity_pip_check'][0] = True @@ -253,7 +255,30 @@ def test_step(self): 'python': self.python_cmd, 'excluded_tests': ' '.join(excluded_tests) }) - super(EB_PyTorch, self).test_step() + + (out, ec) = super(EB_PyTorch, self).test_step(return_output_ec=True) + if ec: + print_warning("Test command had non-zero exit code (%s)!" % ec) + + ran_tests_regex = re.compile(r"^Ran (?P[0-9]+) tests in", re.M) + ran_tests_hits = ran_tests_regex.findall(out) + test_cnt = 0 + for hit in ran_tests_hits: + test_cnt += int(hit) + + failed_test_regex = re.compile(r"^(?P.*) failed!\s*$", re.M) + failed_tests = nub(failed_test_regex.findall(out)) + failed_test_cnt = len(failed_tests) + + if failed_test_cnt: + test_or_tests = 'tests' if failed_test_cnt > 1 else 'test' + msg = "%d %s (out of %d tests) failed: %s" + print_warning(msg, failed_test_cnt, test_or_tests, test_cnt, ', '.join(failed_tests)) + + max_failed_tests = self.cfg['max_failed_tests'] + if failed_test_cnt > max_failed_tests: + raise EasyBuildError("Too many failed tests (%d), maximum allowed is %d", + failed_test_cnt, max_failed_tests) def test_cases_step(self): # Make PyTorch tests not use the user home From 68bc5645e0b324d34ccfd95c24b17c21da6ec68f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 9 Jun 2022 11:15:47 +0200 Subject: [PATCH 07/29] make VEP easyblock compatible with --sanity-check-only --- easybuild/easyblocks/v/vep.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/v/vep.py b/easybuild/easyblocks/v/vep.py index 1744e8cf81..c9f5b9233c 100644 --- a/easybuild/easyblocks/v/vep.py +++ b/easybuild/easyblocks/v/vep.py @@ -30,6 +30,7 @@ import easybuild.tools.environment as env from easybuild.easyblocks.perl import get_major_perl_version from easybuild.framework.easyblock import EasyBlock +from easybuild.tools.build_log import print_warning from easybuild.tools.filetools import apply_regex_substitutions from easybuild.tools.modules import get_software_version, get_software_root from easybuild.tools.run import run_cmd @@ -115,11 +116,22 @@ def sanity_check_step(self): } if 'Bio::EnsEMBL::XS' in [ext[0] for ext in self.cfg['exts_list']]: - perl_majver = get_major_perl_version() - perl_ver = get_software_version('Perl') - perl_libpath = os.path.join('lib', 'perl' + perl_majver, 'site_perl', perl_ver) - bio_ensembl_xs_ext = os.path.join(perl_libpath, 'x86_64-linux-thread-multi', 'Bio', 'EnsEMBL', 'XS.pm') - custom_paths['files'].extend([bio_ensembl_xs_ext]) + # determine Perl version used as dependency; + # take into account that Perl module may not be loaded, for example when --sanity-check-only is used + perl_ver = None + deps = self.cfg.dependencies() + for dep in deps: + if dep['name'] == 'Perl': + perl_ver = dep['version'] + break + + if perl_ver is None: + print_warning("Failed to determine version of Perl dependency!") + else: + perl_majver = perl_ver.split('.')[0] + perl_libpath = os.path.join('lib', 'perl' + perl_majver, 'site_perl', perl_ver) + bio_ensembl_xs_ext = os.path.join(perl_libpath, 'x86_64-linux-thread-multi', 'Bio', 'EnsEMBL', 'XS.pm') + custom_paths['files'].extend([bio_ensembl_xs_ext]) custom_commands = ['vep --help'] From 2868c7feb2bbe36370c2fa0b75edbfac438a522b Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Thu, 9 Jun 2022 13:41:37 +0000 Subject: [PATCH 08/29] LD_LIBRARY_PATH, CPATH and PATH are not guaranteed to be defined, calling .split on None results in an error. --- easybuild/easyblocks/r/rosetta.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/r/rosetta.py b/easybuild/easyblocks/r/rosetta.py index e79e3ba458..072de4e5c3 100644 --- a/easybuild/easyblocks/r/rosetta.py +++ b/easybuild/easyblocks/r/rosetta.py @@ -121,9 +121,12 @@ def configure_step(self): self.log.debug("List of extra environment variables to pass down: %s" % str(env_vars)) # create user.settings file - paths = os.getenv('PATH').split(':') - ld_library_paths = os.getenv('LD_LIBRARY_PATH').split(':') - cpaths = os.getenv('CPATH').split(':') + paths = os.getenv('PATH') + paths = paths.split(':') if paths else [] + ld_library_paths = os.getenv('LD_LIBRARY_PATH') + ld_library_paths = ld_library_paths.split(':') if ld_library_paths else [] + cpaths = os.getenv('CPATH') + cpaths = cpaths.split(':') if cpaths else [] flags = [str(f).strip('-') for f in self.toolchain.variables['CXXFLAGS'].copy()] txt = '\n'.join([ From e242766b2ed6f453278543bcc19db5c91c2133b8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 10 Jun 2022 08:49:49 +0200 Subject: [PATCH 09/29] don't disable disk space check for StarCCM+ (although it makes installer exit with non-zero exit code) --- easybuild/easyblocks/s/star_ccm.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/s/star_ccm.py b/easybuild/easyblocks/s/star_ccm.py index 4d847ad9e2..1e095721c8 100644 --- a/easybuild/easyblocks/s/star_ccm.py +++ b/easybuild/easyblocks/s/star_ccm.py @@ -59,7 +59,9 @@ def install_step(self): if len(cands) == 1: install_script = cands[0] - env.setvar('CHECK_DISK_SPACE', 'OFF') + # depending of the target filesystem the check for available disk space may fail, so disable it; + # note that this makes the installer exit with non-zero exit code... + # env.setvar('CHECK_DISK_SPACE', 'OFF') env.setvar('IATEMPDIR', tempfile.mkdtemp()) cmd = ' '.join([ From 29df4f54730efff0030a1a02fbd15f263dded04f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 10 Jun 2022 08:52:57 +0200 Subject: [PATCH 10/29] tweak warning message for failed PyTorch tests for readability (one test name per line) --- easybuild/easyblocks/p/pytorch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index 0dce898e9b..44da0bf38a 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -272,8 +272,8 @@ def test_step(self): if failed_test_cnt: test_or_tests = 'tests' if failed_test_cnt > 1 else 'test' - msg = "%d %s (out of %d tests) failed: %s" - print_warning(msg, failed_test_cnt, test_or_tests, test_cnt, ', '.join(failed_tests)) + msg = "%d %s (out of %d) failed:\n%s" + print_warning(msg, failed_test_cnt, test_or_tests, test_cnt, '\n'.join('* %s' % t for t in failed_tests)) max_failed_tests = self.cfg['max_failed_tests'] if failed_test_cnt > max_failed_tests: From ec0701877418e22a6119b6b668cf0e52c2442bd8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 10 Jun 2022 08:57:04 +0200 Subject: [PATCH 11/29] make sure that list of failed PyTorch tests in warning is sorted --- easybuild/easyblocks/p/pytorch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index 44da0bf38a..1e3811383e 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -272,8 +272,9 @@ def test_step(self): if failed_test_cnt: test_or_tests = 'tests' if failed_test_cnt > 1 else 'test' + failed_tests_txt = '\n'.join('* %s' % t for t in sorted(failed_tests)) msg = "%d %s (out of %d) failed:\n%s" - print_warning(msg, failed_test_cnt, test_or_tests, test_cnt, '\n'.join('* %s' % t for t in failed_tests)) + print_warning(msg, failed_test_cnt, test_or_tests, test_cnt, failed_tests_txt) max_failed_tests = self.cfg['max_failed_tests'] if failed_test_cnt > max_failed_tests: From b2dc80685e097f0f1a9e36fa2d07b47b1fddc344 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 10 Jun 2022 10:07:49 +0200 Subject: [PATCH 12/29] take into account suggested changes for STAR-CCM+ easyblock --- easybuild/easyblocks/s/star_ccm.py | 62 ++++++++++++++++-------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/easybuild/easyblocks/s/star_ccm.py b/easybuild/easyblocks/s/star_ccm.py index 1e095721c8..23918faef2 100644 --- a/easybuild/easyblocks/s/star_ccm.py +++ b/easybuild/easyblocks/s/star_ccm.py @@ -24,15 +24,15 @@ ## """ EasyBuild support for building and installing STAR-CCM+, implemented as an easyblock + +@author: Kenneth Hoste (Ghent University) """ -import glob import os import tempfile import easybuild.tools.environment as env from easybuild.framework.easyblock import EasyBlock -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import change_dir +from easybuild.tools.filetools import change_dir, find_glob_pattern from easybuild.tools.run import run_cmd @@ -55,38 +55,42 @@ def build_step(self): def install_step(self): """Custom install procedure for STAR-CCM+.""" - cands = glob.glob("./STAR-CCM+%s_*.sh" % self.version) - if len(cands) == 1: - install_script = cands[0] - - # depending of the target filesystem the check for available disk space may fail, so disable it; - # note that this makes the installer exit with non-zero exit code... - # env.setvar('CHECK_DISK_SPACE', 'OFF') - env.setvar('IATEMPDIR', tempfile.mkdtemp()) - - cmd = ' '.join([ - install_script, - "-i silent", - "-DINSTALLDIR=%s" % self.installdir, - "-DINSTALLFLEX=false", - "-DADDSYSTEMPATH=false", - ]) - run_cmd(cmd, log_all=True, simple=True) + + install_script_pattern = "./STAR-CCM+%s_*.sh" % self.version + if self.dry_run: + install_script = install_script_pattern else: - raise EasyBuildError("Found no or multiple possible matches for STAR-CCM+ installation script: %s", cands) + install_script = find_glob_pattern(install_script_pattern) + + # depending of the target filesystem the check for available disk space may fail, so disable it; + # note that this makes the installer exit with non-zero exit code... + # env.setvar('CHECK_DISK_SPACE', 'OFF') + env.setvar('IATEMPDIR', tempfile.mkdtemp()) + + cmd = ' '.join([ + self.cfg['preinstallopts'], + install_script, + "-i silent", + "-DINSTALLDIR=%s" % self.installdir, + "-DINSTALLFLEX=false", + "-DADDSYSTEMPATH=false", + self.cfg['installopts'], + ]) + run_cmd(cmd, log_all=True, simple=True) def find_starccm_subdirs(self): """Determine subdirectory of install directory in which STAR-CCM+ was installed.""" - cwd = change_dir(self.installdir) - cands = glob.glob(os.path.join(self.version + '*', 'STAR-CCM+%s*' % self.version)) - if len(cands) == 1: - self.starccm_subdir = cands[0] - self.log.info("Found STAR-CCM+ subdirectory: %s", self.starccm_subdir) - self.starview_subdir = os.path.join(os.path.dirname(self.starccm_subdir), 'STAR-View+%s' % self.version) + starccm_subdir_pattern = os.path.join(self.version + '*', 'STAR-CCM+%s*' % self.version) + + if self.dry_run: + self.starccm_subdir = starccm_subdir_pattern else: - raise EasyBuildError("Failed to determine the STAR-CCM+ subdirectory in %s: %s", self.installdir, cands) + cwd = change_dir(self.installdir) + self.starccm_subdir = find_glob_pattern(starccm_subdir_pattern) + self.log.info("Found STAR-CCM+ subdirectory: %s", self.starccm_subdir) + change_dir(cwd) - change_dir(cwd) + self.starview_subdir = os.path.join(os.path.dirname(self.starccm_subdir), 'STAR-View+%s' % self.version) def sanity_check_step(self): """Custom sanity check for STAR-CCM+.""" From 7967efcef1c45b7c9fe828e86c252c5d5d1d2700 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 10 Jun 2022 10:44:15 +0200 Subject: [PATCH 13/29] make STAR-CCM+ easyblock compatible with --module-only --force --- easybuild/easyblocks/s/star_ccm.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/s/star_ccm.py b/easybuild/easyblocks/s/star_ccm.py index 23918faef2..c3b6658585 100644 --- a/easybuild/easyblocks/s/star_ccm.py +++ b/easybuild/easyblocks/s/star_ccm.py @@ -32,6 +32,7 @@ import easybuild.tools.environment as env from easybuild.framework.easyblock import EasyBlock +from easybuild.tools.config import build_option from easybuild.tools.filetools import change_dir, find_glob_pattern from easybuild.tools.run import run_cmd @@ -85,10 +86,18 @@ def find_starccm_subdirs(self): if self.dry_run: self.starccm_subdir = starccm_subdir_pattern else: - cwd = change_dir(self.installdir) - self.starccm_subdir = find_glob_pattern(starccm_subdir_pattern) - self.log.info("Found STAR-CCM+ subdirectory: %s", self.starccm_subdir) - change_dir(cwd) + # take into account that install directory may not exist or be totally empty, + # for example when --module-only --force is used + try: + cwd = change_dir(self.installdir) + self.starccm_subdir = find_glob_pattern(starccm_subdir_pattern) + self.log.info("Found STAR-CCM+ subdirectory: %s", self.starccm_subdir) + change_dir(cwd) + except Exception: + if build_option('module_only') and build_option('force'): + self.starccm_subdir = starccm_subdir_pattern + else: + raise self.starview_subdir = os.path.join(os.path.dirname(self.starccm_subdir), 'STAR-View+%s' % self.version) From bf7b77b2d8caea6bce1888b3d4b40432015767bd Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Fri, 10 Jun 2022 12:16:21 +0200 Subject: [PATCH 14/29] also support OpenSSL 3.0 in OpenSSL wrapper easyblock --- easybuild/easyblocks/o/openssl_wrapper.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index 015c107827..f95196c43e 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -112,6 +112,10 @@ def __init__(self, *args, **kwargs): LINUX: ('so.1.1', ), DARWIN: ('1.1.dylib', ), }, + '3.0': { + LINUX: ('so.3', ), + DARWIN: ('3.dylib', ), + }, } os_type = get_os_type() @@ -135,6 +139,7 @@ def __init__(self, *args, **kwargs): openssl_engines = { '1.0': 'engines', '1.1': 'engines-1.1', + '3.0': 'engines-3', } self.target_ssl_engine = openssl_engines[self.majmin_version] From e835499f8fe9ee13cd8ba89cf2c8d790c4c38c60 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 13 Jun 2022 15:09:22 +0000 Subject: [PATCH 15/29] Allow pybind11 to be installed as an extension --- easybuild/easyblocks/p/pybind11.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/p/pybind11.py b/easybuild/easyblocks/p/pybind11.py index 5b8d849aa8..d3f4e1f776 100644 --- a/easybuild/easyblocks/p/pybind11.py +++ b/easybuild/easyblocks/p/pybind11.py @@ -91,13 +91,18 @@ def sanity_check_step(self): # don't add user site directory to sys.path (equivalent to python -s) env.setvar('PYTHONNOUSERSITE', '1', verbose=False) # Get python includes - fake_mod_data = self.load_fake_module(purge=True) + if not self.is_extension: + # for stand-alone Python package installations (not part of a bundle of extensions), + # we need to load the fake module file, otherwise the Python package being installed + # is not "in view", and we will overlook missing dependencies... + fake_mod_data = self.load_fake_module(purge=True) cmd = "%s -c 'import pybind11; print(pybind11.get_include())'" % self.python_cmd out, ec = run_cmd(cmd, simple=False) if ec: raise EasyBuildError("Failed to get pybind11 includes!") python_include = out.strip() - self.clean_up_fake_module(fake_mod_data) + if not self.is_extension: + self.clean_up_fake_module(fake_mod_data) # Check for CMake config and includes custom_paths = { From b39ce7c92545d8847e48584e1b58f970b11b9250 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 13 Jun 2022 16:11:05 +0000 Subject: [PATCH 16/29] update comment according to @micket's suggestion --- easybuild/easyblocks/p/pybind11.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/p/pybind11.py b/easybuild/easyblocks/p/pybind11.py index d3f4e1f776..a67e87dd0f 100644 --- a/easybuild/easyblocks/p/pybind11.py +++ b/easybuild/easyblocks/p/pybind11.py @@ -92,9 +92,8 @@ def sanity_check_step(self): env.setvar('PYTHONNOUSERSITE', '1', verbose=False) # Get python includes if not self.is_extension: - # for stand-alone Python package installations (not part of a bundle of extensions), - # we need to load the fake module file, otherwise the Python package being installed - # is not "in view", and we will overlook missing dependencies... + # only load fake module for stand-alone installations (not for extensions), + # since for extension the necessary modules should already be loaded at this point fake_mod_data = self.load_fake_module(purge=True) cmd = "%s -c 'import pybind11; print(pybind11.get_include())'" % self.python_cmd out, ec = run_cmd(cmd, simple=False) From ede94cca1fad44d6f6fd94b71893d0d543807c5f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 15 Jun 2022 13:04:51 +0200 Subject: [PATCH 17/29] make sure that CMakeMakeCp uses correct build dir --- easybuild/easyblocks/generic/cmakemake.py | 4 +++- easybuild/easyblocks/generic/cmakemakecp.py | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index b9ab3f50b8..46afd2ca01 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -92,6 +92,7 @@ def __init__(self, *args, **kwargs): """Constructor for CMakeMake easyblock""" super(CMakeMake, self).__init__(*args, **kwargs) self._lib_ext = None + self.separate_build_dir = None @property def lib_ext(self): @@ -122,7 +123,8 @@ def configure_step(self, srcdir=None, builddir=None): setup_cmake_env(self.toolchain) if builddir is None and self.cfg.get('separate_build_dir', True): - builddir = create_unused_dir(self.builddir, 'easybuild_obj') + self.separate_build_dir = create_unused_dir(self.builddir, 'easybuild_obj') + builddir = self.separate_build_dir if builddir: mkdir(builddir, parents=True) diff --git a/easybuild/easyblocks/generic/cmakemakecp.py b/easybuild/easyblocks/generic/cmakemakecp.py index 049b9e9abb..bfa4229fbe 100644 --- a/easybuild/easyblocks/generic/cmakemakecp.py +++ b/easybuild/easyblocks/generic/cmakemakecp.py @@ -50,5 +50,9 @@ def configure_step(self, srcdir=None, builddir=None): def install_step(self): """Install by copying specified files and directories.""" if self.cfg.get('separate_build_dir', False): - self.cfg['start_dir'] = os.path.join(self.builddir, 'easybuild_obj') + if self.separate_build_dir: + self.cfg['start_dir'] = self.separate_build_dir + else: + self.cfg['start_dir'] = os.path.join(self.builddir, 'easybuild_obj') + return MakeCp.install_step(self) From 1f574c266a5603e6691d1e5dc2cb92a78a3b27e9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 22 Jun 2022 18:19:16 +0200 Subject: [PATCH 18/29] add more logging to install_pc_files method of OpenSSL wrapper easyblock --- easybuild/easyblocks/o/openssl_wrapper.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index f95196c43e..908313214c 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -426,15 +426,19 @@ def install_pc_files(self): pc_name_suffix = pc_name + '11' pc_exists_cmd = "pkg-config --exists %s" % pc_name_suffix if run_cmd(pc_exists_cmd, simple=True, log_ok=False, log_all=False): + self.log.info("%s exists", pc_name_suffix) pc_name = pc_name_suffix # get requires from pkg-config pc_file['requires'] = [] for require_type in ['Requires', 'Requires.private']: require_print = require_type.lower().replace('.', '-') - requires, _ = run_cmd("pkg-config --print-%s %s" % (require_print, pc_name), simple=False, log_ok=False) + pc_print_cmd = "pkg-config --print-%s %s" % (require_print, pc_name) + out, _ = run_cmd(pc_print_cmd, simple=False, log_ok=False) + self.log.info("Output of '%s': %s", pc_print_cmd, out) - if requires: + if out: + requires = out # use unsuffixed names for components provided by this wrapper for wrap_comp in openssl_components: requires = re.sub(r'^%s[0-9]+$' % wrap_comp, wrap_comp, requires, flags=re.M) @@ -450,9 +454,16 @@ def install_pc_files(self): pc_file['libs'] = "Libs: -L${libdir} -l%s" % c_lib_name pc_file['cflags'] = "Cflags: -I${includedir}" # infer private libs through pkg-config - linker_libs, _ = run_cmd("pkg-config --libs %s" % pc_name, simple=False, log_ok=False) - all_libs, _ = run_cmd("pkg-config --libs --static %s" % pc_name, simple=False, log_ok=False) - libs_priv = "%s " % all_libs.rstrip() + pc_libs_cmd = "pkg-config --libs %s" % pc_name + out, _ = run_cmd(pc_libs_cmd, simple=False, log_ok=False) + self.log.info("Output of '%s': %s", pc_libs_cmd, out) + linker_libs = out + + pc_libs_static_cmd = "pkg-config --libs --static %s" % pc_name + out, _ = run_cmd(pc_libs_static_cmd, simple=False, log_ok=False) + self.log.info("Output of '%s': %s", pc_libs_static_cmd, out) + + libs_priv = "%s " % out.rstrip() for flag in linker_libs.rstrip().split(' '): libs_priv = libs_priv.replace("%s " % flag, '') pc_file['libs'] += "\nLibs.private: %s" % libs_priv From fb6dfa022487f27a67a00f4475b286edb294ca05 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 24 Jun 2022 11:53:14 +0200 Subject: [PATCH 19/29] make WPS easyblock aware of (pre)buildopts --- easybuild/easyblocks/w/wps.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/w/wps.py b/easybuild/easyblocks/w/wps.py index 6bba7795c2..9536e24e29 100644 --- a/easybuild/easyblocks/w/wps.py +++ b/easybuild/easyblocks/w/wps.py @@ -204,7 +204,11 @@ def configure_step(self): # fetch option number based on build type option and selected build type build_type_question = r"\s*(?P[0-9]+).\s*%s\s*\(?%s\)?\s*\n" % (build_type_option, knownbuildtypes[bt]) - cmd = ' '.join([self.cfg['preconfigopts'], './configure', self.cfg['configopts']]) + cmd = ' '.join([ + self.cfg['preconfigopts'], + './configure', + self.cfg['configopts'], + ]) qa = {} no_qa = [".*compiler is.*"] std_qa = { @@ -231,8 +235,11 @@ def configure_step(self): def build_step(self): """Build in install dir using compile script.""" - - cmd = "./%s" % self.compile_script + cmd = ' '.join([ + self.cfg['prebuildopts'], + './' + self.compile_script, + self.cfg['buildopts'], + ]) run_cmd(cmd, log_all=True, simple=True) def test_step(self): From ec82d649180a2b564db08689df893b1e937f957a Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Fri, 24 Jun 2022 18:02:48 +0200 Subject: [PATCH 20/29] add -fallow-argument-mismatch option when building Siesta 4.1.5 or older with GCC 10 and more recent --- easybuild/easyblocks/s/siesta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/s/siesta.py b/easybuild/easyblocks/s/siesta.py index 4a302f1c2c..1eaf44f3f6 100644 --- a/easybuild/easyblocks/s/siesta.py +++ b/easybuild/easyblocks/s/siesta.py @@ -110,7 +110,7 @@ def configure_step(self): gfortran_flags = '' gcc_version = get_software_version('GCCcore') or get_software_version('GCC') - if LooseVersion(gcc_version) >= LooseVersion('10.0'): + if LooseVersion(gcc_version) >= LooseVersion('10.0') and LooseVersion(self.version) <= LooseVersion('4.1.5'): # -fallow-argument-mismatch is required when compiling with GCC 10.x & more recent gfortran_flags = '-fallow-argument-mismatch' From 94c917511d0588bb1e629c1ca2c0d43a788c716d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 28 Jun 2022 12:03:33 +0200 Subject: [PATCH 21/29] Bazel: Avoid writing to $HOME in sanity check --- easybuild/easyblocks/b/bazel.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/b/bazel.py b/easybuild/easyblocks/b/bazel.py index 34900207bd..e1e339ce1d 100644 --- a/easybuild/easyblocks/b/bazel.py +++ b/easybuild/easyblocks/b/bazel.py @@ -119,7 +119,18 @@ def prepare_step(self, *args, **kwargs): """Setup bazel output root""" super(EB_Bazel, self).prepare_step(*args, **kwargs) self.bazel_tmp_dir = tempfile.mkdtemp(suffix='-bazel-tmp', dir=self.builddir) - self.output_user_root = tempfile.mkdtemp(suffix='-bazel-root', dir=self.builddir) + self._make_output_user_root() + + def _make_output_user_root(self): + self._output_user_root = tempfile.mkdtemp(suffix='-bazel-root', dir=self.builddir) + + @property + def output_user_root(self): + try: + return self._output_user_root + except AttributeError: + self._make_output_user_root() + return self._output_user_root def extract_step(self): """Extract Bazel sources.""" @@ -207,6 +218,7 @@ def sanity_check_step(self): } custom_commands = [] if LooseVersion(self.version) >= LooseVersion('1.0'): - custom_commands.append("bazel --help") + # Avoid writes to $HOME + custom_commands.append("bazel --output_user_root=%s --help" % self.output_user_root) super(EB_Bazel, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) From 139c56e0d4939573836ba7a535dab1e32e15bba1 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 28 Jun 2022 13:59:26 +0200 Subject: [PATCH 22/29] Fallback to tempdir for user_root when builddir doesn't exist --- easybuild/easyblocks/b/bazel.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/b/bazel.py b/easybuild/easyblocks/b/bazel.py index e1e339ce1d..f2798184d9 100644 --- a/easybuild/easyblocks/b/bazel.py +++ b/easybuild/easyblocks/b/bazel.py @@ -122,7 +122,13 @@ def prepare_step(self, *args, **kwargs): self._make_output_user_root() def _make_output_user_root(self): - self._output_user_root = tempfile.mkdtemp(suffix='-bazel-root', dir=self.builddir) + if not os.path.isdir(self.builddir): + # Can happen on module-only or sanity-check-only runs + self.log.info("Using temporary folder for user_root as builddir doesn't exist") + dir = None # Will use the EB created temp dir + else: + dir = self.builddir + self._output_user_root = tempfile.mkdtemp(suffix='-bazel-root', dir=dir) @property def output_user_root(self): From 129cd323354c32a967be55fb4d6b6ef136724828 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 1 Jul 2022 09:30:36 +0200 Subject: [PATCH 23/29] TensorFlow: Add Abseil system dependency for 2.9+ --- easybuild/easyblocks/t/tensorflow.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index 1473b4c25b..897000245f 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -134,6 +134,7 @@ def is_version_ok(version_range): available_system_libs = { # Format: (, ): # is ':' + ('Abseil', '2.9.0:'): 'com_google_absl', ('cURL', '2.0.0:'): 'curl', ('double-conversion', '2.0.0:'): 'double_conversion', ('flatbuffers', '2.0.0:'): 'flatbuffers', From 9ba8c062bbd76664c6218d75da4a29e28f416016 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 5 Jul 2022 09:36:28 +0200 Subject: [PATCH 24/29] FFTW: Disable altivec for single-precision with GCC on PPC --- easybuild/easyblocks/f/fftw.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/f/fftw.py b/easybuild/easyblocks/f/fftw.py index 40fd232acd..e9c4de21ab 100644 --- a/easybuild/easyblocks/f/fftw.py +++ b/easybuild/easyblocks/f/fftw.py @@ -203,8 +203,8 @@ def run_all_steps(self, *args, **kwargs): comp_fam = self.toolchain.comp_family() fftw_ver = LooseVersion(self.version) if cpu_arch == POWER and comp_fam == TC_CONSTANT_GCC: - # See https://github.com/FFTW/fftw3/issues/59 which applies to GCC 5.x - 10.x - if prec == 'single' and fftw_ver <= LooseVersion('3.3.9'): + # See https://github.com/FFTW/fftw3/issues/59 which applies to GCC 5 and above + if prec == 'single': self.log.info("Disabling altivec for single precision on POWER with GCC for FFTW/%s" % self.version) prec_configopts.append('--disable-altivec') From a656c8eb68689caf6e9742593e3c320fa822f816 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 5 Jul 2022 16:35:41 +0200 Subject: [PATCH 25/29] FFTW: Restrict PPC single-precision workaround to < 3.4 Co-authored-by: ocaisa --- easybuild/easyblocks/f/fftw.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/f/fftw.py b/easybuild/easyblocks/f/fftw.py index e9c4de21ab..8594e278f2 100644 --- a/easybuild/easyblocks/f/fftw.py +++ b/easybuild/easyblocks/f/fftw.py @@ -204,7 +204,8 @@ def run_all_steps(self, *args, **kwargs): fftw_ver = LooseVersion(self.version) if cpu_arch == POWER and comp_fam == TC_CONSTANT_GCC: # See https://github.com/FFTW/fftw3/issues/59 which applies to GCC 5 and above - if prec == 'single': + # Upper bound of 3.4 (as of yet unreleased) in hope there will eventually be a fix. + if prec == 'single' and fftw_ver < LooseVersion('3.4'): self.log.info("Disabling altivec for single precision on POWER with GCC for FFTW/%s" % self.version) prec_configopts.append('--disable-altivec') From 82011bf838b81de6a5fcd07b72e0c02b33160636 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 6 Jul 2022 10:40:42 +0200 Subject: [PATCH 26/29] don't allow any failed tests by default for PyTorch, raise an error instead of printing warning if no failed tests are allowed, don't compile regex used for findall --- easybuild/easyblocks/p/pytorch.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index 1e3811383e..6ab2251740 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -53,7 +53,7 @@ def extra_options(): 'custom_opts': [[], "List of options for the build/install command. Can be used to change the defaults " + "set by the PyTorch EasyBlock, for example ['USE_MKLDNN=0'].", CUSTOM], 'excluded_tests': [{}, "Mapping of architecture strings to list of tests to be excluded", CUSTOM], - 'max_failed_tests': [10, "Maximum number of failing tests", CUSTOM], + 'max_failed_tests': [0, "Maximum number of failing tests", CUSTOM], }) extra_vars['download_dep_fail'][0] = True extra_vars['sanity_pip_check'][0] = True @@ -256,30 +256,33 @@ def test_step(self): 'excluded_tests': ' '.join(excluded_tests) }) - (out, ec) = super(EB_PyTorch, self).test_step(return_output_ec=True) - if ec: - print_warning("Test command had non-zero exit code (%s)!" % ec) + (tests_out, tests_ec) = super(EB_PyTorch, self).test_step(return_output_ec=True) - ran_tests_regex = re.compile(r"^Ran (?P[0-9]+) tests in", re.M) - ran_tests_hits = ran_tests_regex.findall(out) + ran_tests_hits = re.findall(r"^Ran (?P[0-9]+) tests in", tests_out, re.M) test_cnt = 0 for hit in ran_tests_hits: test_cnt += int(hit) - failed_test_regex = re.compile(r"^(?P.*) failed!\s*$", re.M) - failed_tests = nub(failed_test_regex.findall(out)) + failed_tests = nub(re.findall(r"^(?P.*) failed!\s*$", tests_out, re.M)) failed_test_cnt = len(failed_tests) if failed_test_cnt: + max_failed_tests = self.cfg['max_failed_tests'] + test_or_tests = 'tests' if failed_test_cnt > 1 else 'test' failed_tests_txt = '\n'.join('* %s' % t for t in sorted(failed_tests)) msg = "%d %s (out of %d) failed:\n%s" - print_warning(msg, failed_test_cnt, test_or_tests, test_cnt, failed_tests_txt) - max_failed_tests = self.cfg['max_failed_tests'] - if failed_test_cnt > max_failed_tests: - raise EasyBuildError("Too many failed tests (%d), maximum allowed is %d", - failed_test_cnt, max_failed_tests) + if max_failed_tests == 0: + raise EasyBuildError(msg, failed_test_cnt, test_or_tests, test_cnt, failed_tests_txt) + else: + print_warning(msg, failed_test_cnt, test_or_tests, test_cnt, failed_tests_txt) + + if failed_test_cnt > max_failed_tests: + raise EasyBuildError("Too many failed tests (%d), maximum allowed is %d", + failed_test_cnt, max_failed_tests) + elif tests_ec: + raise EasyBuildError("Test command had non-zero exit code (%s), but no failed tests found?!", tests_ec) def test_cases_step(self): # Make PyTorch tests not use the user home From 1c22b86ccc8270018166eb1202b9e0c509f5c9c8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 6 Jul 2022 10:41:16 +0200 Subject: [PATCH 27/29] extend warning message about failing PyTorch tests to provide some guidance --- easybuild/easyblocks/p/pytorch.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index 6ab2251740..b677e87321 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -276,6 +276,14 @@ def test_step(self): if max_failed_tests == 0: raise EasyBuildError(msg, failed_test_cnt, test_or_tests, test_cnt, failed_tests_txt) else: + msg += '\n\n' + ' '.join([ + "The PyTorch test suite is known to include some flaky tests,", + "which may fail depending on the specifics of the system or the context in which they are run.", + "For this PyTorch installation, EasyBuild allows up to %d tests to fail." % max_failed_tests, + "We recommend to double check that the failing tests listed above ", + "are known to be flaky, or do not affect your intended usage of PyTorch.", + "In case of doubt, reach out to the EasyBuild community (via GitHub, Slack, or mailing list).", + ]) print_warning(msg, failed_test_cnt, test_or_tests, test_cnt, failed_tests_txt) if failed_test_cnt > max_failed_tests: From a5b55a08b44c4f589b434ccaae0bbcf65623819f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 7 Jul 2022 09:47:36 +0200 Subject: [PATCH 28/29] clean up code by pre-formatting first part of warning/error message when some PyTorch tests have failed Co-authored-by: Alexander Grund --- easybuild/easyblocks/p/pytorch.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index b677e87321..d178aba45f 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -270,11 +270,11 @@ def test_step(self): max_failed_tests = self.cfg['max_failed_tests'] test_or_tests = 'tests' if failed_test_cnt > 1 else 'test' - failed_tests_txt = '\n'.join('* %s' % t for t in sorted(failed_tests)) - msg = "%d %s (out of %d) failed:\n%s" + msg = "%d %s (out of %d) failed:\n" % (failed_test_cnt, test_or_tests, test_cnt) + msg += '\n'.join('* %s' % t for t in sorted(failed_tests)) if max_failed_tests == 0: - raise EasyBuildError(msg, failed_test_cnt, test_or_tests, test_cnt, failed_tests_txt) + raise EasyBuildError(msg) else: msg += '\n\n' + ' '.join([ "The PyTorch test suite is known to include some flaky tests,", @@ -284,7 +284,7 @@ def test_step(self): "are known to be flaky, or do not affect your intended usage of PyTorch.", "In case of doubt, reach out to the EasyBuild community (via GitHub, Slack, or mailing list).", ]) - print_warning(msg, failed_test_cnt, test_or_tests, test_cnt, failed_tests_txt) + print_warning(msg) if failed_test_cnt > max_failed_tests: raise EasyBuildError("Too many failed tests (%d), maximum allowed is %d", From 61723abe0de426bb97e36bd901561b7c97ed93c5 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Thu, 7 Jul 2022 19:01:02 +0200 Subject: [PATCH 29/29] prepare release notes for EasyBuild v4.6.0 + bump version to 4.6.0 --- RELEASE_NOTES | 25 ++++++++++++++++++++++++- easybuild/easyblocks/__init__.py | 2 +- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index ed3a0bc25c..c36cf0920a 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -3,7 +3,30 @@ For more detailed information, please see the git log. These release notes can also be consulted at http://easybuild.readthedocs.org/en/latest/Release_notes.html. -The latest version of easybuild-easyblocks provides 245 software-specific easyblocks and 37 generic easyblocks. +The latest version of easybuild-easyblocks provides 246 software-specific easyblocks and 37 generic easyblocks. + + +v4.6.0 (July 8th 2022) +---------------------- + +feature release + +- new software-specific easyblock for STAR-CCM+ (#1613) +- minor enhancements and updates, including: + - update Siesta EasyBlock to support GCC 10+ by adding -fallow-argument-mismatch Fortran compiler option (#2690) + - enable building of shared library for Libint 2.7+ (#2738) + - allow some PyTorch tests to fail + print warning if one or more tests fail (#2742) + - also support OpenSSL 3.0 in OpenSSL wrapper easyblock (#2746) + - add more logging to install_pc_files method of OpenSSL wrapper easyblock (#2752) + - make WPS easyblock aware of (pre)buildopts (#2754) + - add Abseil system dependency for TensorFlow 2.9+ (#2757) + - disable altivec when building FFTW versions < 3.4 with single-precision with GCC on POWER (#2758) +- various bug fixes, including: + - make VEP easyblock compatible with --sanity-check-only (#2743) + - update Rosetta easyblock to take into account that $LD_LIBRARY_PATH, $CPATH, $PATH may not be defined (#2744) + - only load temporary module file during sanity check for pybind11 for stand-alone installations, so it can be installed as extension (#2747) + - make sure that CMakeMakeCp uses correct build dir (#2748) + - enhance Bazel easyblock to avoid writing to $HOME in sanity check (#2756) v4.5.5 (June 8th 2022) diff --git a/easybuild/easyblocks/__init__.py b/easybuild/easyblocks/__init__.py index d8d9cbff8d..0a561629fe 100644 --- a/easybuild/easyblocks/__init__.py +++ b/easybuild/easyblocks/__init__.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.5.6.dev0') +VERSION = LooseVersion('4.6.0') UNKNOWN = 'UNKNOWN'