From 2c688f7f7642d148d374e2675559d8ef5afafbe4 Mon Sep 17 00:00:00 2001 From: Georgios Kafanas Date: Sat, 21 Dec 2024 20:43:18 +0100 Subject: [PATCH] Add test for error logging The toy test file is modified with a patch to fail during compilation. The tests verify that: - the source directory is copied to the error log path, - the log files are copied to the error log path, and - a warning for the compilation failure is reported in stdout. --- .../test_ecs/t/toy/toy-0.0-buggy.eb | 33 ++++++++++ .../test_ecs/t/toy/toy-0.0-test.eb | 2 +- test/framework/filetools.py | 2 +- .../sandbox/sources/toy/toy-0.0_add-bug.patch | 10 +++ test/framework/toy_build.py | 66 +++++++++++++++++-- test/framework/utilities.py | 14 ++++ 6 files changed, 121 insertions(+), 6 deletions(-) create mode 100644 test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-buggy.eb create mode 100644 test/framework/sandbox/sources/toy/toy-0.0_add-bug.patch diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-buggy.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-buggy.eb new file mode 100644 index 0000000000..89fe5979b6 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-buggy.eb @@ -0,0 +1,33 @@ +name = 'toy' +version = '0.0' +versionsuffix = '-buggy' + +homepage = 'https://easybuilders.github.io/easybuild' +description = "Toy C program, 100% toy." + +toolchain = SYSTEM + +sources = [SOURCE_TAR_GZ] +checksums = [[ + 'be662daa971a640e40be5c804d9d7d10', # default (MD5) + '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256) + ('adler32', '0x998410035'), + ('crc32', '0x1553842328'), + ('md5', 'be662daa971a640e40be5c804d9d7d10'), + ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'), + ('size', 273), +]] +patches = [ + 'toy-0.0_add-bug.patch', + ('toy-extra.txt', 'toy-0.0'), +] + +sanity_check_paths = { + 'files': [('bin/yot', 'bin/toy')], + 'dirs': ['bin'], +} + +postinstallcmds = ["echo TOY > %(installdir)s/README"] + +moduleclass = 'tools' +# trailing comment, leave this here, it may trigger bugs with extract_comments() diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb index 90cc7429d3..9184a55d23 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb @@ -18,7 +18,7 @@ checksums = [[ ('size', 273), ]] patches = [ - 'toy-0.0_fix-silly-typo-in-printf-statement.patch', + 'toy-0.0_add-bug.patch', ('toy-extra.txt', 'toy-0.0'), ] diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 2e2b030fc8..03197de67d 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2474,7 +2474,7 @@ def test_index_functions(self): # test with specified path with and without trailing '/'s for path in [test_ecs, test_ecs + '/', test_ecs + '//']: index = ft.create_index(path) - self.assertEqual(len(index), 94) + self.assertEqual(len(index), 95) expected = [ os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), diff --git a/test/framework/sandbox/sources/toy/toy-0.0_add-bug.patch b/test/framework/sandbox/sources/toy/toy-0.0_add-bug.patch new file mode 100644 index 0000000000..7512447168 --- /dev/null +++ b/test/framework/sandbox/sources/toy/toy-0.0_add-bug.patch @@ -0,0 +1,10 @@ +--- a/toy-0.0.orig/toy.source 2014-03-06 18:48:16.000000000 +0100 ++++ b/toy-0.0/toy.source 2020-08-18 12:19:35.000000000 +0200 +@@ -2,6 +2,6 @@ + + int main(int argc, char* argv[]){ + +- printf("I'm a toy, and proud of it.\n"); ++ printf("I'm a toy, and proud of it.\n") + return 0; + } diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 5219ee5aa6..4e792b636e 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -40,8 +40,10 @@ import sys import tempfile import textwrap +import pathlib +import filecmp from easybuild.tools import LooseVersion -from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, TempDirectory from test.framework.package import mock_fpm from unittest import TextTestRunner @@ -156,9 +158,9 @@ def check_toy(self, installpath, outtxt, name='toy', version='0.0', versionprefi devel_module_path = os.path.join(software_path, 'easybuild', '%s-%s-easybuild-devel' % (name, full_version)) self.assertExists(devel_module_path) - def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True, fails=False, verbose=True, - raise_error=False, test_report=None, name='toy', versionsuffix='', testing=True, - raise_systemexit=False, force=True, test_report_regexs=None, debug=True): + def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, tmp_logdir=None, verify=True, fails=False, + verbose=True, raise_error=False, test_report=None, name='toy', versionsuffix='', testing=True, + raise_systemexit=False, force=True, test_report_regexs=None, debug=True, check_errorlog=None): """Perform a toy build.""" if extra_args is None: extra_args = [] @@ -172,6 +174,7 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True '--sourcepath=%s' % self.test_sourcepath, '--buildpath=%s' % self.test_buildpath, '--installpath=%s' % self.test_installpath, + '--errorlogpath=%s' % self.test_errorlogpath, '--unittest-file=%s' % self.logfile, '--robot=%s' % os.pathsep.join([self.test_buildpath, os.path.dirname(__file__)]), ] @@ -183,6 +186,8 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True args.append('--tmpdir=%s' % tmpdir) if test_report is not None: args.append('--dump-test-report=%s' % test_report) + if tmp_logdir is not None: + args.append('--tmp-logdir=%s' % tmp_logdir) args.extend(extra_args) myerr = None try: @@ -228,6 +233,9 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True msg = "Pattern %s found in full test report: %s" % (regex.pattern, test_report_txt) self.assertTrue(regex.search(test_report_txt), msg) + if check_errorlog is not None: + check_errorlog(outtxt, tmp_logdir, self.test_buildpath, self.test_errorlogpath) + return outtxt def run_test_toy_build_with_output(self, *args, **kwargs): @@ -271,6 +279,56 @@ def test_toy_broken(self): # cleanup shutil.rmtree(tmpdir) + def assert_build_files_copied(self, buildpath, errorlogpath): + buildir = pathlib.Path(buildpath) + errorlogdir = pathlib.Path(errorlogpath) + iso_date_pattern = r'????????-??????' + for file in buildir.iterdir(): + file_relative_path = file.relative_to(buildir) + file_copies = list(errorlogdir.glob(f"toy-0.0/{iso_date_pattern}/{file_relative_path}")) + self.assertTrue(len(file_copies) == 1, f"Unique copy of toy build file '{file}' made") + for file_copy in file_copies: + if file_copy.is_file(): + msg = f"File '{file}' copied succesfully" + self.assertTrue(filecmp.cmp(str(file), str(file_copy), shallow=False), msg) + + def assert_log_files_copied(self, tmp_logpath, errorlogpath): + for log_file in pathlib.Path(tmp_logpath).glob("**/*.log"): + file_name = log_file.name + saved_log_files = list(pathlib.Path(errorlogpath).glob(f"**/{file_name}")) + self.assertTrue(len(saved_log_files) == 1, f"Unique copy of log file '{log_file}' made") + for saved_log_file in saved_log_files: + msg = f"Log file '{log_file}' copied succesfully" + self.assertTrue(filecmp.cmp(str(log_file), str(saved_log_file), shallow=False), msg) + + def assert_error_reported(self, outtxt, output_regexs): + for regex_pattern in output_regexs: + regex = re.compile(regex_pattern, re.M) + msg = "Pattern %s found in full test report: %s" % (regex.pattern, outtxt) + self.assertTrue(regex.search(outtxt), msg) + + def check_errorlog(self, outtxt, output_regexs, tmp_logpath, buildpath, errorlogpath): + self.assert_build_files_copied(buildpath, errorlogpath) + self.assert_log_files_copied(tmp_logpath, errorlogpath) + self.assert_error_reported(outtxt, output_regexs) + + def test_toy_broken_compilation(self): + """Test whether log files and compilafor broken compilations are copied to a permanent location""" + tmpdir = TempDirectory() + tmp_logdir = TempDirectory() + broken_compilation_ec = os.path.join(os.path.dirname(__file__), + 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-buggy.eb') + + output_regexs = [r'^\s+toy\.c:5:44: error: expected ;'] + + def check_errorlog(outtxt, tmp_logpath, buildpath, errorlogpath): + self.check_errorlog(outtxt, output_regexs, tmp_logpath, buildpath, errorlogpath) + + self.run_test_toy_build_with_output( + ec_file=broken_compilation_ec, tmpdir=tmpdir.get_path(), tmp_logdir=tmp_logdir.get_path(), + verify=False, fails=True, verbose=False, raise_error=False, + name='toy', versionsuffix='-buggy', check_errorlog=check_errorlog) + def test_toy_tweaked(self): """Test toy build with tweaked easyconfig, for testing extra easyconfig parameters.""" test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs') diff --git a/test/framework/utilities.py b/test/framework/utilities.py index bf44ad828d..4aeda144ca 100644 --- a/test/framework/utilities.py +++ b/test/framework/utilities.py @@ -121,6 +121,8 @@ def setUp(self): os.environ['EASYBUILD_BUILDPATH'] = self.test_buildpath self.test_installpath = tempfile.mkdtemp() os.environ['EASYBUILD_INSTALLPATH'] = self.test_installpath + self.test_errorlogpath = tempfile.mkdtemp() + os.environ['EASYBUILD_ERRORLOGPATH'] = self.test_errorlogpath # make sure that the tests only pick up easyconfigs provided with the tests os.environ['EASYBUILD_ROBOT_PATHS'] = os.path.join(testdir, 'easyconfigs', 'test_ecs') @@ -514,3 +516,15 @@ def find_full_path(base_path, trim=(lambda x: x)): break return full_path + + +class TempDirectory: + def __init__(self): + self.dir = tempfile.mkdtemp() + + def __del__(self): + if os.path.exists(self.dir): + shutil.rmtree(self.dir) + + def get_path(self): + return self.dir