From 952bdf4411e485d10382fb76c9b37fdc46a0714c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Tue, 9 May 2023 13:52:18 +0200 Subject: [PATCH 1/7] common: fix prepare-for-build.sh to properly pass extra arguments --- utils/docker/prepare-for-build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/docker/prepare-for-build.sh b/utils/docker/prepare-for-build.sh index c74e65cb628..332605cd89d 100755 --- a/utils/docker/prepare-for-build.sh +++ b/utils/docker/prepare-for-build.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2016-2022, Intel Corporation +# Copyright 2016-2023, Intel Corporation # # prepare-for-build.sh - is called inside a Docker container; prepares @@ -21,5 +21,5 @@ fi # should be preserved KEEP_TEST_CONFIG=${KEEP_TEST_CONFIG:-0} if [[ "$KEEP_TEST_CONFIG" == 0 ]]; then - ./configure-tests.sh + ./configure-tests.sh $@ fi From 52b139644c39b390633c17ea24251c4692feeac6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Tue, 9 May 2023 13:55:24 +0200 Subject: [PATCH 2/7] common: extend run-build-package.sh with extra checks Make use of extra tests for packages we have up our sleeve. Since we're running this test on Fedora, we'd have to also extend list of installed packages, for these new checks to work. Make the script a little more verbose, in the process. --- utils/docker/images/Dockerfile.fedora-37 | 9 +++++ utils/docker/run-build-package.sh | 42 +++++++++++++++++++----- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/utils/docker/images/Dockerfile.fedora-37 b/utils/docker/images/Dockerfile.fedora-37 index cef6e817458..47f6e92d56e 100644 --- a/utils/docker/images/Dockerfile.fedora-37 +++ b/utils/docker/images/Dockerfile.fedora-37 @@ -55,6 +55,7 @@ ENV TESTS_DEPS "\ libunwind-devel \ ndctl \ openssh-server \ + python3 \ strace" # packaging dependencies @@ -70,6 +71,7 @@ ENV MISC_DEPS "\ lbzip2 \ man \ python3-flake8 \ + python3-pip \ rsync \ shadow-utils \ sudo \ @@ -77,6 +79,11 @@ ENV MISC_DEPS "\ which \ xmlto" +# python scripts dependencies +ENV PYTHON_DEPS "\ + distro \ + xmlrunner" + # Copy install valgrind script COPY install-valgrind.sh install-valgrind.sh @@ -92,6 +99,8 @@ RUN dnf update -y && dnf install -y \ && ./install-valgrind.sh fedora \ && dnf clean all +RUN pip3 install $PYTHON_DEPS + # Add and switch user ENV USER pmdkuser ENV USERPASS pmdkpass diff --git a/utils/docker/run-build-package.sh b/utils/docker/run-build-package.sh index 4d8eda44342..034ae57b866 100755 --- a/utils/docker/run-build-package.sh +++ b/utils/docker/run-build-package.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2016-2021, Intel Corporation +# Copyright 2016-2023, Intel Corporation # # run-build-package.sh - is called inside a Docker container; prepares @@ -17,27 +17,37 @@ git config user.email "test@package.com" git config user.name "test package" git tag -a 1.4.99 -m "1.4" HEAD~1 || true -# Build all and run tests -cd $WORKDIR +echo "## Build package (and run basic tests)" +pushd $WORKDIR export PCHECK_OPTS="-j2 BLACKLIST_FILE=${BLACKLIST_FILE}" make -j$(nproc) $PACKAGE_MANAGER -# Install packages +echo "## Build PMDK once more (clobber from packaging process cleared out some required files)" +make -j$(nproc) + +echo "## Test built packages" +python3 $SCRIPTSDIR/packages/test-build-packages.py -r $(pwd) + +echo "## Install packages" if [[ "$PACKAGE_MANAGER" == "dpkg" ]]; then - cd $PACKAGE_MANAGER + pushd $PACKAGE_MANAGER echo $USERPASS | sudo -S dpkg --install *.deb else RPM_ARCH=$(uname -m) - cd $PACKAGE_MANAGER/$RPM_ARCH + pushd $PACKAGE_MANAGER/$RPM_ARCH echo $USERPASS | sudo -S rpm --install *.rpm fi +popd + +echo "## Test installed packages" +python3 $SCRIPTSDIR/packages/test-packages-installation.py -r $(pwd) -# Compile and run standalone test -cd $WORKDIR/utils/docker/test_package +echo "## Compile and run standalone test" +pushd $SCRIPTSDIR/test_package make -j$(nproc) LIBPMEMOBJ_MIN_VERSION=1.4 ./test_package testfile1 -# Use pmreorder installed in the system +echo "## Use pmreorder installed in the system" pmreorder_version="$(pmreorder -v)" pmreorder_pattern="pmreorder\.py .+$" (echo "$pmreorder_version" | grep -Ev "$pmreorder_pattern") && echo "pmreorder version failed" && exit 1 @@ -45,3 +55,17 @@ pmreorder_pattern="pmreorder\.py .+$" touch testfile2 touch logfile1 pmreorder -p testfile2 -l logfile1 +popd + +echo "## Run tests (against PMDK installed in the system)" +pushd $WORKDIR/src/test +make -j$(nproc) clobber +make -j$(nproc) + +# Prepare test config once more. Now, with path to PMDK set in the OS +# (rather than in the git tree) - for testing packages installed in the system. +$SCRIPTSDIR/configure-tests.sh PKG +./RUNTESTS -t check +popd + +popd From 69cc71bf0bf37f96474ea1208e0caea732c51109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Tue, 9 May 2023 14:00:38 +0200 Subject: [PATCH 3/7] common: remove redundant, not used script --- utils/docker/packages/cleanup-packages.py | 112 ---------------------- 1 file changed, 112 deletions(-) delete mode 100644 utils/docker/packages/cleanup-packages.py diff --git a/utils/docker/packages/cleanup-packages.py b/utils/docker/packages/cleanup-packages.py deleted file mode 100644 index 8a2f8c2ffcc..00000000000 --- a/utils/docker/packages/cleanup-packages.py +++ /dev/null @@ -1,112 +0,0 @@ -#!usr/bin/env python3 -# -# SPDX-License-Identifier: BSD-3-Clause -# Copyright 2020-2023, Intel Corporation - -""" -This module includes functions which clean up environment system after -installation of rpm packages from PMDK library. -""" - -from os import listdir, path, linesep -import distro -from subprocess import check_output -from argparse import ArgumentParser -import re -import json -from pathlib import Path - -PMDK_VERSION = '' -SYSTEM_ARCHITECTURE = '' - - -def get_package_version_and_system_architecture(pmdk_path): - """ - Returns packages version and system architecture from names of directories - from rpm directory. - """ - version = '' - architecture = '' - os_distro=distro.id() - - # first check if there is a json file with version and architecture - pkg_version_file_path = Path(pmdk_path).joinpath("pkgVersion.json") - if pkg_version_file_path.exists() and pkg_version_file_path.is_file(): - with open(pkg_version_file_path) as pkg_version_file: - version_from_file = {} - try: - version_from_file = json.load(pkg_version_file) - version = version_from_file.get('PMDK_VERSION', '') - architecture = version_from_file.get('SYSTEM_ARCHITECTURE', '') - except: - pass - - # if cannot read values from json file, read them from rpms: - if version == '' or architecture == '': - if os_distro != 'ubuntu': - pkg_directory = path.join(pmdk_path, 'rpm') - for elem in listdir(pkg_directory): - if '.src.rpm' in elem: - version = re.search(r'[\s]*pmdk-([\S]+).src.rpm', elem).group(1) - else: - architecture = elem - - else: - pkg_directory = path.join(pmdk_path, 'dpkg') - for elem in listdir(pkg_directory): - if '.changes' in elem: - # looks for the version number of dpkg package in dpkg package name - version = re.search(r'pmdk_*(.+)_(.+).changes', elem).group(1) - architecture = re.search(r'pmdk_*(.+)_(.+).changes', elem).group(2) - - return version, architecture - - -def uninstall_rpm_packages(pmdk_path): - """ - Removes binaries installed from packages from PMDK library. - """ - try: - rpm_to_uninstall = check_output( - 'rpm -qa | grep ' + PMDK_VERSION, cwd=pmdk_path, shell=True) - pkg_to_uninstall = rpm_to_uninstall.decode( - 'UTF-8').replace(linesep, ' ') - except: - pkg_to_uninstall = '' - - if pkg_to_uninstall: - check_output('rpm -e ' + pkg_to_uninstall, cwd=pmdk_path, shell=True) - - -def uninstall_dpkg_packages(pmdk_path): - """ - Removes binaries installed from packages from PMDK library. - """ - try: - dpkg_to_uninstall = check_output( - 'dpkg-query --show | grep ' + PMDK_VERSION + ''' | awk '{print $1}' ''', cwd=pmdk_path, shell=True) - pkg_to_uninstall = dpkg_to_uninstall.decode( - 'UTF-8').replace(linesep, ' ') - except: - pkg_to_uninstall = '' - - if pkg_to_uninstall: - check_output('dpkg -r ' + pkg_to_uninstall, cwd=pmdk_path, shell=True) - - -if __name__ == '__main__': - parser = ArgumentParser( - description='''Clean up system environment after installation rpm - packages from PMDK library''') - parser.add_argument("-r", "--pmdk-path", required=True, - help="the PMDK library root path.") - - args = parser.parse_args() - PMDK_VERSION, SYSTEM_ARCHITECTURE =\ - get_package_version_and_system_architecture(args.pmdk_path) - - os_distro=distro.id() - if os_distro != 'ubuntu': - uninstall_rpm_packages(args.pmdk_path) - else: - uninstall_dpkg_packages(args.pmdk_path) From 37693b689c7413eb386ae93833b9e1b182a134f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Mon, 8 May 2023 16:54:04 +0200 Subject: [PATCH 4/7] common: add param to package testing script to skip daxio --- utils/docker/packages/test-build-packages.py | 18 +++++++++++++++--- utils/docker/run-build-package.sh | 3 ++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/utils/docker/packages/test-build-packages.py b/utils/docker/packages/test-build-packages.py index 82247028e2e..5d96c772009 100644 --- a/utils/docker/packages/test-build-packages.py +++ b/utils/docker/packages/test-build-packages.py @@ -10,8 +10,12 @@ -if all required packages are built, -if built packages are consistent with names of libraries read from .so files and other elements (tools and "PMDK"). + Required arguments: -r the PMDK library root path. + +Optional arguments: +--skip-daxio to be set if daxio was not built (e.g., if NDCTL_ENABLE was set to 'n') """ from os import listdir, path, linesep @@ -148,10 +152,12 @@ def find_missing_packages(packages_path, pmdk_path, pmdk_debuginfo_package_name) 'pmempool': PACKAGES_INFO(basic=True, devel=False, debug=False, debuginfo=True, debug_debuginfo=False), 'pmreorder': PACKAGES_INFO(basic=True, devel=False, debug=False, - debuginfo=False, debug_debuginfo=False), - 'daxio': PACKAGES_INFO(basic=True, devel=False, debug=False, - debuginfo=True, debug_debuginfo=False) + debuginfo=False, debug_debuginfo=False) } + if not skip_daxio: + tools['daxio'] = PACKAGES_INFO(basic=True, devel=False, debug=False, + debuginfo=True, debug_debuginfo=False) + tools_packages = get_names_of_packages(tools) missing_tools_packages = [ elem for elem in tools_packages if elem not in built_packages] @@ -237,11 +243,17 @@ def test_completeness_of_name_of_libraries_and_tools(self): if __name__ == '__main__': path_argument = '-r' + daxio_build_argument = "--skip-daxio" + skip_daxio = False if '-h' in sys.argv or '--help' in sys.argv: print(__doc__) unittest.main() elif path_argument in sys.argv: pmdk_path = parse_argument(path_argument) + if daxio_build_argument in sys.argv: + skip_daxio = True + index = sys.argv.index(daxio_build_argument) + sys.argv.pop(index) if pmdk_path: PMDK_VERSION, SYSTEM_ARCHITECTURE =\ get_package_version_and_system_architecture(pmdk_path) diff --git a/utils/docker/run-build-package.sh b/utils/docker/run-build-package.sh index 034ae57b866..a06b8c85c49 100755 --- a/utils/docker/run-build-package.sh +++ b/utils/docker/run-build-package.sh @@ -26,7 +26,8 @@ echo "## Build PMDK once more (clobber from packaging process cleared out some r make -j$(nproc) echo "## Test built packages" -python3 $SCRIPTSDIR/packages/test-build-packages.py -r $(pwd) +[ "$NDCTL_ENABLE" == "n" ] && extra_params="--skip-daxio" || extra_params="" +python3 $SCRIPTSDIR/packages/test-build-packages.py -r $(pwd) ${extra_params} echo "## Install packages" if [[ "$PACKAGE_MANAGER" == "dpkg" ]]; then From c58670b14007480c7be960b0eadba514def12385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Tue, 9 May 2023 14:14:16 +0200 Subject: [PATCH 5/7] common: remove (now redundant) second script to test packages 'build-and-test-pmdk-packages.sh' script was using the extra checks for packages, but now they are all part of 'run-build-package.sh'. Packages GHA workflow now use the old 'MAKE_PKG' command. --- .github/workflows/pkg_linux.yml | 2 +- utils/docker/build-CI.sh | 1 - utils/docker/build-and-test-pmdk-packages.sh | 67 ------- utils/docker/packages/install-packages.py | 188 ------------------- 4 files changed, 1 insertion(+), 257 deletions(-) delete mode 100755 utils/docker/build-and-test-pmdk-packages.sh delete mode 100644 utils/docker/packages/install-packages.py diff --git a/.github/workflows/pkg_linux.yml b/.github/workflows/pkg_linux.yml index ee2a84d7d64..230dfe5c1ad 100644 --- a/.github/workflows/pkg_linux.yml +++ b/.github/workflows/pkg_linux.yml @@ -23,7 +23,7 @@ jobs: PMDK_CXX: g++ VALGRIND: 1 SRC_CHECKERS: 0 - CHECK_PKGS: 1 + MAKE_PKG: 1 steps: - name: Clone the git repo diff --git a/utils/docker/build-CI.sh b/utils/docker/build-CI.sh index 4498d3dc574..ce2313536a4 100755 --- a/utils/docker/build-CI.sh +++ b/utils/docker/build-CI.sh @@ -38,7 +38,6 @@ if [[ $MAKE_PKG -eq 1 ]] ; then command="./run-build-package.sh"; fi if [[ $COVERAGE -eq 1 ]] ; then command="./run-coverage.sh"; fi if [[ $BANDIT -eq 1 ]] ; then command="./run-bandit.sh"; fi if [[ "$COVERITY" -eq 1 ]]; then command="./run-coverity.sh"; fi -if [[ $CHECK_PKGS -eq 1 ]] ; then command="./build-and-test-pmdk-packages.sh"; fi if [ -n "$DNS_SERVER" ]; then DNS_SETTING=" --dns=$DNS_SERVER "; fi if [[ -f $CI_FILE_SKIP_BUILD_PKG_CHECK ]]; then BUILD_PACKAGE_CHECK=n; else BUILD_PACKAGE_CHECK=y; fi diff --git a/utils/docker/build-and-test-pmdk-packages.sh b/utils/docker/build-and-test-pmdk-packages.sh deleted file mode 100755 index 224f54bc627..00000000000 --- a/utils/docker/build-and-test-pmdk-packages.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: BSD-3-Clause -# Copyright 2023, Intel Corporation - -# build-and-test-pmdk-packages.sh - is called inside a Docker container; Script for building and testing pmdk packages. - -set -eo pipefail - -# Prepare build environment -# Adding 'PKG' parameter is required for PKG tests. -./prepare-for-build.sh PKG - -# build_pmdk_package -- build pmdk package depending on the Linux distribution. -function build_pmdk_package { - cd $WORKDIR - echo "********** make package **********" - date - if [ $OS = opensuse/leap ] || [ $OS = rockylinux/rockylinux ]; then - make rpm EXPERIMENTAL=y BUILD_PACKAGE_CHECK=n PMEM2_INSTALL=y - elif [ $OS = ubuntu ]; then - make dpkg EXPERIMENTAL=y BUILD_PACKAGE_CHECK=n PMEM2_INSTALL=y - fi -} - -# Build pmdk to create src/nondebug directory used by python scripts in tests. -function build_pmdk { - cd $WORKDIR - echo "********** make pmdk **********" - make EXTRA_CFLAGS=-DUSE_VALGRIND -} - -# test_built_packages -- test if the packages are built correctly. -function test_built_packages { - echo "********** test built packages **********" - python3 $SCRIPTSDIR/packages/test-build-packages.py -r $(pwd) -} - -# install_packages -- install built packages. -function install_packages { - echo "********** install packages **********" - sudo python3 $SCRIPTSDIR/packages/install-packages.py -r $(pwd) -} - -# test_installed_packages -- test if the packages were installed correctly. -function test_installed_packages { - echo "********** test installed packages **********" - python3 $SCRIPTSDIR/packages/test-packages-installation.py -r $(pwd) -} - -function build_unittests { - echo "********** build unittests **********" - cd $WORKDIR/src/test && make clobber - make test -} - -function run_check_unittests { - echo "********** run tests **********" - cd $WORKDIR/src/test && ./RUNTESTS -t check -o 60m -} - -build_pmdk_package -build_pmdk -test_built_packages -install_packages -test_installed_packages -build_unittests -run_check_unittests diff --git a/utils/docker/packages/install-packages.py b/utils/docker/packages/install-packages.py deleted file mode 100644 index b79b37cd2c3..00000000000 --- a/utils/docker/packages/install-packages.py +++ /dev/null @@ -1,188 +0,0 @@ -#!usr/bin/env python3 -# -# SPDX-License-Identifier: BSD-3-Clause -# Copyright 2020-2023, Intel Corporation - -""" -This module includes functions which install packages from PMDK library. -""" - -from os import listdir, path, linesep -from subprocess import check_output -from argparse import ArgumentParser -import distro -import re -import json - -PMDK_VERSION = '' -SYSTEM_ARCHITECTURE = '' - -def get_package_version_and_system_architecture(pmdk_path): - """ - Returns packages version and system architecture from names of directories - from packages directory. - """ - os_distro=distro.id() - if os_distro != 'ubuntu': - pkg_directory = path.join(pmdk_path, 'rpm') - else: - pkg_directory = path.join(pmdk_path, 'dpkg') - - version = '' - architecture = '' - for elem in listdir(pkg_directory): - if os_distro != 'ubuntu': - if '.src.rpm' in elem: - # looks for the version number of package in package name - version = re.search(r'[\s]*pmdk-([\S]+).src.rpm', elem).group(1) - else: - architecture = elem - - elif os_distro == 'ubuntu': - if '.changes' in elem: - # looks for the version number of packages in package name - version = re.search(r'pmdk_*(.+)_(.+).changes', elem).group(1) - architecture = re.search(r'pmdk_*(.+)_(.+).changes', elem).group(2) - return version, architecture - - -def save_pkg_version(pkg_version_file_path): - values_to_save = { - 'PMDK_VERSION': PMDK_VERSION, - 'SYSTEM_ARCHITECTURE': SYSTEM_ARCHITECTURE - } - with open(pkg_version_file_path, 'w') as file: - json.dump(values_to_save, file, indent=2) - - -def get_built_rpm_packages(pmdk_path): - """ - Returns built pkg packages from pkg directory. - """ - path_to_rpm_files = path.join(pmdk_path, 'rpm', SYSTEM_ARCHITECTURE) - packages = listdir(path_to_rpm_files) - return packages - - -def get_built_dpkg_packages(pmdk_path): - """ - Returns built pkg packages from pkg directory. - """ - path_to_dpkg_files = path.join(pmdk_path, 'dpkg') - packages ='' - for elem in listdir(path_to_dpkg_files): - if '.deb' in elem: - packages += elem + ' ' - - return packages - - -def install_rpm_packages(pmdk_path): - """ - Install packages from PMDK library. - """ - packages = ' '.join(get_built_rpm_packages(pmdk_path)) - path_to_rpm_files = path.join(pmdk_path, 'rpm', SYSTEM_ARCHITECTURE) - check_output('rpm -i --nodeps ' + packages, - cwd=path_to_rpm_files, shell=True) - - -def install_dpkg_packages(pmdk_path): - """ - Install packages from PMDK library. - """ - packages = get_built_dpkg_packages(pmdk_path) - path_to_dpkg_files = path.join(pmdk_path, 'dpkg') - check_output('dpkg -i ' + packages, - cwd=path_to_dpkg_files, shell=True) - - -def get_names_of_rpm_content(): - """ - Returns names of elements, for which are installed from packages from PMDK - library. - """ - packages_path = path.join(args.pmdk_path, 'rpm', SYSTEM_ARCHITECTURE) - installed_packages = check_output( - 'ls | grep ' + PMDK_VERSION, cwd=packages_path, shell=True) - delimiter = '-' - - installed_packages = installed_packages.decode( - 'UTF-8').split(linesep) - - libraries_names = [item.split( - '-')[0] for item in installed_packages if item.split('-')[0]] - return set(libraries_names) - - -def get_names_of_dpkg_content(): - """ - Returns names of elements, for which are installed from packages from PMDK - library. - """ - packages_path = path.join(args.pmdk_path, 'dpkg') - installed_packages = check_output( - 'ls | grep ' + PMDK_VERSION + ' | grep .deb', cwd=packages_path, shell=True) - - installed_packages = installed_packages.decode( - 'UTF-8').split(linesep) - - libraries_names = [item.split( - '_')[0] for item in installed_packages if item.split('-')[0]] - return set(libraries_names) - - -def get_installed_packages(so_path, split_param, packages_path): - """ - Returns names of packages from PMDK library, which are installed. - """ - libraries = get_names_of_pkg_content_func() - installed_packages = [] - - for library in libraries: - if library == "pmempool" and check_output( - 'find /usr/bin/ -name ' + library, cwd=args.pmdk_path, - shell=True): - installed_packages.append(library) - elif library == "libpmemobj++" and check_output( - 'find /usr/include/' + library + ' -name *.hpp', - cwd=args.pmdk_path, shell=True): - installed_packages.append(library) - elif library == "pmdk": - pass - elif check_output('find ' + so_path + ' -name ' + library + '.so', - cwd=args.pmdk_path, shell=True): - installed_packages.append(library) - return installed_packages - - -if __name__ == '__main__': - parser = ArgumentParser( - description='Install packages from PMDK library') - parser.add_argument("-r", "--pmdk-path", required=True, - help="the PMDK library root path.") - - args = parser.parse_args() - os_distro=distro.id() - PMDK_VERSION, SYSTEM_ARCHITECTURE =\ - get_package_version_and_system_architecture(args.pmdk_path) - save_pkg_version(args.pmdk_path + "/pkgVersion.json") - if os_distro != 'ubuntu': - so_path = '/usr/lib64/' - split_param = '-' - packages_path = path.join(args.pmdk_path, 'rpm', SYSTEM_ARCHITECTURE) - install_cmd = 'rpm -i --nodeps ' - install_func = install_rpm_packages - get_names_of_pkg_content_func = get_names_of_rpm_content - else: - so_path = '/lib/x86_64-linux-gnu/' - split_param = '_' - packages_path = path.join(args.pmdk_path, 'dpkg') - install_cmd = 'dpkg -i ' - install_func = install_dpkg_packages - get_names_of_pkg_content_func = get_names_of_dpkg_content - - if not get_installed_packages(so_path, packages_path, get_names_of_pkg_content_func): - install_func(args.pmdk_path) - else: - print("PMDK library is still installed") From 36670d5a9e3f8576ea72c2ed65c0ad1c44ec0180 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Tue, 9 May 2023 14:24:29 +0200 Subject: [PATCH 6/7] common: move all MAKE_PKG checks to nightly build update RockyLinux 8 image to work with new MAKE_PKG scripts. Don't execute rpm checks on Fedora - it's duplicated now by RockyLinux --- .github/workflows/gha.yml | 9 ++--- .github/workflows/nightly.yml | 7 ++-- .github/workflows/pkg_linux.yml | 39 --------------------- utils/docker/images/Dockerfile.rockylinux-8 | 8 +++++ 4 files changed, 16 insertions(+), 47 deletions(-) delete mode 100644 .github/workflows/pkg_linux.yml diff --git a/.github/workflows/gha.yml b/.github/workflows/gha.yml index 3eed7399709..3cf419c0066 100644 --- a/.github/workflows/gha.yml +++ b/.github/workflows/gha.yml @@ -26,16 +26,13 @@ jobs: SRC_CHECKERS: 0 strategy: matrix: - CONFIG: ["N=1 OS=ubuntu OS_VER=22.04 FAULT_INJECTION=1 TEST_BUILD=debug", + CONFIG: ["N=1 OS=ubuntu OS_VER=22.04 FAULT_INJECTION=1 TEST_BUILD=debug PUSH_IMAGE=1", "N=2 OS=ubuntu OS_VER=22.04 FAULT_INJECTION=1 TEST_BUILD=nondebug UBSAN=1", "N=3 OS=ubuntu OS_VER=22.04 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=debug SRC_CHECKERS=1", "N=4 OS=ubuntu OS_VER=22.04 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=nondebug", - "N=5 OS=fedora OS_VER=37 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=debug", + "N=5 OS=fedora OS_VER=37 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=debug PUSH_IMAGE=1", "N=6 OS=fedora OS_VER=37 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=nondebug AUTO_DOC_UPDATE=1", - "N=7 OS=fedora OS_VER=37 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 PUSH_IMAGE=1", - "N=8 OS=ubuntu OS_VER=22.04 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 NDCTL_ENABLE=n PUSH_IMAGE=1", - "N=9 OS=ubuntu OS_VER=22.04 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 NDCTL_ENABLE=n PMDK_CC=clang PMDK_CXX=clang++", - "N=10 OS=ubuntu OS_VER=22.04 COVERAGE=1 FAULT_INJECTION=1 TEST_BUILD=debug" + "N=7 OS=ubuntu OS_VER=22.04 COVERAGE=1 FAULT_INJECTION=1 TEST_BUILD=debug" ] steps: - name: Clone the git repo diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 95e1cf6f7fd..c6ee3369a38 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -30,10 +30,13 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - CONFIG: ["OS=debian OS_VER=11 FAULT_INJECTION=1 TEST_BUILD=debug", + CONFIG: ["OS=ubuntu OS_VER=22.04 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 NDCTL_ENABLE=n", + "OS=ubuntu OS_VER=22.04 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 NDCTL_ENABLE=n PMDK_CC=clang PMDK_CXX=clang++", + "OS=debian OS_VER=11 FAULT_INJECTION=1 TEST_BUILD=debug", "OS=debian OS_VER=11 FAULT_INJECTION=1 TEST_BUILD=nondebug PUSH_IMAGE=1", "OS=opensuse-leap OS_VER=15 FAULT_INJECTION=1 TEST_BUILD=debug", - "OS=opensuse-leap OS_VER=15 FAULT_INJECTION=1 TEST_BUILD=nondebug PUSH_IMAGE=1", + "OS=opensuse-leap OS_VER=15 FAULT_INJECTION=1 TEST_BUILD=nondebug", + "OS=opensuse-leap OS_VER=15 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 PUSH_IMAGE=1", "OS=rockylinux OS_VER=8 TEST_BUILD=debug", "OS=rockylinux OS_VER=8 TEST_BUILD=nondebug", "OS=rockylinux OS_VER=8 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 PUSH_IMAGE=1", diff --git a/.github/workflows/pkg_linux.yml b/.github/workflows/pkg_linux.yml deleted file mode 100644 index 230dfe5c1ad..00000000000 --- a/.github/workflows/pkg_linux.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Run extra checks for making package. -name: PKG_linux - -on: [push, pull_request] - -jobs: - linux: - name: PKG_Linux - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - CONFIG: ["OS=ubuntu OS_VER=22.04", - "OS=rockylinux OS_VER=9", - "OS=opensuse-leap OS_VER=15" - ] - env: - GITHUB_REPO: pmem/pmdk - DOCKER_REPO: ghcr.io/pmem/pmdk - HOST_WORKDIR: /home/runner/work/pmdk/pmdk - WORKDIR: utils/docker - PMDK_CC: gcc - PMDK_CXX: g++ - VALGRIND: 1 - SRC_CHECKERS: 0 - MAKE_PKG: 1 - - steps: - - name: Clone the git repo - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Pull or rebuild the image - run: cd $WORKDIR && ${{ matrix.CONFIG }} ./pull-or-rebuild-image.sh - - # This step will create docker containers, build pmdk packages and test it. - - name: Build the CI and test packages - run: cd $WORKDIR && ${{ matrix.CONFIG }} ./build-CI.sh diff --git a/utils/docker/images/Dockerfile.rockylinux-8 b/utils/docker/images/Dockerfile.rockylinux-8 index 03a2daab5f8..b562f6a13ef 100644 --- a/utils/docker/images/Dockerfile.rockylinux-8 +++ b/utils/docker/images/Dockerfile.rockylinux-8 @@ -71,6 +71,7 @@ ENV MISC_DEPS "\ man \ python3 \ python3-flake8 \ + python3-pip \ rsync \ shadow-utils \ sudo \ @@ -78,6 +79,11 @@ ENV MISC_DEPS "\ which \ xmlto" +# python scripts dependencies +ENV PYTHON_DEPS "\ + distro \ + xmlrunner" + # Copy install valgrind script COPY install-valgrind.sh install-valgrind.sh @@ -96,6 +102,8 @@ RUN dnf update -y && dnf install -y epel-release \ && ./install-valgrind.sh rockylinux \ && dnf clean all +RUN pip3 install $PYTHON_DEPS + # Add ans switch user ENV USER pmdkuser ENV USERPASS pmdkpass From 61fb7b526463d5ac07414d4e756666fee7ca4227 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Tue, 9 May 2023 14:29:57 +0200 Subject: [PATCH 7/7] common: move packages' tests into utils/docker/test_packages dir --- utils/docker/run-build-package.sh | 4 ++-- utils/docker/test_package/README | 7 +++++-- .../test-built-packages.py} | 0 .../test-packages-installation.py | 0 4 files changed, 7 insertions(+), 4 deletions(-) rename utils/docker/{packages/test-build-packages.py => test_package/test-built-packages.py} (100%) rename utils/docker/{packages => test_package}/test-packages-installation.py (100%) diff --git a/utils/docker/run-build-package.sh b/utils/docker/run-build-package.sh index a06b8c85c49..c211441a524 100755 --- a/utils/docker/run-build-package.sh +++ b/utils/docker/run-build-package.sh @@ -27,7 +27,7 @@ make -j$(nproc) echo "## Test built packages" [ "$NDCTL_ENABLE" == "n" ] && extra_params="--skip-daxio" || extra_params="" -python3 $SCRIPTSDIR/packages/test-build-packages.py -r $(pwd) ${extra_params} +python3 $SCRIPTSDIR/test_package/test-built-packages.py -r $(pwd) ${extra_params} echo "## Install packages" if [[ "$PACKAGE_MANAGER" == "dpkg" ]]; then @@ -41,7 +41,7 @@ fi popd echo "## Test installed packages" -python3 $SCRIPTSDIR/packages/test-packages-installation.py -r $(pwd) +python3 $SCRIPTSDIR/test_package/test-packages-installation.py -r $(pwd) echo "## Compile and run standalone test" pushd $SCRIPTSDIR/test_package diff --git a/utils/docker/test_package/README b/utils/docker/test_package/README index de81a6a5b8e..e4015df0f78 100644 --- a/utils/docker/test_package/README +++ b/utils/docker/test_package/README @@ -2,5 +2,8 @@ Persistent Memory Development Kit This is utils/docker/test_package/README. -This directory contains simple application which uses libpmemobj. -It can be used to test whether libpmemobj was installed properly. +This directory contains: +- Simple application which uses libpmemobj. + It can be used to test whether libpmemobj was installed properly. +- Test python modules with various checks for PMDK packages. + Each module contains a detailed description of its contents. diff --git a/utils/docker/packages/test-build-packages.py b/utils/docker/test_package/test-built-packages.py similarity index 100% rename from utils/docker/packages/test-build-packages.py rename to utils/docker/test_package/test-built-packages.py diff --git a/utils/docker/packages/test-packages-installation.py b/utils/docker/test_package/test-packages-installation.py similarity index 100% rename from utils/docker/packages/test-packages-installation.py rename to utils/docker/test_package/test-packages-installation.py