Merge pull request #6013 from matejak/quick_tests

Introduce quick tests
ComplianceAsCode · Aug 20, 2020 · a328fe7 · a328fe7
2 parents 00dc6ef + 2db38ce
commit a328fe7
Show file tree

Hide file tree

Showing 4 changed files with 32 additions and 26 deletions.
diff --git a/build_product b/build_product
@@ -8,14 +8,14 @@
 # ARG_OPTIONAL_BOOLEAN([debug],[],[Make a debug build with draft profiles],[off])
 # ARG_OPTIONAL_BOOLEAN([derivatives],[],[Also build derivatives of products if applicable],[off])
 # ARG_USE_ENV([ADDITIONAL_CMAKE_OPTIONS],[],[Whitespace-separated string of arguments to pass to CMake])
-# ARG_POSITIONAL_INF([product],[Products to build],[1])
+# ARG_POSITIONAL_INF([product],[Products to build, ALL means all products],[0],[ALL])
 # ARG_DEFAULTS_POS([])
 # ARG_TYPE_GROUP_SET([oval_ver],[VERSION],[oval],[5.10,5.11,auto])
 # ARG_TYPE_GROUP_SET([builder_type],[BUILDER],[builder],[make,ninja,auto])
 # ARG_HELP([Wipes out contents of the 'build' directory and builds only and only the given products.])
 # ARGBASH_GO()
 # needed because of Argbash --> m4_ignore([
-### START OF CODE GENERATED BY Argbash v2.8.1 one line above ###
+### START OF CODE GENERATED BY Argbash v2.9.0 one line above ###
 # Argbash is a bash code generator used to get arguments parsing right.
 # Argbash is FREE SOFTWARE, see https://argbash.io for more info
 
@@ -24,11 +24,10 @@
 
 die()
 {
-	local _ret=$2
-	test -n "$_ret" || _ret=1
-	test "$_PRINT_HELP" = yes && print_help >&2
+	local _ret="${2:-1}"
+	test "${_PRINT_HELP:-no}" = yes && print_help >&2
 	echo "$1" >&2
-	exit ${_ret}
+	exit "${_ret}"
 }
 
 # validators
@@ -64,7 +63,7 @@ begins_with_short_option()
 
 # THE DEFAULTS INITIALIZATION - POSITIONALS
 _positionals=()
-_arg_product=('' )
+_arg_product=("ALL")
 # THE DEFAULTS INITIALIZATION - OPTIONALS
 _arg_oval="auto"
 _arg_builder="auto"
@@ -76,8 +75,8 @@ _arg_derivatives="off"
 print_help()
 {
 	printf '%s\n' "Wipes out contents of the 'build' directory and builds only and only the given products."
-	printf 'Usage: %s [-o|--oval <VERSION>] [-b|--builder <BUILDER>] [-j|--jobs <arg>] [--(no-)debug] [--(no-)derivatives] [-h|--help] <product-1> [<product-2>] ... [<product-n>] ...\n' "$0"
-	printf '\t%s\n' "<product>: Products to build"
+	printf 'Usage: %s [-o|--oval <VERSION>] [-b|--builder <BUILDER>] [-j|--jobs <arg>] [--(no-)debug] [--(no-)derivatives] [-h|--help] [<product-1>] ... [<product-n>] ...\n' "$0"
+	printf '\t%s\n' "<product>: Products to build, ALL means all products (defaults for <product>: 'ALL')"
 	printf '\t%s\n' "-o, --oval: OVAL version. Can be one of: '5.10', '5.11' and 'auto' (default: 'auto')"
 	printf '\t%s\n' "-b, --builder: Builder engine. Can be one of: 'make', 'ninja' and 'auto' (default: 'auto')"
 	printf '\t%s\n' "-j, --jobs: Count of simultaneous jobs (default: 'auto')"
@@ -157,21 +156,14 @@ parse_commandline()
 }
 
 
-handle_passed_args_count()
-{
-	local _required_args_string="'product'"
-	test "${_positionals_count}" -ge 1 || _PRINT_HELP=yes die "FATAL ERROR: Not enough positional arguments - we require at least 1 (namely: $_required_args_string), but got only ${_positionals_count}." 1
-}
-
-
 assign_positional_args()
 {
 	local _positional_name _shift_for=$1
-	_positional_names="_arg_product "
-	_our_args=$((${#_positionals[@]} - 1))
+	_positional_names=""
+	_our_args=$((${#_positionals[@]} - 0))
 	for ((ii = 0; ii < _our_args; ii++))
 	do
-		_positional_names="$_positional_names _arg_product[$((ii + 1))]"
+		_positional_names="$_positional_names _arg_product[$((ii + 0))]"
 	done
 
 	shift "$_shift_for"
@@ -184,7 +176,6 @@ assign_positional_args()
 }
 
 parse_commandline "$@"
-handle_passed_args_count
 assign_positional_args 1 "${_positionals[@]}"
 
 # OTHER STUFF GENERATED BY Argbash
@@ -245,7 +236,7 @@ handle_wrong_products() {
 	done
 	possible_products=$'\n'"$(printf ' * %s\n' "${all_cmake_lowercase[@]}")"
 
-	printf 'Choose one or more product names from the list: %s' "$possible_products"
+	printf '%s is not a valid product, choose one or more product names from the list: %s\n' "$1" "$possible_products"
 	exit 1
 }
 
@@ -324,6 +315,8 @@ else
 	autodetect_builder
 fi
 
+test "${_arg_product[0]}" = ALL && _arg_product=("${all_cmake_products[@]}")
+
 cmake_enable_args=()
 for chosen_product in "${_arg_product[@]}"; do
 	if is_product "$chosen_product"; then

diff --git a/cmake/SSGCommon.cmake b/cmake/SSGCommon.cmake
@@ -285,6 +285,7 @@ macro(ssg_build_ansible_playbooks PRODUCT)
         NAME "${PRODUCT}-ansible-playbooks-generated-for-all-rules"
         COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${CMAKE_SOURCE_DIR}/tests/ansible_playbooks_generated_for_all_rules.py" --build-dir "${CMAKE_BINARY_DIR}" --product "${PRODUCT}"
     )
+    set_tests_properties("${PRODUCT}-ansible-playbooks-generated-for-all-rules" PROPERTIES LABELS quick)
     if("${PRODUCT}" MATCHES "rhel")
         add_test(
             NAME "${PRODUCT}-ansible-assert-playbooks-schema"
@@ -453,14 +454,17 @@ macro(ssg_build_xccdf_final PRODUCT)
         NAME "verify-references-ssg-${PRODUCT}-xccdf.xml"
         COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/verify_references.py" --rules-with-invalid-checks --ovaldefs-unused "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-xccdf.xml"
     )
+    set_tests_properties("verify-references-ssg-${PRODUCT}-xccdf.xml" PROPERTIES LABELS quick)
     add_test(
         NAME "verify-ssg-${PRODUCT}-xccdf.xml-override-true-all-profile-titles"
         COMMAND "${XMLLINT_EXECUTABLE}" --xpath "//*[local-name()=\"Profile\"]/*[local-name()=\"title\"][not(@override=\"true\")]" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-xccdf.xml"
     )
+    set_tests_properties("verify-ssg-${PRODUCT}-xccdf.xml-override-true-all-profile-titles" PROPERTIES LABELS quick)
     add_test(
         NAME "verify-ssg-${PRODUCT}-xccdf.xml-override-true-all-profile-descriptions"
         COMMAND "${XMLLINT_EXECUTABLE}" --xpath "//*[local-name()=\"Profile\"]/*[local-name()=\"description\"][not(@override=\"true\")]" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-xccdf.xml"
     )
+    set_tests_properties("verify-ssg-${PRODUCT}-xccdf.xml-override-true-all-profile-descriptions" PROPERTIES LABELS quick)
     # Sets WILL_FAIL property for all '*-override-true-all-profile-*' tests to
     # true as it is expected that XPath of a passing test will be empty (and
     # non-zero exit code is returned in such case).
@@ -1251,6 +1255,7 @@ macro(ssg_define_guide_and_table_tests)
             NAME "unique-cces"
 	    COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/tests/assert_cces_unique.sh"
         )
+        set_tests_properties("unique-cces" PROPERTIES LABELS quick)
     endif()
 endmacro()
 

diff --git a/docs/manual/developer_guide.adoc b/docs/manual/developer_guide.adoc
@@ -311,14 +311,15 @@ bash/rhel7-script-pci-dss.sh
 ### Testing
 
 To ensure validity of built artifacts prior to installation, we recommend
-running our test suite against the build output. This is done with CTest:
+running our test suite against the build output. This is done with CTest.
+It is a good idea to execute quick tests first using the `-L quick` option passed to `ctest`.
 
 ```bash
 cd content/
-cd build/
-cmake ../
-make -j4
-ctest -j4
+./build_product
+cd build
+ctest -L quick
+ctest -LE quick -j4
 ```
 
 Note: CTest does not run link:https://github.com/ComplianceAsCode/content/tree/master/tests[SSG Test Suite] which provides simple system of test scenarios for testing profiles and rule remediations.

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
@@ -23,6 +23,7 @@ add_test(
     NAME "max-path-len"
     COMMAND "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/ensure_paths_are_short.py"
 )
+set_tests_properties("max-path-len" PROPERTIES LABELS quick)
 
 add_test(
     NAME "test-rule-dir-json"
@@ -38,29 +39,35 @@ add_test(
     NAME "validate-parse-platform"
     COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/test_parse_platform.py" "${CMAKE_SOURCE_DIR}"
 )
+set_tests_properties("validate-parse-platform" PROPERTIES LABELS quick)
 
 add_test(
     NAME "stable-profile-ids"
     COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/stable_profile_ids.py" "${CMAKE_BINARY_DIR}"
 )
+set_tests_properties("stable-profile-ids" PROPERTIES LABELS quick)
 
 add_test(
     NAME "stable-profiles"
     COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/test_profile_stability.py" "${CMAKE_BINARY_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/data/profile_stability"
 )
+set_tests_properties("stable-profiles" PROPERTIES LABELS quick)
 
 add_test(
     NAME "machine-only-rules"
     COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/test_machine_only_rules.py" --source_dir "${CMAKE_SOURCE_DIR}" --build_dir "${CMAKE_BINARY_DIR}"
 )
+set_tests_properties("machine-only-rules" PROPERTIES LABELS quick)
 
 if (SSG_BATS_TESTS_ENABLED AND BATS_EXECUTABLE)
     add_test(
         NAME "bash-unit-tests"
         COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/unit/bash/execute_tests.sh" "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/unit/bash" "${CMAKE_BINARY_DIR}/tests"
     )
+    set_tests_properties("bash-unit-tests" PROPERTIES LABELS quick)
 endif()
 add_test(
     NAME "macros-oval"
     COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/test_macros_oval.py" "--verbose"
 )
+set_tests_properties("macros-oval" PROPERTIES LABELS quick)