feat(decl): add test suites support and adapt logging

cmd/declarative/config/config.go

@@ -34,8 +34,8 @@ const (
 	DescriptionFlagName = "description"
 	// TestIDFlagName is the name of the flag allowing to specify the test identifier.
 	TestIDFlagName = "test-id"
-	// LabelsFlagName is the name of the flag allowing to specify labels.
-	LabelsFlagName = "labels"
+	// BaggageFlagName is the name of the flag allowing to specify the baggage.
+	BaggageFlagName = "baggage"
 	// TimeoutFlagName is the name of the flag allowing to specify the test timeout.
 	TimeoutFlagName = "timeout"
 )
@@ -51,8 +51,8 @@ type Config struct {
 	DescriptionEnvKey string
 	// TestIDEnvKey is the environment variable key corresponding to TestIDFlagName.
 	TestIDEnvKey string
-	// LabelsEnvKey is the environment variable key corresponding to LabelsFlagName.
-	LabelsEnvKey string
+	// BaggageEnvKey is the environment variable key corresponding to BaggageFlagName.
+	BaggageEnvKey string
 	// TimeoutEnvKey is the environment variable key corresponding to TimeoutFlagName.
 	TimeoutEnvKey string
 
@@ -82,9 +82,9 @@ type Config struct {
 	//
 	// A process having a test ID in the form <testUID> (i.e.: the leaf process) is the only one that is monitored.
 	TestID string
-	// Labels is the string containing the comma-separated list of labels in the form <labelX>=<labelXValue>. It is used
-	// for logging purposes and to potentially generate the child process/container labels.
-	Labels string
+	// Baggage is the string encoding a set of supported key-value pairs. It is used for logging purposes and to
+	// potentially generate the child process/container baggages.
+	Baggage string
 }
 
 var containerImagePullPolicies = map[builder.ImagePullPolicy][]string{
@@ -101,7 +101,7 @@ func New(declarativeEnvKey, envKeysPrefix string) *Config {
 		DescriptionFileEnvKey: envKeyFromFlagName(envKeysPrefix, DescriptionFileFlagName),
 		DescriptionEnvKey:     envKeyFromFlagName(envKeysPrefix, DescriptionFlagName),
 		TestIDEnvKey:          envKeyFromFlagName(envKeysPrefix, TestIDFlagName),
-		LabelsEnvKey:          envKeyFromFlagName(envKeysPrefix, LabelsFlagName),
+		BaggageEnvKey:         envKeyFromFlagName(envKeysPrefix, BaggageFlagName),
 		TimeoutEnvKey:         envKeyFromFlagName(envKeysPrefix, TimeoutFlagName),
 	}
 	return commonConf
@@ -133,13 +133,13 @@ func (c *Config) InitCommandFlags(cmd *cobra.Command) {
 
 	// Hidden flags.
 	flags.StringVar(&c.TestID, TestIDFlagName, "",
-		"(used during process chain building) The test identifier in the form <ignorePrefix><testUID>. It is "+
-			"used to propagate the test UID to child processes/container in the process chain")
-	flags.StringVar(&c.Labels, LabelsFlagName, "",
-		"(used during process chain building) The list of comma-separated labels in the form <labelX>=<labelXValue>. "+
-			"It is used for logging purposes and to potentially generate the child process/container labels")
+		"(used during process chain building) The test identifier in the form <ignorePrefix><testUID>. It is used to "+
+			"propagate the test UID to child processes/container in the process chain")
+	flags.StringVar(&c.Baggage, BaggageFlagName, "",
+		"(used during process chain building) The string encoding a set of supported key-value pais. It is used for "+
+			"logging purposes and to potentially generate the child process/container baggage")
 	_ = flags.MarkHidden(TestIDFlagName)
-	_ = flags.MarkHidden(LabelsFlagName)
+	_ = flags.MarkHidden(BaggageFlagName)
 }
 
 // envKeyFromFlagName converts the provided flag name into the corresponding environment variable key.

cmd/declarative/test/test.go

@@ -33,8 +33,8 @@ import (
 
 	"github.com/falcosecurity/event-generator/cmd/declarative/config"
 	"github.com/falcosecurity/event-generator/pkg/alert/retriever/grpcretriever"
+	"github.com/falcosecurity/event-generator/pkg/baggage"
 	containerbuilder "github.com/falcosecurity/event-generator/pkg/container/builder"
-	"github.com/falcosecurity/event-generator/pkg/label"
 	processbuilder "github.com/falcosecurity/event-generator/pkg/process/builder"
 	"github.com/falcosecurity/event-generator/pkg/test"
 	testbuilder "github.com/falcosecurity/event-generator/pkg/test/builder"
@@ -44,6 +44,7 @@ import (
 	runnerbuilder "github.com/falcosecurity/event-generator/pkg/test/runner/builder"
 	stepbuilder "github.com/falcosecurity/event-generator/pkg/test/step/builder"
 	sysbuilder "github.com/falcosecurity/event-generator/pkg/test/step/syscall/builder"
+	"github.com/falcosecurity/event-generator/pkg/test/suite"
 	"github.com/falcosecurity/event-generator/pkg/test/tester"
 	"github.com/falcosecurity/event-generator/pkg/test/tester/reportencoder/jsonencoder"
 	"github.com/falcosecurity/event-generator/pkg/test/tester/reportencoder/textencoder"
@@ -185,17 +186,17 @@ func (cw *CommandWrapper) run(cmd *cobra.Command, _ []string) {
 		os.Exit(1)
 	}
 
-	labels, err := label.ParseSet(cw.Labels)
+	baseBag, err := baggage.NewFromString(cw.Baggage)
 	if err != nil {
-		logger.Error(err, "Error parsing labels")
+		logger.Error(err, "Error parsing baggage")
 		cancelAndExit()
 	}
 
-	logger = enrichLoggerWithLabels(logger, labels)
+	logger = enrichLoggerWithBaggage(logger, baseBag)
 
-	description, err := loadTestsDescription(logger, cw.TestsDescriptionFile, cw.TestsDescription)
+	testSuites, err := loadTestSuites(logger, cw.TestsDescriptionFile, cw.TestsDescription)
 	if err != nil {
-		logger.Error(err, "Error loading tests description")
+		logger.Error(err, "Error loading test suites")
 		cancelAndExit()
 	}
 
@@ -205,10 +206,8 @@ func (cw *CommandWrapper) run(cmd *cobra.Command, _ []string) {
 		cancelAndExit()
 	}
 
-	// Retrieve the already populated test ID.
-	testID := cw.TestID
 	// The test ID absence is used to uniquely identify the root process in the process chain.
-	isRootProcess := testID == ""
+	isRootProcess := cw.TestID == ""
 
 	// globalWaitGroup accounts for all spawned goroutines, while testerWaitGroup accounts for all goroutines producing
 	// reports.
@@ -223,11 +222,16 @@ func (cw *CommandWrapper) run(cmd *cobra.Command, _ []string) {
 	var testr tester.Tester
 	if verifyExpectedOutcome := isRootProcess && !cw.skipOutcomeVerification; verifyExpectedOutcome {
 		// Ensure each test is associated with a rule.
-		for testIndex := range description.Tests {
-			testDesc := &description.Tests[testIndex]
-			if testDesc.Rule == nil {
-				logger := logger.WithValues("testName", testDesc.Name, "testIndex", testIndex)
-				logger.Error(errRuleNameNotDefined, "Error verifying test rule name presence")
+		for _, testSuite := range testSuites {
+			// Notice: there cannot be more than a test suite specifying no rule name, as rule names are guaranteed to
+			// be distinct among test suites. For this reason, it is ok to just stop on the first invalid test suite.
+			if testSuite.RuleName == suite.NoRuleNamePlaceholder {
+				logger := logger.WithValues("testSuite", testSuite.RuleName)
+				for _, testInfo := range testSuite.TestsInfo {
+					logger := logger.WithValues("testFile", testInfo.SourceName, "testName", testInfo.Test.Name,
+						"testIndex", testInfo.Index)
+					logger.Error(errRuleNameNotDefined, "Error verifying test rule name presence")
+				}
 				cancelAndExit()
 			}
 		}
@@ -250,97 +254,58 @@ func (cw *CommandWrapper) run(cmd *cobra.Command, _ []string) {
 
 	// Prepare parameters shared by all runners.
 	runnerEnviron := cw.buildRunnerEnviron(cmd)
-	var runnerLabels *label.Set
-	if labels != nil {
-		runnerLabels = labels.Clone()
-	}
-
-	// Build and run the tests.
-	for testIndex := range description.Tests {
-		testDesc := &description.Tests[testIndex]
-		logger := logger.WithValues("testName", testDesc.Name)
-
-		var testUID uuid.UUID
-		if isRootProcess {
-			logger = logger.WithValues("testIndex", testIndex)
 
-			// Generate a new UID for the test.
-			testUID = uuid.New()
-			testID = newTestID(&testUID)
-			ensureProcessChainLeaf(testDesc)
-		} else {
-			// Extract UID from test ID.
-			var err error
-			testUID, err = extractTestUID(testID)
-			if err != nil {
-				logger.Error(err, "Error extracting test UID from test ID", "testId", testID)
-				waitAndExit()
-			}
-		}
-
-		logger = logger.WithValues("testUid", testUID)
-
-		runnerLogger := logger.WithName("runner")
-		runnerDescription := &runner.Description{
-			Environ:                   runnerEnviron,
-			TestDescriptionEnvKey:     cw.DescriptionEnvKey,
-			TestDescriptionFileEnvKey: cw.DescriptionFileEnvKey,
-			TestIDEnvKey:              cw.TestIDEnvKey,
-			TestIDIgnorePrefix:        testIDIgnorePrefix,
-			LabelsEnvKey:              cw.LabelsEnvKey,
-			Labels:                    runnerLabels,
-		}
-		runnerInstance, err := runnerBuilder.Build(testDesc.Runner, runnerLogger, runnerDescription)
-		if err != nil {
-			logger.Error(err, "Error creating runner")
-			waitAndExit()
-		}
-
-		logger.Info("Starting test execution...")
-		if err := runnerInstance.Run(ctx, testID, testIndex, testDesc); err != nil {
-			logRunnerError(logger, err)
+	// Run test suites.
+	baseLogger := logger
+	for _, testSuite := range testSuites {
+		logger := baseLogger.WithValues("testSuiteName", testSuite.RuleName)
+		logInfoIf(logger, isRootProcess, "Starting test suite execution...")
+		success := cw.runTestSuite(ctx, baseLogger, testSuite, runnerBuilder, runnerEnviron, baseBag, testr,
+			&globalWaitGroup, &testerWaitGroup, isRootProcess)
+		if !success {
+			logInfoIf(logger, isRootProcess, "Test suite execution failed")
 			waitAndExit()
 		}
 
-		logger.Info("Test execution completed")
-
-		if testr != nil {
-			produceReport(&globalWaitGroup, &testerWaitGroup, testr, &testUID, testDesc, cw.reportFormat)
-		}
+		logInfoIf(logger, isRootProcess, "Test suite execution completed")
 	}
 
 	testerWaitGroup.Wait()
 	cancel()
 	globalWaitGroup.Wait()
 }
 
-// enrichLoggerWithLabels creates a new logger, starting from the provided one, with the information extracted from the
-// provided labels.
-func enrichLoggerWithLabels(logger logr.Logger, labels *label.Set) logr.Logger {
-	if labels == nil {
+// enrichLoggerWithBaggage creates a new logger, starting from the provided one, with the information extracted from the
+// provided baggage.
+func enrichLoggerWithBaggage(logger logr.Logger, bag *baggage.Baggage) logr.Logger {
+	if bag == nil {
 		return logger
 	}
 
-	logger = logger.WithValues("testIndex", labels.TestIndex, "procIndex", labels.ProcIndex, "inContainer",
-		labels.IsContainer)
-	if labels.IsContainer {
-		if imageName := labels.ImageName; imageName != "" {
+	logger = logger.WithValues("testSuiteName", bag.TestSuiteName, "testName", bag.TestName, "testSourceName",
+		bag.TestSourceName, "testSourceIndex", bag.TestSourceIndex)
+	if bag.ProcIndex != -1 {
+		logger = logger.WithValues("procIndex", bag.ProcIndex)
+	}
+	if bag.IsContainer {
+		logger = logger.WithValues("inContainer", bag.IsContainer)
+		if imageName := bag.ContainerImageName; imageName != "" {
 			logger = logger.WithValues("containerImageName", imageName)
 		}
-		if containerName := labels.ContainerName; containerName != "" {
+		if containerName := bag.ContainerName; containerName != "" {
 			logger = logger.WithValues("containerName", containerName)
 		}
 	}
 	return logger
 }
 
-// loadTestsDescription loads the YAML tests description from a different source, depending on the content of the
-// provided values:
-// - if the provided descriptionFilePath is not empty, the description is loaded from the specified file
-// - if the provided description is not empty, the description is loaded from its content
-// - otherwise, it is loaded from standard input.
-func loadTestsDescription(logger logr.Logger, descriptionFilePath, description string) (*loader.Description, error) {
-	ldr := loader.New()
+// loadTestSuites loads the test suites from a different source, depending on the content of the provided values:
+// - if the provided descriptionFilePath is not empty, the test suites are loaded from the specified file;
+// - if the provided description is not empty, the test suites are loaded from its content;
+// - otherwise, they are loaded from standard input.
+func loadTestSuites(logger logr.Logger, descriptionFilePath, description string) ([]*suite.Suite, error) {
+	descLoader := loader.New()
+	suiteLoader := suite.NewLoader(descLoader)
 
 	if descriptionFilePath != "" {
 		descriptionFilePath = filepath.Clean(descriptionFilePath)
@@ -354,14 +319,17 @@ func loadTestsDescription(logger logr.Logger, descriptionFilePath, description s
 			}
 		}()
 
-		return ldr.Load(descriptionFile)
+		sources := []suite.Source{suite.NewSourceFromFile(descriptionFile)}
+		return suiteLoader.Load(sources)
 	}
 
 	if description != "" {
-		return ldr.Load(strings.NewReader(description))
+		sources := []suite.Source{suite.NewSourceFromReader("<description flag>", strings.NewReader(description))}
+		return suiteLoader.Load(sources)
 	}
 
-	return ldr.Load(os.Stdin)
+	sources := []suite.Source{suite.NewSourceFromReader("<stdin>", os.Stdin)}
+	return suiteLoader.Load(sources)
 }
 
 // createRunnerBuilder creates a new runner builder.
@@ -441,6 +409,91 @@ func (cw *CommandWrapper) appendFlags(environ []string, flagSets ...*pflag.FlagS
 	return environ
 }
 
+// logInfoIf outputs to the provided logger the provided informational message only if the provided condition is true.
+func logInfoIf(logger logr.Logger, cond bool, msg string) {
+	if cond {
+		logger.Info(msg)
+	}
+}
+
+// runTestSuite runs the provided test suite.
+// TODO: simplify the following function signature once the termination logic is relaxed.
+func (cw *CommandWrapper) runTestSuite(ctx context.Context, baseLogger logr.Logger, testSuite *suite.Suite,
+	runnerBuilder runner.Builder, runnerEnviron []string, baseBag *baggage.Baggage, testr tester.Tester,
+	globalWaitGroup, testerWaitGroup *sync.WaitGroup, isRootProcess bool) bool {
+	// Build and run the tests.
+	for _, testInfo := range testSuite.TestsInfo {
+		// Init baggage and logger for the current test.
+		var bag *baggage.Baggage
+		if baseBag != nil {
+			bag = baseBag.Clone()
+		}
+		logger := baseLogger
+
+		testSourceName, testSourceIndex, testDesc := testInfo.SourceName, testInfo.Index, testInfo.Test
+
+		testID := cw.TestID
+		var testUID uuid.UUID
+		if isRootProcess {
+			// The root process must initialize the baggage.
+			bag = &baggage.Baggage{
+				TestSuiteName:   testSuite.RuleName,
+				TestName:        testDesc.Name,
+				TestSourceName:  testSourceName,
+				TestSourceIndex: testSourceIndex,
+				// ProcIndex is set to -1 on the root process.
+				ProcIndex: -1,
+			}
+			logger = enrichLoggerWithBaggage(logger, bag)
+
+			// Generate a new UID for the test.
+			testUID = uuid.New()
+			testID = newTestID(&testUID)
+			ensureProcessChainLeaf(testDesc)
+		} else {
+			// Extract UID from test ID.
+			var err error
+			testUID, err = extractTestUID(testID)
+			if err != nil {
+				logger.Error(err, "Error extracting test UID from test ID", "testId", testID)
+				return false
+			}
+		}
+
+		logger = logger.WithValues("testUid", testUID)
+
+		runnerLogger := logger.WithName("runner")
+		runnerDescription := &runner.Description{
+			Environ:                   runnerEnviron,
+			TestDescriptionEnvKey:     cw.DescriptionEnvKey,
+			TestDescriptionFileEnvKey: cw.DescriptionFileEnvKey,
+			TestIDEnvKey:              cw.TestIDEnvKey,
+			TestIDIgnorePrefix:        testIDIgnorePrefix,
+			BaggageEnvKey:             cw.BaggageEnvKey,
+			Baggage:                   bag,
+		}
+		runnerInstance, err := runnerBuilder.Build(testDesc.Runner, runnerLogger, runnerDescription)
+		if err != nil {
+			logger.Error(err, "Error creating runner")
+			return false
+		}
+
+		logInfoIf(logger, isRootProcess, "Starting test execution...")
+		if err := runnerInstance.Run(ctx, testID, testDesc); err != nil {
+			logRunnerError(logger, err)
+			return false
+		}
+
+		logInfoIf(logger, isRootProcess, "Test execution completed")
+
+		if testr != nil {
+			produceReport(globalWaitGroup, testerWaitGroup, testr, &testUID, testDesc, cw.reportFormat)
+		}
+	}
+
+	return true
+}
+
 // newTestID creates a new test ID from the provided test UID.
 func newTestID(uid *uuid.UUID) string {
 	return fmt.Sprintf("%s%s", testIDIgnorePrefix, uid.String())