diff --git a/Sources/Testing/Issues/Issue.swift b/Sources/Testing/Issues/Issue.swift index 9a2555177..e321a1b9d 100644 --- a/Sources/Testing/Issues/Issue.swift +++ b/Sources/Testing/Issues/Issue.swift @@ -38,6 +38,9 @@ public struct Issue: Sendable { /// confirmed too few or too many times. indirect case confirmationMiscounted(actual: Int, expected: any RangeExpression & Sendable) + @_spi(Experimental) + case confirmationPollingFailed + /// An issue due to an `Error` being thrown by a test function and caught by /// the testing library. /// @@ -286,6 +289,8 @@ extension Issue.Kind: CustomStringConvertible { } } return "Confirmation was confirmed \(actual.counting("time")), but expected to be confirmed \(String(describingForTest: expected)) time(s)" + case .confirmationPollingFailed: + return "Confirmation polling failed" case let .errorCaught(error): return "Caught error: \(error)" case let .timeLimitExceeded(timeLimitComponents: timeLimitComponents): @@ -465,6 +470,8 @@ extension Issue.Kind { .expectationFailed(Expectation.Snapshot(snapshotting: expectation)) case .confirmationMiscounted: .unconditional + case .confirmationPollingFailed: + .unconditional case let .errorCaught(error), let .valueAttachmentFailed(error): .errorCaught(ErrorSnapshot(snapshotting: error)) case let .timeLimitExceeded(timeLimitComponents: timeLimitComponents): diff --git a/Sources/Testing/Polling/Polling.swift b/Sources/Testing/Polling/Polling.swift new file mode 100644 index 000000000..851e68ecd --- /dev/null +++ b/Sources/Testing/Polling/Polling.swift @@ -0,0 +1,570 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +internal let defaultPollingConfiguration = ( + maxPollingIterations: 1000, + pollingInterval: Duration.milliseconds(1) +) + +/// A type describing an error thrown when polling fails. +@_spi(Experimental) +public struct PollingFailedError: Error, Equatable {} + +/// Confirm that some expression eventually returns true +/// +/// - Parameters: +/// - comment: An optional comment to apply to any issues generated by this +/// function. +/// - maxPollingIterations: The maximum amount of times to attempt polling. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or +/// suite. +/// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then +/// polling will be attempted 1000 times before recording an issue. +/// `maxPollingIterations` must be greater than 0. +/// - pollingInterval: The minimum amount of time to wait between polling +/// attempts. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then +/// polling will wait at least 1 millisecond between polling attempts. +/// `pollingInterval` must be greater than 0. +/// - isolation: The actor to which `body` is isolated, if any. +/// - sourceLocation: The source location to whych any recorded issues should +/// be attributed. +/// - body: The function to invoke. +/// +/// Use polling confirmations to check that an event while a test is running in +/// complex scenarios where other forms of confirmation are insufficient. For +/// example, waiting on some state to change that cannot be easily confirmed +/// through other forms of `confirmation`. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public func confirmPassesEventually( + _ comment: Comment? = nil, + maxPollingIterations: Int? = nil, + pollingInterval: Duration? = nil, + isolation: isolated (any Actor)? = #isolation, + sourceLocation: SourceLocation = #_sourceLocation, + _ body: @escaping () async throws -> Bool +) async { + let poller = Poller( + pollingBehavior: .passesOnce, + pollingIterations: getValueFromPollingTrait( + providedValue: maxPollingIterations, + default: defaultPollingConfiguration.maxPollingIterations, + \ConfirmPassesEventuallyConfigurationTrait.maxPollingIterations + ), + pollingInterval: getValueFromPollingTrait( + providedValue: pollingInterval, + default: defaultPollingConfiguration.pollingInterval, + \ConfirmPassesEventuallyConfigurationTrait.pollingInterval + ), + comment: comment, + sourceLocation: sourceLocation + ) + await poller.evaluate(isolation: isolation) { + do { + return try await body() + } catch { + return false + } + } +} + +/// Require that some expression eventually returns true +/// +/// - Parameters: +/// - comment: An optional comment to apply to any issues generated by this +/// function. +/// - maxPollingIterations: The maximum amount of times to attempt polling. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or +/// suite. +/// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then +/// polling will be attempted 1000 times before recording an issue. +/// `maxPollingIterations` must be greater than 0. +/// - pollingInterval: The minimum amount of time to wait between polling +/// attempts. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then +/// polling will wait at least 1 millisecond between polling attempts. +/// `pollingInterval` must be greater than 0. +/// - isolation: The actor to which `body` is isolated, if any. +/// - sourceLocation: The source location to whych any recorded issues should +/// be attributed. +/// - body: The function to invoke. +/// +/// - Throws: A `PollingFailedError` will be thrown if the expression never +/// returns true. +/// +/// Use polling confirmations to check that an event while a test is running in +/// complex scenarios where other forms of confirmation are insufficient. For +/// example, waiting on some state to change that cannot be easily confirmed +/// through other forms of `confirmation`. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public func requirePassesEventually( + _ comment: Comment? = nil, + maxPollingIterations: Int? = nil, + pollingInterval: Duration? = nil, + isolation: isolated (any Actor)? = #isolation, + sourceLocation: SourceLocation = #_sourceLocation, + _ body: @escaping () async throws -> Bool +) async throws { + let poller = Poller( + pollingBehavior: .passesOnce, + pollingIterations: getValueFromPollingTrait( + providedValue: maxPollingIterations, + default: defaultPollingConfiguration.maxPollingIterations, + \ConfirmPassesEventuallyConfigurationTrait.maxPollingIterations + ), + pollingInterval: getValueFromPollingTrait( + providedValue: pollingInterval, + default: defaultPollingConfiguration.pollingInterval, + \ConfirmPassesEventuallyConfigurationTrait.pollingInterval + ), + comment: comment, + sourceLocation: sourceLocation + ) + let passed = await poller.evaluate(raiseIssue: false, isolation: isolation) { + do { + return try await body() + } catch { + return false + } + } + if !passed { + throw PollingFailedError() + } +} + +/// Confirm that some expression eventually returns a non-nil value +/// +/// - Parameters: +/// - comment: An optional comment to apply to any issues generated by this +/// function. +/// - maxPollingIterations: The maximum amount of times to attempt polling. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or +/// suite. +/// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then +/// polling will be attempted 1000 times before recording an issue. +/// `maxPollingIterations` must be greater than 0. +/// - pollingInterval: The minimum amount of time to wait between polling +/// attempts. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then +/// polling will wait at least 1 millisecond between polling attempts. +/// `pollingInterval` must be greater than 0. +/// - isolation: The actor to which `body` is isolated, if any. +/// - sourceLocation: The source location to whych any recorded issues should +/// be attributed. +/// - body: The function to invoke. +/// +/// - Returns: The first non-nil value returned by `body`. +/// +/// - Throws: A `PollingFailedError` will be thrown if `body` never returns a +/// non-optional value. +/// +/// Use polling confirmations to check that an event while a test is running in +/// complex scenarios where other forms of confirmation are insufficient. For +/// example, waiting on some state to change that cannot be easily confirmed +/// through other forms of `confirmation`. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +@discardableResult +public func confirmPassesEventually( + _ comment: Comment? = nil, + maxPollingIterations: Int? = nil, + pollingInterval: Duration? = nil, + isolation: isolated (any Actor)? = #isolation, + sourceLocation: SourceLocation = #_sourceLocation, + _ body: @escaping () async throws -> R? +) async throws -> R where R: Sendable { + let recorder = PollingRecorder() + let poller = Poller( + pollingBehavior: .passesOnce, + pollingIterations: getValueFromPollingTrait( + providedValue: maxPollingIterations, + default: defaultPollingConfiguration.maxPollingIterations, + \ConfirmPassesEventuallyConfigurationTrait.maxPollingIterations + ), + pollingInterval: getValueFromPollingTrait( + providedValue: pollingInterval, + default: defaultPollingConfiguration.pollingInterval, + \ConfirmPassesEventuallyConfigurationTrait.pollingInterval + ), + comment: comment, + sourceLocation: sourceLocation + ) + await poller.evaluate(isolation: isolation) { + do { + return try await recorder.record(value: body()) + } catch { + return false + } + } + + if let value = await recorder.lastValue { + return value + } + throw PollingFailedError() +} + +/// Confirm that some expression always returns true +/// +/// - Parameters: +/// - comment: An optional comment to apply to any issues generated by this +/// function. +/// - maxPollingIterations: The maximum amount of times to attempt polling. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmAlwaysPassesConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmAlwaysPassesConfigurationTrait`` has been added, then +/// polling will be attempted 1000 times before recording an issue. +/// `maxPollingIterations` must be greater than 0. +/// - pollingInterval: The minimum amount of time to wait between polling +/// attempts. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmAlwaysPassesConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmAlwaysPassesConfigurationTrait`` has been added, then +/// polling will wait at least 1 millisecond between polling attempts. +/// `pollingInterval` must be greater than 0. +/// - isolation: The actor to which `body` is isolated, if any. +/// - sourceLocation: The source location to whych any recorded issues should +/// be attributed. +/// - body: The function to invoke. +/// +/// Use polling confirmations to check that an event while a test is running in +/// complex scenarios where other forms of confirmation are insufficient. For +/// example, confirming that some state does not change. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public func confirmAlwaysPasses( + _ comment: Comment? = nil, + maxPollingIterations: Int? = nil, + pollingInterval: Duration? = nil, + isolation: isolated (any Actor)? = #isolation, + sourceLocation: SourceLocation = #_sourceLocation, + _ body: @escaping () async throws -> Bool +) async { + let poller = Poller( + pollingBehavior: .passesAlways, + pollingIterations: getValueFromPollingTrait( + providedValue: maxPollingIterations, + default: defaultPollingConfiguration.maxPollingIterations, + \ConfirmAlwaysPassesConfigurationTrait.maxPollingIterations + ), + pollingInterval: getValueFromPollingTrait( + providedValue: pollingInterval, + default: defaultPollingConfiguration.pollingInterval, + \ConfirmAlwaysPassesConfigurationTrait.pollingInterval + ), + comment: comment, + sourceLocation: sourceLocation + ) + await poller.evaluate(isolation: isolation) { + do { + return try await body() + } catch { + return false + } + } +} + +/// Require that some expression always returns true +/// +/// - Parameters: +/// - comment: An optional comment to apply to any issues generated by this +/// function. +/// - maxPollingIterations: The maximum amount of times to attempt polling. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmAlwaysPassesConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmAlwaysPassesConfigurationTrait`` has been added, then +/// polling will be attempted 1000 times before recording an issue. +/// `maxPollingIterations` must be greater than 0. +/// - pollingInterval: The minimum amount of time to wait between polling +/// attempts. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmAlwaysPassesConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmAlwaysPassesConfigurationTrait`` has been added, then +/// polling will wait at least 1 millisecond between polling attempts. +/// `pollingInterval` must be greater than 0. +/// - isolation: The actor to which `body` is isolated, if any. +/// - sourceLocation: The source location to whych any recorded issues should +/// be attributed. +/// - body: The function to invoke. +/// +/// - Throws: A `PollingFailedError` will be thrown if the expression ever +/// returns false. +/// +/// Use polling confirmations to check that an event while a test is running in +/// complex scenarios where other forms of confirmation are insufficient. For +/// example, confirming that some state does not change. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public func requireAlwaysPasses( + _ comment: Comment? = nil, + maxPollingIterations: Int? = nil, + pollingInterval: Duration? = nil, + isolation: isolated (any Actor)? = #isolation, + sourceLocation: SourceLocation = #_sourceLocation, + _ body: @escaping () async throws -> Bool +) async throws { + let poller = Poller( + pollingBehavior: .passesAlways, + pollingIterations: getValueFromPollingTrait( + providedValue: maxPollingIterations, + default: defaultPollingConfiguration.maxPollingIterations, + \ConfirmAlwaysPassesConfigurationTrait.maxPollingIterations + ), + pollingInterval: getValueFromPollingTrait( + providedValue: pollingInterval, + default: defaultPollingConfiguration.pollingInterval, + \ConfirmAlwaysPassesConfigurationTrait.pollingInterval + ), + comment: comment, + sourceLocation: sourceLocation + ) + let passed = await poller.evaluate(raiseIssue: false, isolation: isolation) { + do { + return try await body() + } catch { + return false + } + } + if !passed { + throw PollingFailedError() + } +} + +/// A helper function to de-duplicate the logic of grabbing configuration from +/// either the passed-in value (if given), the hardcoded default, and the +/// appropriate configuration trait. +/// +/// The provided value, if non-nil is returned. Otherwise, this looks for +/// the last `TraitKind` specified, and if one exists, returns the value +/// as determined by `keyPath`. +/// If no configuration trait has been applied, then this returns the `default`. +/// +/// - Parameters: +/// - providedValue: The value provided by the test author when calling +/// `confirmPassesEventually` or `confirmAlwaysPasses`. +/// - default: The harded coded default value, as defined in +/// `defaultPollingConfiguration` +/// - keyPath: The keyPath mapping from `TraitKind` to the desired value type. +private func getValueFromPollingTrait( + providedValue: Value?, + default: Value, + _ keyPath: KeyPath +) -> Value { + if let providedValue { return providedValue } + guard let test = Test.current else { return `default` } + let possibleTraits = test.traits.compactMap { $0 as? TraitKind } + let traitValues = possibleTraits.compactMap { $0[keyPath: keyPath] } + return traitValues.last ?? `default` +} + +/// A type to record the last value returned by a closure returning an optional +/// This is only used in the `confirm` polling functions evaluating an optional. +private actor PollingRecorder { + var lastValue: R? + + /// Record a new value to be returned + func record(value: R) { + self.lastValue = value + } + + func record(value: R?) -> Bool { + if let value { + self.lastValue = value + return true + } else { + return false + } + } +} + +/// A type for managing polling +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +private struct Poller { + enum PollingBehavior { + /// Continuously evaluate the expression until the first time it returns + /// true. + /// If it does not pass once by the time the timeout is reached, then a + /// failure will be reported. + case passesOnce + + /// Continuously evaluate the expression until the first time it returns + /// false. + /// If the expression returns false, then a failure will be reported. + /// If the expression only returns true before the timeout is reached, then + /// no failure will be reported. + /// If the expression does not finish evaluating before the timeout is + /// reached, then a failure will be reported. + case passesAlways + + /// Process the result of a polled expression and decide whether to continue polling. + /// + /// - Parameters: + /// - expressionResult: The result of the polled expression + /// + /// - Returns: A poll result (if polling should stop), or nil (if polling should continue) + func processFinishedExpression( + expressionResult result: Bool + ) -> PollResult? { + switch self { + case .passesOnce: + if result { + return .finished + } else { + return nil + } + case .passesAlways: + if !result { + return .failed + } else { + return nil + } + } + } + } + + /// The result of polling expressions + enum PollResult { + /// The polling ran for the total number of iterations + case ranToCompletion + /// The expression exited early, and we will report a success status. + case finished + /// The expression returned false under PollingBehavior.passesAlways + case failed + /// The polling was cancelled before polling could finish + case cancelled + + /// Process the poll result into an issue + /// + /// - Parameters: + /// - comment: The comment to record as part of the issue + /// - sourceContext: The source context for the issue + /// - pollingBehavior: The polling behavior used. + /// - Returns: An issue if one should be recorded, otherwise nil. + func issue( + comment: Comment?, + sourceContext: SourceContext, + pollingBehavior: PollingBehavior + ) -> Issue? { + let issueKind: Issue.Kind + switch self { + case .finished, .cancelled: + return nil + case .ranToCompletion: + if case .passesAlways = pollingBehavior { + return nil + } + issueKind = .confirmationPollingFailed + case .failed: + issueKind = .confirmationPollingFailed + } + return Issue( + kind: issueKind, + comments: Array(comment), + sourceContext: sourceContext + ) + } + } + + /// The polling behavior (poll until the expression first passes, or poll + /// while the expression continues to pass) + let pollingBehavior: PollingBehavior + + // How many times to poll + let pollingIterations: Int + // Minimum waiting period between polling + let pollingInterval: Duration + + /// A comment from the test author associated with the polling + let comment: Comment? + + /// The source location that asked for polling. + let sourceLocation: SourceLocation + + /// Evaluate polling, and process the result, raising an issue if necessary. + /// + /// - Parameters: + /// - raiseIssue: Whether or not to raise an issue. + /// This should only be false for `requirePassesEventually` or + /// `requireAlwaysPasses`. + /// - isolation: The isolation to use + /// - body: The expression to poll + /// + /// - Returns: Whether or not polling passed. + /// + /// - Side effects: If polling fails (see `PollingBehavior`), then this will + /// record an issue. + @discardableResult func evaluate( + raiseIssue: Bool = true, + isolation: isolated (any Actor)?, + _ body: @escaping () async -> Bool + ) async -> Bool { + precondition(pollingIterations > 0) + precondition(pollingInterval > Duration.zero) + let result = await poll( + expression: body + ) + if let issue = result.issue( + comment: comment, + sourceContext: .init(backtrace: .current(), sourceLocation: sourceLocation), + pollingBehavior: pollingBehavior + ) { + if raiseIssue { + issue.record() + } + return false + } else { + return true + } + } + + /// This function contains the logic for continuously polling an expression, + /// as well as processing the results of that expression + /// + /// - Parameters: + /// - expression: An expression to continuously evaluate + /// - behavior: The polling behavior to use + /// - timeout: How long to poll for unitl the timeout triggers. + /// - Returns: The result of this polling. + private func poll( + isolation: isolated (any Actor)? = #isolation, + expression: @escaping () async -> Bool + ) async -> PollResult { + for iteration in 0.. Self { + ConfirmPassesEventuallyConfigurationTrait( + maxPollingIterations: maxPollingIterations, + pollingInterval: pollingInterval + ) + } +} + +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +extension Trait where Self == ConfirmAlwaysPassesConfigurationTrait { + /// Specifies defaults for ``confirmPassesAlways`` in the test or suite. + /// + /// - Parameters: + /// - maxPollingIterations: The maximum amount of times to attempt polling. + /// If nil, polling will be attempted up to 1000 times. + /// `maxPollingIterations` must be greater than 0. + /// - pollingInterval: The minimum amount of time to wait between polling + /// attempts. + /// If nil, polling will wait at least 1 millisecond between polling + /// attempts. + /// `pollingInterval` must be greater than 0. + public static func confirmAlwaysPassesDefaults( + maxPollingIterations: Int? = nil, + pollingInterval: Duration? = nil + ) -> Self { + ConfirmAlwaysPassesConfigurationTrait( + maxPollingIterations: maxPollingIterations, + pollingInterval: pollingInterval + ) + } +} diff --git a/Tests/TestingTests/PollingTests.swift b/Tests/TestingTests/PollingTests.swift new file mode 100644 index 000000000..4f97e15bd --- /dev/null +++ b/Tests/TestingTests/PollingTests.swift @@ -0,0 +1,352 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing + +@Suite("Polling Tests") +struct PollingTests { + @Suite("confirmPassesEventually") + struct PassesOnceBehavior { + @Test("Simple passing expressions") func trivialHappyPath() async throws { + await confirmPassesEventually { true } + try await requirePassesEventually { true } + + let value = try await confirmPassesEventually { 1 } + + #expect(value == 1) + } + + @Test("Simple failing expressions") func trivialSadPath() async throws { + let issues = await runTest { + await confirmPassesEventually { false } + _ = try await confirmPassesEventually { Optional.none } + await #expect(throws: PollingFailedError()) { + try await requirePassesEventually { false } + } + } + #expect(issues.count == 3) + } + + @Test("When the value changes from false to true during execution") func changingFromFail() async { + let incrementor = Incrementor() + + await confirmPassesEventually { + await incrementor.increment() == 2 + // this will pass only on the second invocation + // This checks that we really are only running the expression until + // the first time it passes. + } + + // and then we check the count just to double check. + #expect(await incrementor.count == 2) + } + + @Test("Thrown errors are treated as returning false") + func errorsReported() async { + let issues = await runTest { + await confirmPassesEventually { + throw PollingTestSampleError.ohNo + } + } + #expect(issues.count == 1) + } + + @Test("Waits up to 1000 times before failing") + func defaultPollingCount() async { + let incrementor = Incrementor() + _ = await runTest { + // this test will intentionally fail. + await confirmPassesEventually(pollingInterval: .nanoseconds(1)) { + await incrementor.increment() == 0 + } + } + #expect(await incrementor.count == 1000) + } + + @Suite( + "Configuration traits", + .confirmPassesEventuallyDefaults(maxPollingIterations: 100) + ) + struct WithConfigurationTraits { + @Test("When no test or callsite configuration provided, uses the suite configuration") + func testUsesSuiteConfiguration() async throws { + let incrementor = Incrementor() + var test = Test { + await confirmPassesEventually(pollingInterval: .nanoseconds(1)) { + await incrementor.increment() == 0 + } + } + test.traits = Test.current?.traits ?? [] + await runTest(test: test) + let count = await incrementor.count + #expect(count == 100) + } + + @Test( + "When test configuration porvided, uses the test configuration", + .confirmPassesEventuallyDefaults(maxPollingIterations: 10) + ) + func testUsesTestConfigurationOverSuiteConfiguration() async { + let incrementor = Incrementor() + var test = Test { + // this test will intentionally fail. + await confirmPassesEventually(pollingInterval: .nanoseconds(1)) { + await incrementor.increment() == 0 + } + } + test.traits = Test.current?.traits ?? [] + await runTest(test: test) + #expect(await incrementor.count == 10) + } + + @Test( + "When callsite configuration provided, uses that", + .confirmPassesEventuallyDefaults(maxPollingIterations: 10) + ) + func testUsesCallsiteConfiguration() async { + let incrementor = Incrementor() + var test = Test { + // this test will intentionally fail. + await confirmPassesEventually(maxPollingIterations: 50, pollingInterval: .nanoseconds(1)) { + await incrementor.increment() == 0 + } + } + test.traits = Test.current?.traits ?? [] + await runTest(test: test) + #expect(await incrementor.count == 50) + } + } + } + + @Suite("confirmAlwaysPasses") + struct PassesAlwaysBehavior { + @Test("Simple passing expressions") func trivialHappyPath() async throws { + await confirmAlwaysPasses { true } + try await requireAlwaysPasses { true } + } + + @Test("Simple failing expressions") func trivialSadPath() async { + let issues = await runTest { + await confirmAlwaysPasses { false } + await #expect(throws: PollingFailedError()) { + try await requireAlwaysPasses { false } + } + } + #expect(issues.count == 1) + } + + @Test("if the closures starts off as true, but becomes false") + func changingFromFail() async { + let incrementor = Incrementor() + let issues = await runTest { + await confirmAlwaysPasses { + await incrementor.increment() == 2 + // this will pass only on the first invocation + // This checks that we fail the test if it starts failing later during + // polling + } + } + #expect(issues.count == 1) + } + + @Test("if the closure continues to pass") + func continuousCalling() async { + let incrementor = Incrementor() + + await confirmAlwaysPasses { + _ = await incrementor.increment() + return true + } + + #expect(await incrementor.count > 1) + } + + @Test("Thrown errors will automatically exit & fail") func errorsReported() async { + let issues = await runTest { + await confirmAlwaysPasses { + throw PollingTestSampleError.ohNo + } + } + #expect(issues.count == 1) + } + + @Test("Waits up to 1000 times before passing") + func defaultPollingCount() async { + let incrementor = Incrementor() + await confirmAlwaysPasses(pollingInterval: .nanoseconds(1)) { + await incrementor.increment() != 0 + } + #expect(await incrementor.count == 1000) + } + + @Suite( + "Configuration traits", + .confirmAlwaysPassesDefaults(maxPollingIterations: 100) + ) + struct WithConfigurationTraits { + @Test("When no test or callsite configuration provided, uses the suite configuration") + func testUsesSuiteConfiguration() async throws { + let incrementor = Incrementor() + await confirmAlwaysPasses(pollingInterval: .nanoseconds(1)) { + await incrementor.increment() != 0 + } + let count = await incrementor.count + #expect(count == 100) + } + + @Test( + "When test configuration porvided, uses the test configuration", + .confirmAlwaysPassesDefaults(maxPollingIterations: 10) + ) + func testUsesTestConfigurationOverSuiteConfiguration() async { + let incrementor = Incrementor() + await confirmAlwaysPasses(pollingInterval: .nanoseconds(1)) { + await incrementor.increment() != 0 + } + #expect(await incrementor.count == 10) + } + + @Test( + "When callsite configuration provided, uses that", + .confirmAlwaysPassesDefaults(maxPollingIterations: 10) + ) + func testUsesCallsiteConfiguration() async { + let incrementor = Incrementor() + await confirmAlwaysPasses(maxPollingIterations: 50, pollingInterval: .nanoseconds(1)) { + await incrementor.increment() != 0 + } + #expect(await incrementor.count == 50) + } + } + } + + @Suite("Duration Tests", .disabled("time-sensitive")) + struct DurationTests { + @Suite("confirmPassesEventually") + struct PassesOnceBehavior { + let delta = Duration.milliseconds(100) + + @Test("Simple passing expressions") func trivialHappyPath() async { + let duration = await Test.Clock().measure { + await confirmPassesEventually { true } + } + #expect(duration.isCloseTo(other: .zero, within: delta)) + } + + @Test("Simple failing expressions") func trivialSadPath() async { + let duration = await Test.Clock().measure { + let issues = await runTest { + await confirmPassesEventually { false } + } + #expect(issues.count == 1) + } + #expect(duration.isCloseTo(other: .seconds(2), within: delta)) + } + + @Test("When the value changes from false to true during execution") + func changingFromFail() async { + let incrementor = Incrementor() + + let duration = await Test.Clock().measure { + await confirmPassesEventually { + await incrementor.increment() == 2 + // this will pass only on the second invocation + // This checks that we really are only running the expression until + // the first time it passes. + } + } + + // and then we check the count just to double check. + #expect(await incrementor.count == 2) + #expect(duration.isCloseTo(other: .zero, within: delta)) + } + + @Test("Doesn't wait after the last iteration") + func lastIteration() async { + let duration = await Test.Clock().measure { + let issues = await runTest { + await confirmPassesEventually( + maxPollingIterations: 10, + pollingInterval: .seconds(1) // Wait a long time to handle jitter. + ) { false } + } + #expect(issues.count == 1) + } + #expect( + duration.isCloseTo( + other: .seconds(9), + within: .milliseconds(500) + ) + ) + } + } + + @Suite("confirmAlwaysPasses") + struct PassesAlwaysBehavior { + let delta = Duration.milliseconds(100) + + @Test("Simple passing expressions") func trivialHappyPath() async { + let duration = await Test.Clock().measure { + await confirmAlwaysPasses { true } + } + #expect(duration.isCloseTo(other: .seconds(1), within: delta)) + } + + @Test("Simple failing expressions") func trivialSadPath() async { + let duration = await Test.Clock().measure { + let issues = await runTest { + await confirmAlwaysPasses { false } + } + #expect(issues.count == 1) + } + #expect(duration.isCloseTo(other: .zero, within: delta)) + } + + @Test("Doesn't wait after the last iteration") + func lastIteration() async { + let duration = await Test.Clock().measure { + await confirmAlwaysPasses( + maxPollingIterations: 10, + pollingInterval: .seconds(1) // Wait a long time to handle jitter. + ) { true } + } + #expect( + duration.isCloseTo( + other: .seconds(9), + within: .milliseconds(500) + ) + ) + } + } + } +} + +private enum PollingTestSampleError: Error { + case ohNo + case secondCase +} + +extension DurationProtocol { + fileprivate func isCloseTo(other: Self, within delta: Self) -> Bool { + var distance = self - other + if (distance < Self.zero) { + distance *= -1 + } + return distance <= delta + } +} + +private actor Incrementor { + var count = 0 + func increment() -> Int { + count += 1 + return count + } +} diff --git a/Tests/TestingTests/TestSupport/TestingAdditions.swift b/Tests/TestingTests/TestSupport/TestingAdditions.swift index 4648f96af..56455a710 100644 --- a/Tests/TestingTests/TestSupport/TestingAdditions.swift +++ b/Tests/TestingTests/TestSupport/TestingAdditions.swift @@ -94,6 +94,55 @@ func runTestFunction(named name: String, in containingType: Any.Type, configurat await runner.run() } +/// Create a ``Test`` instance for the expression and run it, returning any +/// issues recorded. +/// +/// - Parameters: +/// - testFunction: The test expression to run +/// +/// - Returns: The list of issues recorded. +@discardableResult +func runTest( + testFunction: @escaping @Sendable () async throws -> Void +) async -> [Issue] { + let issues = Locked(rawValue: [Issue]()) + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case let .issueRecorded(issue) = event.kind { + issues.withLock { + $0.append(issue) + } + } + } + await Test(testFunction: testFunction).run(configuration: configuration) + return issues.rawValue +} + +/// Runs the passed-in `Test`, returning any issues recorded. +/// +/// - Parameters: +/// - test: The test to run +/// +/// - Returns: The list of issues recorded. +@discardableResult +func runTest( + test: Test +) async -> [Issue] { + let issues = Locked(rawValue: [Issue]()) + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case let .issueRecorded(issue) = event.kind { + issues.withLock { + $0.append(issue) + } + } + } + await test.run(configuration: configuration) + return issues.rawValue +} + extension Runner { /// Initialize an instance of this type that runs the free test function /// named `testName` in the module specified in `fileID`.