From 303a4103b12a422f4fe87ae91304c6c121de26d1 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 6 May 2025 22:13:09 +0900 Subject: [PATCH 1/3] First --- Tests/LiveKitTests/Audio/EchoTesting.swift | 113 +++++++++++++++++++++ Tests/LiveKitTests/Support/Room.swift | 10 +- 2 files changed, 119 insertions(+), 4 deletions(-) create mode 100644 Tests/LiveKitTests/Audio/EchoTesting.swift diff --git a/Tests/LiveKitTests/Audio/EchoTesting.swift b/Tests/LiveKitTests/Audio/EchoTesting.swift new file mode 100644 index 000000000..2513f1351 --- /dev/null +++ b/Tests/LiveKitTests/Audio/EchoTesting.swift @@ -0,0 +1,113 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@preconcurrency import AVFoundation +@testable import LiveKit +import LiveKitWebRTC +import XCTest + +struct EchoTesterCase { + let title: String + let appleVp: Bool + let captureOptions: AudioCaptureOptions? +} + +struct EchoTestResult: CustomStringConvertible { + let vad: Int + let peak: Float + + var description: String { + "VAD: \(vad), Peak: \(peak)" + } +} + +class EchoTests: LKTestCase { + static let startTest = "start_test" + static let stopTest = "stop_test" + static let rmsKey = "lk.rms" // RMS level of the far-end signal + static let peakKey = "lk.peak" // Peak level of the far-end signal + static let vadKey = "lk.vad" // Number of speech events in the far-end signal + + func runEchoAgent(testCase: EchoTesterCase) async throws -> EchoTestResult { + // No-VP + try! AudioManager.shared.setVoiceProcessingEnabled(testCase.appleVp) + + return try await withRooms([RoomTestingOptions(canPublish: true, canPublishData: true, canSubscribe: true)]) { rooms in + // Alias to Room + let room1 = rooms[0] + + // Sleep for 3 seconds for agent to join... + try? await Task.sleep(nanoseconds: 3 * 1_000_000_000) + + // Find agent + let echoAgent: RemoteParticipant? = room1.remoteParticipants.values.first { $0.kind == .agent } + XCTAssert(echoAgent != nil, "Agent participant not found") // Echo agent must be running + guard let agentIdentity = echoAgent?.identity else { + XCTFail("Echo agent's identity is nil") + fatalError() + } + + var vadResult = 0 + try await room1.registerTextStreamHandler(for: Self.vadKey) { reader, _ in + let resultString = try await reader.readAll() + let result = Int(resultString) ?? 0 + if result > vadResult { + print("VAD \(vadResult) -> \(result)") + vadResult = result + } + } + + var peakResult: Float = 0 + try await room1.registerTextStreamHandler(for: Self.peakKey) { reader, _ in + let resultString = try await reader.readAll() + let result = Float(resultString) ?? 0 + if result > peakResult { + print("PEAK \(peakResult) -> \(result)") + peakResult = result + } + } + + // Enable mic + try await room1.localParticipant.setMicrophone(enabled: true, captureOptions: testCase.captureOptions) + + // Bypass VP + // AudioManager.shared.isVoiceProcessingBypassed = true + + // Start test + _ = try await room1.localParticipant.performRpc(destinationIdentity: agentIdentity, method: Self.startTest, payload: "") + + // Sleep for 30 seconds... + try? await Task.sleep(nanoseconds: 30 * 1_000_000_000) + + return EchoTestResult(vad: vadResult, peak: peakResult) + } + } + + func testEchoAgent() async throws { + let testCases = [ + EchoTesterCase(title: "Default", appleVp: true, captureOptions: nil), + EchoTesterCase(title: "RTC VP Only", appleVp: false, captureOptions: AudioCaptureOptions(echoCancellation: true, + autoGainControl: true, + noiseSuppression: true, + highpassFilter: true)), + ] + + for testCase in testCases { + let result = try await runEchoAgent(testCase: testCase) + print("Result: \(testCase.title) \(result)") + } + } +} diff --git a/Tests/LiveKitTests/Support/Room.swift b/Tests/LiveKitTests/Support/Room.swift index 15a9762a7..e6b42a6f0 100644 --- a/Tests/LiveKitTests/Support/Room.swift +++ b/Tests/LiveKitTests/Support/Room.swift @@ -86,9 +86,9 @@ extension LKTestCase { } // Set up variable number of Rooms - func withRooms(_ options: [RoomTestingOptions] = [], - sharedKey: String = UUID().uuidString, - _ block: @escaping ([Room]) async throws -> Void) async throws + func withRooms(_ options: [RoomTestingOptions] = [], + sharedKey: String = UUID().uuidString, + _ block: @escaping ([Room]) async throws -> T) async throws -> T { let e2eeOptions = E2EEOptions(keyProvider: BaseKeyProvider(isSharedKey: true, sharedKey: sharedKey)) @@ -180,7 +180,7 @@ extension LKTestCase { let allRooms = rooms.map(\.room) // Execute block - try await block(allRooms) + let result = try await block(allRooms) // Disconnect all Rooms concurrently try await withThrowingTaskGroup(of: Void.self) { group in @@ -191,6 +191,8 @@ extension LKTestCase { } try await group.waitForAll() } + + return result } } From 576edcbec804981783ee5cf13d47bd08b5e4da7b Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 8 May 2025 01:04:09 +0900 Subject: [PATCH 2/3] Updates --- Tests/LiveKitTests/Audio/EchoTesting.swift | 139 +++++++++++++++------ 1 file changed, 100 insertions(+), 39 deletions(-) diff --git a/Tests/LiveKitTests/Audio/EchoTesting.swift b/Tests/LiveKitTests/Audio/EchoTesting.swift index 2513f1351..3c04f8070 100644 --- a/Tests/LiveKitTests/Audio/EchoTesting.swift +++ b/Tests/LiveKitTests/Audio/EchoTesting.swift @@ -19,21 +19,6 @@ import LiveKitWebRTC import XCTest -struct EchoTesterCase { - let title: String - let appleVp: Bool - let captureOptions: AudioCaptureOptions? -} - -struct EchoTestResult: CustomStringConvertible { - let vad: Int - let peak: Float - - var description: String { - "VAD: \(vad), Peak: \(peak)" - } -} - class EchoTests: LKTestCase { static let startTest = "start_test" static let stopTest = "stop_test" @@ -41,9 +26,50 @@ class EchoTests: LKTestCase { static let peakKey = "lk.peak" // Peak level of the far-end signal static let vadKey = "lk.vad" // Number of speech events in the far-end signal - func runEchoAgent(testCase: EchoTesterCase) async throws -> EchoTestResult { + struct TestCase { + let title: String + let enableAppleVp: Bool + let captureOptions: AudioCaptureOptions? + } + + struct TestResult: CustomStringConvertible { + let vadCount: Int + let maxPeak: Float + + var description: String { + "VAD: \(vadCount), Peak: \(maxPeak)" + } + } + + // Actor to safely manage state across concurrent contexts + actor TestStateActor { + private(set) var vadResult: Int = 0 + private(set) var peakResult: Float = -120.0 + + func updateVad(_ newValue: Int) { + if newValue > vadResult { + print("Updating VAD \(vadResult) -> \(newValue)") + vadResult = newValue + } + } + + func updatePeak(_ newValue: Float) { + if newValue > peakResult { + print("Updating PEAK \(peakResult) -> \(newValue)") + peakResult = newValue + } + } + + func getResults() -> TestResult { + TestResult(vadCount: vadResult, maxPeak: peakResult) + } + } + + func runEchoAgent(testCase: TestCase) async throws -> TestResult { // No-VP - try! AudioManager.shared.setVoiceProcessingEnabled(testCase.appleVp) + try! AudioManager.shared.setVoiceProcessingEnabled(testCase.enableAppleVp) + // Bypass VP + AudioManager.shared.isVoiceProcessingBypassed = !testCase.enableAppleVp return try await withRooms([RoomTestingOptions(canPublish: true, canPublishData: true, canSubscribe: true)]) { rooms in // Alias to Room @@ -60,31 +86,37 @@ class EchoTests: LKTestCase { fatalError() } - var vadResult = 0 - try await room1.registerTextStreamHandler(for: Self.vadKey) { reader, _ in + let state = TestStateActor() + + try await room1.registerTextStreamHandler(for: Self.vadKey) { [state] reader, _ in let resultString = try await reader.readAll() - let result = Int(resultString) ?? 0 - if result > vadResult { - print("VAD \(vadResult) -> \(result)") - vadResult = result - } + guard let result = Int(resultString) else { return } + await state.updateVad(result) } - var peakResult: Float = 0 - try await room1.registerTextStreamHandler(for: Self.peakKey) { reader, _ in + try await room1.registerTextStreamHandler(for: Self.peakKey) { [state] reader, _ in let resultString = try await reader.readAll() - let result = Float(resultString) ?? 0 - if result > peakResult { - print("PEAK \(peakResult) -> \(result)") - peakResult = result - } + guard let result = Float(resultString) else { return } + await state.updatePeak(result) } // Enable mic try await room1.localParticipant.setMicrophone(enabled: true, captureOptions: testCase.captureOptions) - // Bypass VP - // AudioManager.shared.isVoiceProcessingBypassed = true + // Sleep for 1 seconds... + try? await Task.sleep(nanoseconds: 1 * 1_000_000_000) + + // Check Apple VP + XCTAssert(testCase.enableAppleVp == AudioManager.shared.isVoiceProcessingEnabled) + XCTAssert(testCase.enableAppleVp != AudioManager.shared.isVoiceProcessingBypassed) + + // Check APM is enabled + let apmConfig = RTC.audioProcessingModule.config + print("APM Config: \(apmConfig.toDebugString()))") + XCTAssert((testCase.captureOptions?.echoCancellation ?? false) == apmConfig.isEchoCancellationEnabled) + XCTAssert((testCase.captureOptions?.autoGainControl ?? false) == apmConfig.isAutoGainControl1Enabled) + XCTAssert((testCase.captureOptions?.noiseSuppression ?? false) == apmConfig.isNoiseSuppressionEnabled) + XCTAssert((testCase.captureOptions?.highpassFilter ?? false) == apmConfig.isHighpassFilterEnabled) // Start test _ = try await room1.localParticipant.performRpc(destinationIdentity: agentIdentity, method: Self.startTest, payload: "") @@ -92,22 +124,51 @@ class EchoTests: LKTestCase { // Sleep for 30 seconds... try? await Task.sleep(nanoseconds: 30 * 1_000_000_000) - return EchoTestResult(vad: vadResult, peak: peakResult) + // Stop test + _ = try await room1.localParticipant.performRpc(destinationIdentity: agentIdentity, method: Self.stopTest, payload: "") + + // Get final results from the actor + return await state.getResults() } } func testEchoAgent() async throws { + let defaultTestCase = TestCase(title: "Default", enableAppleVp: true, captureOptions: nil) + let allCaptureOptions = AudioCaptureOptions(echoCancellation: true, + autoGainControl: true, + noiseSuppression: true, + highpassFilter: true) let testCases = [ - EchoTesterCase(title: "Default", appleVp: true, captureOptions: nil), - EchoTesterCase(title: "RTC VP Only", appleVp: false, captureOptions: AudioCaptureOptions(echoCancellation: true, - autoGainControl: true, - noiseSuppression: true, - highpassFilter: true)), + TestCase(title: "None", enableAppleVp: false, captureOptions: nil), + TestCase(title: "RTC VP Only", enableAppleVp: false, captureOptions: allCaptureOptions), + TestCase(title: "Both", enableAppleVp: true, captureOptions: allCaptureOptions), ] + // Run default test first + print("Running Default test case...") + let defaultResult = try await runEchoAgent(testCase: defaultTestCase) + print("Result: \(defaultTestCase.title) \(defaultResult)") + + print("\n======= Test Results Summary =======") + print("Default: \(defaultResult)") + + // Run other test cases and compare with default for testCase in testCases { let result = try await runEchoAgent(testCase: testCase) print("Result: \(testCase.title) \(result)") + + let vadDiff = result.vadCount - defaultResult.vadCount + let peakDiff = result.maxPeak - defaultResult.maxPeak + + print("\(testCase.title): \(result)") + print(" Compared to Default:") + print(" - VAD difference: \(vadDiff > 0 ? "+" : "")\(vadDiff) events") + print(" - Peak difference: \(peakDiff > 0 ? "+" : "")\(String(format: "%.2f", peakDiff)) dB") + + // Optional basic analysis + let vadPercentChange = defaultResult.vadCount > 0 ? Float(vadDiff) / Float(defaultResult.vadCount) * 100 : 0 + print(" - VAD % change: \(String(format: "%.1f", vadPercentChange))%") } + print("===================================") } } From 525e4217cf6e2b340e01985b65169813c97f593a Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 8 May 2025 01:44:19 +0900 Subject: [PATCH 3/3] Fix --- Tests/LiveKitTests/Audio/EchoTesting.swift | 25 +++++++++++++++------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/Tests/LiveKitTests/Audio/EchoTesting.swift b/Tests/LiveKitTests/Audio/EchoTesting.swift index 3c04f8070..a3640b32b 100644 --- a/Tests/LiveKitTests/Audio/EchoTesting.swift +++ b/Tests/LiveKitTests/Audio/EchoTesting.swift @@ -33,11 +33,12 @@ class EchoTests: LKTestCase { } struct TestResult: CustomStringConvertible { + let testCase: TestCase let vadCount: Int let maxPeak: Float var description: String { - "VAD: \(vadCount), Peak: \(maxPeak)" + "\(testCase.title) VAD: \(vadCount), Peak: \(maxPeak)" } } @@ -60,8 +61,8 @@ class EchoTests: LKTestCase { } } - func getResults() -> TestResult { - TestResult(vadCount: vadResult, maxPeak: peakResult) + func getResults(for testCase: TestCase) -> TestResult { + TestResult(testCase: testCase, vadCount: vadResult, maxPeak: peakResult) } } @@ -128,7 +129,7 @@ class EchoTests: LKTestCase { _ = try await room1.localParticipant.performRpc(destinationIdentity: agentIdentity, method: Self.stopTest, payload: "") // Get final results from the actor - return await state.getResults() + return await state.getResults(for: testCase) } } @@ -149,18 +150,26 @@ class EchoTests: LKTestCase { let defaultResult = try await runEchoAgent(testCase: defaultTestCase) print("Result: \(defaultTestCase.title) \(defaultResult)") - print("\n======= Test Results Summary =======") - print("Default: \(defaultResult)") + // Store results for each test case + var testResults: [TestResult] = [] - // Run other test cases and compare with default + // Run other test cases for testCase in testCases { + print("Running \(testCase.title) test case...") let result = try await runEchoAgent(testCase: testCase) + testResults.append(result) print("Result: \(testCase.title) \(result)") + } + + // Print summary after all tests have completed + print("\n======= Test Results Summary =======") + print("Default: \(defaultResult)") + for result in testResults { let vadDiff = result.vadCount - defaultResult.vadCount let peakDiff = result.maxPeak - defaultResult.maxPeak - print("\(testCase.title): \(result)") + print("\(result)") print(" Compared to Default:") print(" - VAD difference: \(vadDiff > 0 ? "+" : "")\(vadDiff) events") print(" - Peak difference: \(peakDiff > 0 ? "+" : "")\(String(format: "%.2f", peakDiff)) dB")