Migrate tests to Swift Testing (#922)

- Better support for concurrency
- Explicit serialization, tags, etc.
- Parametrization

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: pblazej

.github/workflows/ci.yaml

@@ -16,9 +16,8 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  TEST_TARGETS: "LiveKitCoreTests LiveKitObjCTests"
-  TEST_TIMEOUT_MINUTES: 15
-  TEST_RETRY_ATTEMPTS: 2
+  TEST_TIMEOUT_MINUTES: 30
+  TEST_RETRY_ATTEMPTS: 3
 
 jobs:
   build-and-test:
@@ -92,6 +91,9 @@ jobs:
 
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
+    env:
+      LIBDISPATCH_COOPERATIVE_POOL_STRICT: ${{ matrix.strict-concurrency-env == true && '1' || '0' }}
+      NSUnbufferedIO: "YES"
     defaults:
       run:
         shell: bash
@@ -115,44 +117,22 @@ jobs:
       - name: Swift Version
         run: xcrun swift --version
 
-      - name: Build for Testing
-        run: |
-          set -o pipefail && xcodebuild build-for-testing \
-            -scheme LiveKit \
-            -destination 'platform=${{ matrix.platform }}' \
-            -enableAddressSanitizer ${{ matrix.asan == true && 'YES' || 'NO' }} \
-            -enableThreadSanitizer ${{ matrix.tsan == true && 'YES' || 'NO' }} \
-            APPLICATION_EXTENSION_API_ONLY=${{ matrix.extension-api-only == true && 'YES' || 'NO' }} \
-            | xcbeautify --renderer github-actions
-
-      - name: Run Tests
-        uses: nick-fields/retry@v3
-        env:
-          LIBDISPATCH_COOPERATIVE_POOL_STRICT: ${{ matrix.strict-concurrency-env == true && '1' || '0' }}
+      - name: Build & Test
+        uses: nick-fields/retry@v4
         with:
           timeout_minutes: ${{ env.TEST_TIMEOUT_MINUTES }}
           max_attempts: ${{ env.TEST_RETRY_ATTEMPTS }}
           command: |
-            set -euo pipefail
-            IFS=' ' read -r -a tests <<< "${{ env.TEST_TARGETS }}"
-
-            for test in "${tests[@]}"; do
-              echo "::group::$test"
-              if ! xcodebuild test-without-building \
-                -scheme LiveKit \
-                -destination 'platform=${{ matrix.platform }}' \
-                -enableAddressSanitizer ${{ matrix.asan == true && 'YES' || 'NO' }} \
-                -enableThreadSanitizer ${{ matrix.tsan == true && 'YES' || 'NO' }} \
-                APPLICATION_EXTENSION_API_ONLY=${{ matrix.extension-api-only == true && 'YES' || 'NO' }} \
-                -only-testing:$test \
-                -parallel-testing-enabled NO \
-                | xcbeautify --renderer github-actions
-              then
-                echo "::error::Test failed: $test"
-                exit 1
-              fi
-              echo "::endgroup::"
-            done
+            set -o pipefail && xcodebuild test \
+              -scheme LiveKit \
+              -destination 'platform=${{ matrix.platform }}' \
+              -enableAddressSanitizer ${{ matrix.asan == true && 'YES' || 'NO' }} \
+              -enableThreadSanitizer ${{ matrix.tsan == true && 'YES' || 'NO' }} \
+              APPLICATION_EXTENSION_API_ONLY=${{ matrix.extension-api-only == true && 'YES' || 'NO' }} \
+              -only-testing:LiveKitCoreTests \
+              -only-testing:LiveKitObjCTests \
+              -parallel-testing-enabled NO \
+              | xcbeautify --renderer github-actions
 
       - name: Build for Release
         if: ${{ matrix.symbol-graph }}
@@ -170,7 +150,7 @@ jobs:
 
       - name: Upload Symbol Graph
         if: ${{ matrix.symbol-graph }}
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: symbol-graph-${{ matrix.platform }}
           path: symbol-graph
@@ -200,7 +180,7 @@ jobs:
           sparse-checkout: Sources/LiveKit/LiveKit.docc
 
       - name: Download Symbol Graphs
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v8
         with:
           pattern: symbol-graph-*
           path: symbol-graphs
@@ -221,7 +201,7 @@ jobs:
         run: zip -r docs.zip docs
 
       - name: Upload Archive
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: docs
           path: docs.zip
@@ -275,7 +255,7 @@ jobs:
 
       - name: Comment on PR
         if: steps.check-changes.outputs.has_changes == 'false'
-        uses: mshick/add-pr-comment@v2
+        uses: mshick/add-pr-comment@v3
         with:
           message: |
             ⚠️ This PR does not contain any files in the `.changes` directory.

.swiftlint.yml

@@ -22,6 +22,14 @@ type_name:
     - OS
 
 custom_rules:
+  no_xctest_in_swift_tests:
+    name: "No XCTest in Swift Tests"
+    regex: "^import XCTest"
+    message: "Use Swift Testing (import Testing) instead of XCTest in Swift test targets."
+    severity: error
+    included:
+      - "Tests/LiveKitCoreTests/.*\\.swift"
+      - "Tests/LiveKitAudioTests/.*\\.swift"
   no_manual_task_management:
     name: "No Manual Task Management"
     regex: "(let|var)\\s+\\w+\\s*:\\s*Task<Void,"

AGENTS.md

@@ -174,7 +174,13 @@ Usage notes:
 
 <skill>
 <name>swift-concurrency</name>
-<description>'Expert guidance on Swift Concurrency best practices, patterns, and implementation. Use when developers mention: (1) Swift Concurrency, async/await, actors, or tasks, (2) "use Swift Concurrency" or "modern concurrency patterns", (3) migrating to Swift 6, (4) data races or thread safety issues, (5) refactoring closures to async/await, (6) @MainActor, Sendable, or actor isolation, (7) concurrent code architecture or performance optimization, (8) concurrency-related linter warnings (SwiftLint or similar; e.g. async_without_await, Sendable/actor isolation/MainActor lint).'</description>
+<description>'Diagnose data races, convert callback-based code to async/await, implement actor isolation patterns, resolve Sendable conformance issues, and guide Swift 6 migration. Use when developers mention: (1) Swift Concurrency, async/await, actors, or tasks, (2) "use Swift Concurrency" or "modern concurrency patterns", (3) migrating to Swift 6, (4) data races or thread safety issues, (5) refactoring closures to async/await, (6) @MainActor, Sendable, or actor isolation, (7) concurrent code architecture or performance optimization, (8) concurrency-related linter warnings (SwiftLint or similar; e.g. async_without_await, Sendable/actor isolation/MainActor lint).'</description>
+<location>global</location>
+</skill>
+
+<skill>
+<name>swift-testing-expert</name>
+<description>'Expert guidance for Swift Testing: test structure, #expect/#require macros, traits and tags, parameterized tests, test plans, parallel execution, async waiting patterns, and XCTest migration. Use when writing new Swift tests, modernizing XCTest suites, debugging flaky tests, or improving test quality and maintainability in Apple-platform or Swift server projects.'</description>
 <location>global</location>
 </skill>
 

Tests/LiveKitAudioTests/AudioEngineAvailability.swift

@@ -16,34 +16,35 @@
 
 @preconcurrency import AVFoundation
 @testable import LiveKit
+import Testing
 #if canImport(LiveKitTestSupport)
 import LiveKitTestSupport
 #endif
 
-class AudioEngineAvailabilityTests: LKTestCase {
+@Suite(.serialized, .tags(.audio)) struct AudioEngineAvailabilityTests {
     // Check if audio engine will stop when availability is set to .none,
     // then resume (restart) when availability is set back to .default.
-    func testRecording() async throws {
+    @Test func recording() throws {
         // Test without enabling VP
         try AudioManager.shared.setVoiceProcessingEnabled(false)
 
         // First check
-        XCTAssertFalse(AudioManager.shared.isEngineRunning)
+        #expect(!AudioManager.shared.isEngineRunning)
 
         // Start
         try AudioManager.shared.startLocalRecording()
-        XCTAssertTrue(AudioManager.shared.isEngineRunning)
+        #expect(AudioManager.shared.isEngineRunning)
 
         // Disable both input & output
         try AudioManager.shared.setEngineAvailability(.none)
-        XCTAssertFalse(AudioManager.shared.isEngineRunning)
+        #expect(!AudioManager.shared.isEngineRunning)
 
         // Re-enable both input & output (default)
         try AudioManager.shared.setEngineAvailability(.default)
-        XCTAssertTrue(AudioManager.shared.isEngineRunning)
+        #expect(AudioManager.shared.isEngineRunning)
 
         // Stop
         try AudioManager.shared.stopLocalRecording()
-        XCTAssertFalse(AudioManager.shared.isEngineRunning)
+        #expect(!AudioManager.shared.isEngineRunning)
     }
 }

Tests/LiveKitAudioTests/AudioEngineObserver.swift

@@ -16,6 +16,7 @@
 
 @preconcurrency import AVFoundation
 @testable import LiveKit
+import Testing
 #if canImport(LiveKitTestSupport)
 import LiveKitTestSupport
 #endif
@@ -33,11 +34,10 @@ final class TestEngineObserver: AudioEngineObserver, @unchecked Sendable {
     }
 }
 
-class AudioEngineObserverTests: LKTestCase {
+@Suite(.serialized, .tags(.audio)) struct AudioEngineObserverTests {
     // Error codes returned in an `AudioEngineObserver` should propagate through the WebRTC's AudioDeviceModule and
     // the SDK should throw in such cases for both device and manual rendering modes.
-    func testObserverFail() async throws {
-        //
+    @Test func observerFail() throws {
         let testObserver = TestEngineObserver()
 
         // Set test engine observer
@@ -47,7 +47,7 @@ class AudioEngineObserverTests: LKTestCase {
         try AudioManager.shared.setVoiceProcessingEnabled(false)
 
         // First check
-        XCTAssertFalse(AudioManager.shared.isEngineRunning)
+        #expect(!AudioManager.shared.isEngineRunning)
 
         #if os(iOS) || os(visionOS) || os(tvOS)
         try AVAudioSession.sharedInstance().setCategory(AVAudioSession.Category.playAndRecord)
@@ -57,24 +57,24 @@ class AudioEngineObserverTests: LKTestCase {
 
         // Attempt to start
         try AudioManager.shared.startLocalRecording()
-        XCTAssertTrue(AudioManager.shared.isEngineRunning)
+        #expect(AudioManager.shared.isEngineRunning)
 
         testObserver.shouldSucceed = false
 
         // Stop
-        XCTAssertThrowsError(try AudioManager.shared.stopLocalRecording())
-        XCTAssertFalse(AudioManager.shared.isEngineRunning)
+        #expect(throws: (any Error).self) { try AudioManager.shared.stopLocalRecording() }
+        #expect(!AudioManager.shared.isEngineRunning)
 
         testObserver.shouldSucceed = true
 
         try AudioManager.shared.stopLocalRecording()
-        XCTAssertFalse(AudioManager.shared.isEngineRunning)
+        #expect(!AudioManager.shared.isEngineRunning)
 
         testObserver.shouldSucceed = false
 
         // Attempt to start, should fail
-        XCTAssertThrowsError(try AudioManager.shared.startLocalRecording())
-        XCTAssertFalse(AudioManager.shared.isEngineRunning)
+        #expect(throws: (any Error).self) { try AudioManager.shared.startLocalRecording() }
+        #expect(!AudioManager.shared.isEngineRunning)
 
         // Switch to manual mode
         try AudioManager.shared.setManualRenderingMode(true)
@@ -83,6 +83,6 @@ class AudioEngineObserverTests: LKTestCase {
 
         // Attempt to start
         try AudioManager.shared.startLocalRecording()
-        XCTAssertFalse(AudioManager.shared.isEngineRunning)
+        #expect(!AudioManager.shared.isEngineRunning)
     }
 }

Tests/LiveKitAudioTests/AudioEnginePermission.swift

@@ -16,38 +16,39 @@
 
 @preconcurrency import AVFoundation
 @testable import LiveKit
+import Testing
 #if canImport(LiveKitTestSupport)
 import LiveKitTestSupport
 #endif
 
-class AudioEnginePermissionTests: LKTestCase {
+@Suite(.serialized, .tags(.audio)) struct AudioEnginePermissionTests {
     #if os(iOS) || os(visionOS) || os(tvOS)
     // Check if audio engine will fail to start instead of crashing when `AVAudioSession.category` isn't
     // configured correctly. Only for non-macOS platforms.
-    func testAudioSessionPermission() async throws {
+    @Test func audioSessionPermission() throws {
         // Test without enabling VP
         try AudioManager.shared.setVoiceProcessingEnabled(false)
 
         // First check
-        XCTAssertFalse(AudioManager.shared.isEngineRunning)
+        #expect(!AudioManager.shared.isEngineRunning)
 
         // Set no engine observer
         AudioManager.shared.set(engineObservers: [])
 
         // Attempt to start, should fail
-        XCTAssertThrowsError(try AudioManager.shared.startLocalRecording())
-        XCTAssertFalse(AudioManager.shared.isEngineRunning)
+        #expect(throws: (any Error).self) { try AudioManager.shared.startLocalRecording() }
+        #expect(!AudioManager.shared.isEngineRunning)
 
         // Set audio session engine observers
         AudioManager.shared.set(engineObservers: [AudioSessionEngineObserver()])
 
         // Attempt to start
         try AudioManager.shared.startLocalRecording()
-        XCTAssertTrue(AudioManager.shared.isEngineRunning)
+        #expect(AudioManager.shared.isEngineRunning)
 
         // Stop
         try AudioManager.shared.stopLocalRecording()
-        XCTAssertFalse(AudioManager.shared.isEngineRunning)
+        #expect(!AudioManager.shared.isEngineRunning)
 
         print("Category: \(AVAudioSession.sharedInstance().category)")
     }