Add LiveKit device benchmark matrix

Author: MarlonJD

.gitignore

@@ -6,7 +6,13 @@ xcuserdata/
 *.xcodeproj
 !Examples/DeviceHarness/LiveKitNativeDeviceHarness.xcodeproj/
 !Examples/DeviceHarness/LiveKitNativeDeviceHarness.xcodeproj/project.pbxproj
+!Examples/OfficialLiveKitBenchmarkHarness/OfficialLiveKitBenchmarkHarness.xcodeproj/
+!Examples/OfficialLiveKitBenchmarkHarness/OfficialLiveKitBenchmarkHarness.xcodeproj/project.pbxproj
 *.xcworkspace
+!Examples/OfficialLiveKitBenchmarkHarness/OfficialLiveKitBenchmarkHarness.xcodeproj/project.xcworkspace/
+!Examples/OfficialLiveKitBenchmarkHarness/OfficialLiveKitBenchmarkHarness.xcodeproj/project.xcworkspace/xcshareddata/
+!Examples/OfficialLiveKitBenchmarkHarness/OfficialLiveKitBenchmarkHarness.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/
+!Examples/OfficialLiveKitBenchmarkHarness/OfficialLiveKitBenchmarkHarness.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
 *.hmap
 *.ipa
 *.dSYM.zip

Benchmarks/Baselines/official-livekit-webrtc-template.csv

@@ -14,6 +14,7 @@ srtcp.packet_protect_unprotect,Official LiveKit Swift SDK + WebRTC baseline,0,0,
 dtls_srtp.exporter_split,Official LiveKit Swift SDK + WebRTC baseline,0,0,0,replace with measured release-build value
 dtls_srtp.session_protect_unprotect,Official LiveKit Swift SDK + WebRTC baseline,0,0,0,replace with measured release-build value
 h264.packetize_depacketize,Official LiveKit Swift SDK + WebRTC baseline,0,0,0,replace with measured release-build value
+h265.packetize_depacketize,Official LiveKit Swift SDK + WebRTC baseline,0,0,0,replace with measured release-build value
 vp8.payload_depacketize,Official LiveKit Swift SDK + WebRTC baseline,0,0,0,replace with measured release-build value
 opus.rtp_packetize_depacketize,Official LiveKit Swift SDK + WebRTC baseline,0,0,0,replace with measured release-build value
 sctp.dcep_open_ack_roundtrip,Official LiveKit Swift SDK + WebRTC baseline,0,0,0,replace with measured release-build value

Benchmarks/LiveKitNativeBenchmarks/main.swift

@@ -404,6 +404,10 @@ func makeBenchmarks(blackhole: BenchmarkBlackhole) throws -> [BenchmarkCase] {
     h264NALUnit.append(Data(repeating: 0x11, count: 3_000))
     let h264Packetizer = H264RTPPacketizer(payloadType: 102, mtu: 1_200)
 
+    var h265NALUnit = H265NALUnitHeader.payloadHeader(type: H265NALUnitType.idrWithRADL.rawValue)
+    h265NALUnit.append(Data(repeating: 0x33, count: 3_000))
+    let h265Packetizer = H265RTPPacketizer(payloadType: 104, mtu: 1_200)
+
     var vp8Payload = Data([0x10])
     vp8Payload.append(Data([0x9D, 0x01, 0x2A, 0x80, 0x02, 0xE0, 0x01]))
     vp8Payload.append(Data(repeating: 0x22, count: 1_000))
@@ -568,6 +572,20 @@ func makeBenchmarks(blackhole: BenchmarkBlackhole) throws -> [BenchmarkCase] {
             }
             blackhole.consume(units + packets.count)
         },
+        BenchmarkCase(name: "h265.packetize_depacketize", category: "video", operationsPerSample: 50) {
+            let packets = try h265Packetizer.packetize(
+                nalUnits: [h265NALUnit],
+                timestamp: 90_000,
+                ssrc: 11,
+                startingSequenceNumber: 1
+            )
+            let depacketizer = H265RTPDepacketizer()
+            var units = 0
+            for packet in packets {
+                units += try depacketizer.append(packet).count
+            }
+            blackhole.consume(units + packets.count)
+        },
         BenchmarkCase(name: "vp8.payload_depacketize", category: "video", operationsPerSample: 500) {
             let fragment = try vp8Depacketizer.depacketize(vp8Packet)
             blackhole.consume(fragment.payload.count + Int(fragment.sequenceNumber))

Examples/DeviceHarness/DeviceHarnessDiagnosticsSnapshot.swift

@@ -12,8 +12,11 @@ struct DeviceHarnessDiagnosticsSnapshot {
     var subscriberRemoteAudioPublicationSIDs: [String]
     var subscriberRemoteVideoPublicationSIDs: [String]
     var batteryLevel: Float
+    var batteryState: String
     var thermalState: String
     var memoryPeakResidentBytes: Int64
+    var appUptimeSeconds: Double
+    var processCPUTimeSeconds: Double
 
     var textReport: String {
         [
@@ -48,8 +51,11 @@ struct DeviceHarnessDiagnosticsSnapshot {
             "apple_microphone_active_mode: \(diagnostics.appleMicrophoneModeState.activeMode.rawValue)",
             "apple_microphone_ui_available: \(diagnostics.appleMicrophoneModeState.systemUserInterfaceAvailable)",
             "battery_level: \(batteryLevel)",
+            "battery_state: \(batteryState)",
             "thermal_state: \(thermalState)",
             "memory_peak_resident_bytes: \(memoryPeakResidentBytes)",
+            "app_uptime_seconds: \(Self.formatSeconds(appUptimeSeconds))",
+            "process_cpu_time_seconds: \(Self.formatSeconds(processCPUTimeSeconds))",
         ].joined(separator: "\n")
     }
 
@@ -60,4 +66,8 @@ struct DeviceHarnessDiagnosticsSnapshot {
     private static func formatSSRCs(_ ssrcs: [UInt32]) -> String {
         ssrcs.isEmpty ? "none" : ssrcs.sorted().map(String.init).joined(separator: ",")
     }
+
+    private static func formatSeconds(_ seconds: Double) -> String {
+        String(format: "%.3f", seconds)
+    }
 }

Examples/DeviceHarness/DeviceHarnessViewModel.swift

@@ -45,6 +45,7 @@ final class DeviceHarnessViewModel: ObservableObject {
     private var observers: [NSObjectProtocol] = []
     private var didRunAutomation = false
     private var allowDirectICE = false
+    private var foregroundRecoveryEnabled = false
     private var subscriberToken = ""
     private var subscriberConnectTask: Task<Void, Never>?
     private var observedLifecycleEvents: Set<String> = []
@@ -60,6 +61,7 @@ final class DeviceHarnessViewModel: ObservableObject {
     private var foregroundSoakReconnectAttemptCount = 0
     private var isSoakReconnectInProgress = false
     private var soakReconnectLeaseExpiresAt: Date?
+    private let appLaunchDate = Date()
     private static let voiceIsolationRecommendationKey =
         "LiveKitNativeDeviceHarness.didSuggestAppleVoiceIsolation"
 
@@ -70,6 +72,8 @@ final class DeviceHarnessViewModel: ObservableObject {
         execution_environment: \(Self.executionEnvironment)
         device_model: \(UIDevice.current.model)
         system_version: \(UIDevice.current.systemName) \(UIDevice.current.systemVersion)
+        app_uptime_seconds: \(Self.formatSeconds(appUptimeSeconds))
+        process_cpu_time_seconds: \(Self.formatSeconds(DeviceHarnessProcessSampler.cpuTimeSeconds()))
         """
         status = "App launch observed."
         diagnosticsText = appInstallReport
@@ -129,7 +133,7 @@ final class DeviceHarnessViewModel: ObservableObject {
                 try await publishInitialMedia(
                     room: room,
                     audioOptions: AudioCaptureOptions(echoCancellation: true),
-                    cameraOptions: CameraCaptureOptions(width: 1280, height: 720, framesPerSecond: 30),
+                    cameraOptions: CameraCaptureOptions(width: 720, height: 1280, framesPerSecond: 30),
                     cameraSimulcast: true
                 )
             }
@@ -211,6 +215,7 @@ final class DeviceHarnessViewModel: ObservableObject {
         subscriberToken = environment["LIVEKIT_NATIVE_DEVICE_HARNESS_SUBSCRIBER_TOKEN"] ?? subscriberToken
         batterySaverWebRTC = environment["LIVEKIT_NATIVE_DEVICE_HARNESS_BATTERY_SAVER_WEBRTC"] == "1"
         allowDirectICE = environment["LIVEKIT_NATIVE_DEVICE_HARNESS_ALLOW_DIRECT_ICE"] == "1"
+        foregroundRecoveryEnabled = environment["LIVEKIT_NATIVE_DEVICE_HARNESS_FOREGROUND_RECOVERY"] == "1"
         observedLifecycleEvents.removeAll()
         emitHarnessLog("scripted_validation: autostart requested")
         if batterySaverWebRTC {
@@ -242,7 +247,7 @@ final class DeviceHarnessViewModel: ObservableObject {
                 diagnosticIntervalSeconds: Self.positiveInteger(
                     environment["LIVEKIT_NATIVE_DEVICE_HARNESS_DIAGNOSTIC_INTERVAL_SECONDS"]
                 ) ?? 60,
-                reconnectAfterSeconds: Self.positiveInteger(
+                reconnectAfterSeconds: Self.nonNegativeInteger(
                     environment["LIVEKIT_NATIVE_DEVICE_HARNESS_RECONNECT_AFTER_SECONDS"]
                 )
             )
@@ -398,8 +403,11 @@ final class DeviceHarnessViewModel: ObservableObject {
             subscriberRemoteAudioPublicationSIDs: subscriberRemotePublications.audio,
             subscriberRemoteVideoPublicationSIDs: subscriberRemotePublications.video,
             batteryLevel: UIDevice.current.batteryLevel,
+            batteryState: UIDevice.current.batteryState.benchmarkDescription,
             thermalState: thermalState.description,
-            memoryPeakResidentBytes: DeviceHarnessMemorySampler.peakResidentBytes()
+            memoryPeakResidentBytes: DeviceHarnessMemorySampler.peakResidentBytes(),
+            appUptimeSeconds: appUptimeSeconds,
+            processCPUTimeSeconds: DeviceHarnessProcessSampler.cpuTimeSeconds()
         ).textReport
         emitHarnessLog(diagnosticsText)
         await abortSoakIfThermalStateDisallowed(thermalState)
@@ -759,10 +767,14 @@ final class DeviceHarnessViewModel: ObservableObject {
             if isSoakAutomationRunning {
                 observedLifecycleEvents.insert("background-foreground-recovery")
                 await refreshDiagnostics(reason: "background-foreground-recovery")
-                scheduleForegroundSoakRecoveryIfNeeded(initialDiagnostics: diagnostics)
+                if foregroundRecoveryEnabled {
+                    scheduleForegroundSoakRecoveryIfNeeded(initialDiagnostics: diagnostics)
+                }
                 return
             }
-            await recoverAfterForegroundIfNeeded(initialDiagnostics: diagnostics)
+            if foregroundRecoveryEnabled {
+                await recoverAfterForegroundIfNeeded(initialDiagnostics: diagnostics)
+            }
             observedLifecycleEvents.insert("background-foreground-recovery")
             await refreshDiagnostics(reason: "background-foreground-recovery")
         }
@@ -1058,7 +1070,7 @@ final class DeviceHarnessViewModel: ObservableObject {
         let clampedDurationMinutes = min(max(durationMinutes, 1), 24 * 60)
         let totalSeconds = clampedDurationMinutes * 60
         let intervalSeconds = min(max(diagnosticIntervalSeconds, 5), totalSeconds)
-        let reconnectAtSeconds = min(
+        let reconnectAtSeconds = reconnectAfterSeconds == 0 ? nil : min(
             max(reconnectAfterSeconds ?? max(30, totalSeconds / 2), 5),
             max(5, totalSeconds - intervalSeconds)
         )
@@ -1093,14 +1105,16 @@ final class DeviceHarnessViewModel: ObservableObject {
         emitHarnessLog(Self.soakHarnessBuildMarker)
         emitHarnessLog(soakLowLoadProfileMarker)
         emitHarnessLog(
-            "soak_started: duration_minutes=\(clampedDurationMinutes) diagnostic_interval_seconds=\(intervalSeconds) reconnect_after_seconds=\(reconnectAtSeconds)"
+            "soak_started: duration_minutes=\(clampedDurationMinutes) diagnostic_interval_seconds=\(intervalSeconds) reconnect_after_seconds=\(reconnectAtSeconds.map(String.init) ?? "disabled")"
         )
         await refreshDiagnostics(reason: "soak-start", includeDataChannels: false)
 
         while Int(Date().timeIntervalSince(start)) < totalSeconds {
             let elapsedSeconds = Int(Date().timeIntervalSince(start))
             var reconnectReason: String?
-            if reconnectCount + foregroundSoakReconnectCount == 0 && elapsedSeconds >= reconnectAtSeconds {
+            if let reconnectAtSeconds,
+               reconnectCount + foregroundSoakReconnectCount == 0,
+               elapsedSeconds >= reconnectAtSeconds {
                 reconnectReason = "scheduled"
             }
 
@@ -1251,9 +1265,10 @@ final class DeviceHarnessViewModel: ObservableObject {
     }
 
     private var soakLowLoadProfileMarker: String {
-        let captureProfile = batterySaverWebRTC ? "320x180@10fps" : "default"
+        let captureProfile = batterySaverWebRTC ? "320x180@10fps" : "720x1280@30fps"
         let bitrate = batterySaverWebRTC ? "250000" : "default"
-        return "soak_low_load_profile: battery_saver_webrtc=\(batterySaverWebRTC) capture=\(captureProfile) h264_bitrate=\(bitrate) active_video_layers_target=1 decode_subscriber_video=true subscriber_adaptive=false subscriber_allow_pause=false ice_transport_policy=\(allowDirectICE ? "all" : "relay")"
+        let activeVideoLayersTarget = batterySaverWebRTC ? 1 : 3
+        return "soak_low_load_profile: battery_saver_webrtc=\(batterySaverWebRTC) capture=\(captureProfile) h264_bitrate=\(bitrate) active_video_layers_target=\(activeVideoLayersTarget) decode_subscriber_video=true subscriber_adaptive=false subscriber_allow_pause=false ice_transport_policy=\(allowDirectICE ? "all" : "relay")"
     }
 
     private func reconnectForSoak(reason: String, attempt: Int) async -> Bool {
@@ -1741,6 +1756,13 @@ final class DeviceHarnessViewModel: ObservableObject {
         return integer
     }
 
+    private static func nonNegativeInteger(_ value: String?) -> Int? {
+        guard let value, let integer = Int(value), integer >= 0 else {
+            return nil
+        }
+        return integer
+    }
+
     private static func joinedOrNone(_ values: [String]) -> String {
         values.isEmpty ? "none" : values.sorted().joined(separator: ",")
     }
@@ -1816,6 +1838,14 @@ final class DeviceHarnessViewModel: ObservableObject {
         "physical-device"
         #endif
     }
+
+    private var appUptimeSeconds: Double {
+        Date().timeIntervalSince(appLaunchDate)
+    }
+
+    private static func formatSeconds(_ seconds: Double) -> String {
+        String(format: "%.3f", seconds)
+    }
 }
 
 private enum DeviceHarnessMemorySampler {
@@ -1828,6 +1858,37 @@ private enum DeviceHarnessMemorySampler {
     }
 }
 
+private enum DeviceHarnessProcessSampler {
+    static func cpuTimeSeconds() -> Double {
+        var usage = rusage()
+        guard getrusage(RUSAGE_SELF, &usage) == 0 else {
+            return -1
+        }
+        return seconds(from: usage.ru_utime) + seconds(from: usage.ru_stime)
+    }
+
+    private static func seconds(from time: timeval) -> Double {
+        Double(time.tv_sec) + Double(time.tv_usec) / 1_000_000
+    }
+}
+
+private extension UIDevice.BatteryState {
+    var benchmarkDescription: String {
+        switch self {
+        case .unknown:
+            return "unknown"
+        case .unplugged:
+            return "unplugged"
+        case .charging:
+            return "charging"
+        case .full:
+            return "full"
+        @unknown default:
+            return "unknown"
+        }
+    }
+}
+
 private struct SubscriberCompanionDiscoveryError: LocalizedError {
     var missingKinds: [String]
     var remoteAudioSIDs: [String]

Examples/DeviceHarness/Info.plist

@@ -35,6 +35,8 @@
 	<string>Camera access is required to validate LiveKitNative video publishing on physical iOS devices.</string>
 	<key>NSMicrophoneUsageDescription</key>
 	<string>Microphone access is required to validate LiveKitNative audio publishing and playout on physical iOS devices.</string>
+	<key>NSLocalNetworkUsageDescription</key>
+	<string>Local network access is required to connect to the benchmark LiveKit server.</string>
 	<key>UIBackgroundModes</key>
 	<array>
 		<string>audio</string>

Examples/DeviceHarness/README.md

@@ -15,8 +15,8 @@ The harness intentionally keeps the app small:
   in-process subscriber connection.
 - Includes a Battery Saver WebRTC toggle that opts into `.batterySaver` with
   automatic Low Power Mode and thermal adaptation.
-- Records diagnostics for lifecycle, route-change, interruption, battery,
-  thermal, media continuity, and data-channel state.
+- Records diagnostics for lifecycle, route-change, interruption, battery
+  level/state, thermal, media continuity, and data-channel state.
 - Emits app launch, permission, and diagnostics snapshots to the device log,
   including `execution_environment: physical-device` so proof jobs cannot pass
   with simulator-only logs.
@@ -63,6 +63,10 @@ physical-device room token. Pass `--allow-provisioning-updates` when Xcode must
 create or refresh the physical-device provisioning profile, or pass
 `--use-installed-app` when the harness app is already installed with the
 requested bundle id and the runner only needs to launch and collect soak proof.
+For short A/B decision runs against a previous library, set
+`--benchmark-log-only --reconnect-after-seconds 0`, run both libraries for
+10 minutes unplugged, and compare the logs with
+`scripts/write_device_power_benchmark_report.sh`.
 The report uses the same `release-gate-report.txt` field contract enforced by
 the strict release gate. The lower-level
 `scripts/write_device_harness_gate_report.sh` and

Examples/OfficialLiveKitBenchmarkHarness/Info.plist

@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "https://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>CFBundleDisplayName</key>
+	<string>Official LiveKit Benchmark</string>
+	<key>CFBundleExecutable</key>
+	<string>$(EXECUTABLE_NAME)</string>
+	<key>CFBundleIdentifier</key>
+	<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
+	<key>CFBundleName</key>
+	<string>$(PRODUCT_NAME)</string>
+	<key>CFBundlePackageType</key>
+	<string>APPL</string>
+	<key>CFBundleShortVersionString</key>
+	<string>$(MARKETING_VERSION)</string>
+	<key>CFBundleVersion</key>
+	<string>$(CURRENT_PROJECT_VERSION)</string>
+	<key>UILaunchScreen</key>
+	<dict/>
+	<key>UISupportedInterfaceOrientations</key>
+	<array>
+		<string>UIInterfaceOrientationPortrait</string>
+		<string>UIInterfaceOrientationLandscapeLeft</string>
+		<string>UIInterfaceOrientationLandscapeRight</string>
+	</array>
+	<key>UISupportedInterfaceOrientations~ipad</key>
+	<array>
+		<string>UIInterfaceOrientationPortrait</string>
+		<string>UIInterfaceOrientationPortraitUpsideDown</string>
+		<string>UIInterfaceOrientationLandscapeLeft</string>
+		<string>UIInterfaceOrientationLandscapeRight</string>
+	</array>
+	<key>NSCameraUsageDescription</key>
+	<string>Camera access is required to benchmark the official LiveKit Swift SDK on physical iOS devices.</string>
+	<key>NSMicrophoneUsageDescription</key>
+	<string>Microphone access is required to benchmark the official LiveKit Swift SDK on physical iOS devices.</string>
+	<key>NSLocalNetworkUsageDescription</key>
+	<string>Local network access is required to connect to the benchmark LiveKit server.</string>
+	<key>UIBackgroundModes</key>
+	<array>
+		<string>audio</string>
+		<string>voip</string>
+	</array>
+</dict>
+</plist>