Improve gesture tracking responsiveness

Author: NakaokaRei

TrackpadAir/Helper/ImageHelper.swift

@@ -9,17 +9,18 @@ import Foundation
 import AVFoundation
 import Cocoa
 
+private let imageContext = CIContext(options: [.cacheIntermediates: false])
+
 func NSImageFromSampleBuffer(_ sampleBuffer: CMSampleBuffer) -> NSImage? {
     guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
         return nil
     }
 
     let ciImage = CIImage(cvImageBuffer: pixelBuffer)
-    let context = CIContext(options: nil)
     let width = CVPixelBufferGetWidth(pixelBuffer)
     let height = CVPixelBufferGetHeight(pixelBuffer)
 
-    guard let cgImage = context.createCGImage(ciImage, from: CGRect(x: 0, y: 0, width: width, height: height)) else {
+    guard let cgImage = imageContext.createCGImage(ciImage, from: CGRect(x: 0, y: 0, width: width, height: height)) else {
         return nil
     }
 

TrackpadAir/Manager/HandGestureProcessor.swift

@@ -91,19 +91,180 @@ enum HandGestureState: String, CaseIterable, Identifiable {
     }
 }
 
-class HandGestureProcessor {
+struct HandGestureUpdate {
+    let gesture: HandGestureState
+    let fingerTips: FingerTips?
+    let didTransition: Bool
+}
+
+final class HandGestureProcessor {
+    struct Configuration {
+        let smoothingAlpha: CGFloat
+        let activationThreshold: CGFloat
+        let releaseThreshold: CGFloat
+        let confirmationFrames: Int
+        let missingFramesBeforeReset: Int
+
+        static let balanced = Configuration(
+            smoothingAlpha: 0.55,
+            activationThreshold: 0.34,
+            releaseThreshold: 0.46,
+            confirmationFrames: 2,
+            missingFramesBeforeReset: 2
+        )
+    }
+
+    private let configuration: Configuration
+    private var filteredFrame: HandPoseFrame?
+    private var stableGesture: HandGestureState = .none
+    private var candidateGesture: HandGestureState?
+    private var candidateFrameCount = 0
+    private var missingFrameCount = 0
 
-    static let threshold: Double = 6
+    init(configuration: Configuration = .balanced) {
+        self.configuration = configuration
+    }
+
+    func process(frame: HandPoseFrame?) -> HandGestureUpdate {
+        guard let frame else {
+            return processMissingFrame()
+        }
+
+        missingFrameCount = 0
+        let smoothedFrame = smooth(frame)
+        filteredFrame = smoothedFrame
+
+        let detectedGesture: HandGestureState
+        if stableGesture != .none,
+           Self.matches(
+               stableGesture,
+               frame: smoothedFrame,
+               threshold: configuration.releaseThreshold
+           ) {
+            detectedGesture = stableGesture
+        } else {
+            detectedGesture = Self.classify(
+                frame: smoothedFrame,
+                threshold: configuration.activationThreshold
+            )
+        }
 
-    static func process(fingerTips: FingerTips) -> HandGestureState {
+        let didTransition = confirm(detectedGesture)
+        return HandGestureUpdate(
+            gesture: stableGesture,
+            fingerTips: smoothedFrame.fingerTips,
+            didTransition: didTransition
+        )
+    }
+
+    func reset() {
+        filteredFrame = nil
+        stableGesture = .none
+        candidateGesture = nil
+        candidateFrameCount = 0
+        missingFrameCount = 0
+    }
+
+    static func classify(
+        frame: HandPoseFrame,
+        threshold: CGFloat = Configuration.balanced.activationThreshold
+    ) -> HandGestureState {
         for definition in GestureDefinition.priorityOrdered {
-            if definition.matches(fingerTips: fingerTips, threshold: threshold) {
+            if definition.matches(frame: frame, threshold: threshold) {
                 return definition.state
             }
         }
-
         return .none
     }
+
+    private static func matches(
+        _ gesture: HandGestureState,
+        frame: HandPoseFrame,
+        threshold: CGFloat
+    ) -> Bool {
+        guard let definition = GestureDefinition.priorityOrdered.first(where: {
+            $0.state == gesture
+        }) else {
+            return false
+        }
+        return definition.matches(frame: frame, threshold: threshold)
+    }
+
+    private func processMissingFrame() -> HandGestureUpdate {
+        missingFrameCount += 1
+        guard missingFrameCount >= configuration.missingFramesBeforeReset else {
+            return HandGestureUpdate(
+                gesture: stableGesture,
+                fingerTips: filteredFrame?.fingerTips,
+                didTransition: false
+            )
+        }
+
+        let didTransition = stableGesture != .none
+        filteredFrame = nil
+        stableGesture = .none
+        candidateGesture = nil
+        candidateFrameCount = 0
+        return HandGestureUpdate(
+            gesture: .none,
+            fingerTips: nil,
+            didTransition: didTransition
+        )
+    }
+
+    private func confirm(_ detectedGesture: HandGestureState) -> Bool {
+        guard detectedGesture != stableGesture else {
+            candidateGesture = nil
+            candidateFrameCount = 0
+            return false
+        }
+
+        if candidateGesture == detectedGesture {
+            candidateFrameCount += 1
+        } else {
+            candidateGesture = detectedGesture
+            candidateFrameCount = 1
+        }
+
+        guard candidateFrameCount >= configuration.confirmationFrames else {
+            return false
+        }
+
+        stableGesture = detectedGesture
+        candidateGesture = nil
+        candidateFrameCount = 0
+        return true
+    }
+
+    private func smooth(_ frame: HandPoseFrame) -> HandPoseFrame {
+        guard let previous = filteredFrame else {
+            return frame
+        }
+
+        return HandPoseFrame(
+            fingerTips: FingerTips(
+                thumb: smooth(previous.fingerTips.thumb, frame.fingerTips.thumb),
+                index: smooth(previous.fingerTips.index, frame.fingerTips.index),
+                middle: smooth(previous.fingerTips.middle, frame.fingerTips.middle),
+                ring: smooth(previous.fingerTips.ring, frame.fingerTips.ring),
+                little: smooth(previous.fingerTips.little, frame.fingerTips.little)
+            ),
+            handScale: interpolate(previous.handScale, frame.handScale),
+            confidence: frame.confidence,
+            aspectRatio: frame.aspectRatio
+        )
+    }
+
+    private func smooth(_ previous: CGPoint, _ current: CGPoint) -> CGPoint {
+        CGPoint(
+            x: interpolate(previous.x, current.x),
+            y: interpolate(previous.y, current.y)
+        )
+    }
+
+    private func interpolate(_ previous: CGFloat, _ current: CGFloat) -> CGFloat {
+        previous + configuration.smoothingAlpha * (current - previous)
+    }
 }
 
 private struct GestureDefinition {
@@ -142,25 +303,36 @@ private struct GestureDefinition {
         self.excludedPairs = excludedPairs
     }
 
-    func matches(fingerTips: FingerTips, threshold: Double) -> Bool {
+    func matches(frame: HandPoseFrame, threshold: CGFloat) -> Bool {
+        let distanceThreshold = frame.handScale * threshold
         let requiredPairsMatch = requiredPairs.allSatisfy {
-            distance(fingerTips: fingerTips, pair: $0) < threshold
+            distance(
+                fingerTips: frame.fingerTips,
+                pair: $0,
+                aspectRatio: frame.aspectRatio
+            ) < distanceThreshold
         }
         let excludedPairsMatch = excludedPairs.allSatisfy {
-            distance(fingerTips: fingerTips, pair: $0) >= threshold
+            distance(
+                fingerTips: frame.fingerTips,
+                pair: $0,
+                aspectRatio: frame.aspectRatio
+            ) >= distanceThreshold
         }
 
         return requiredPairsMatch && excludedPairsMatch
     }
 
     private func distance(
         fingerTips: FingerTips,
-        pair: (FingerTipKeyPath, FingerTipKeyPath)
+        pair: (FingerTipKeyPath, FingerTipKeyPath),
+        aspectRatio: CGFloat
     ) -> Double {
-        CoordinateHelper.distance(
-            p1: pair.0.point(in: fingerTips),
-            p2: pair.1.point(in: fingerTips)
-        )
+        let p1 = pair.0.point(in: fingerTips)
+        let p2 = pair.1.point(in: fingerTips)
+        let dx = (p1.x - p2.x) * aspectRatio
+        let dy = p1.y - p2.y
+        return hypot(dx, dy)
     }
 }