diff --git a/Gaze/Constants/EyeTrackingConstants.swift b/Gaze/Constants/EyeTrackingConstants.swift deleted file mode 100644 index a408228..0000000 --- a/Gaze/Constants/EyeTrackingConstants.swift +++ /dev/null @@ -1,87 +0,0 @@ -// -// EyeTrackingConstants.swift -// Gaze -// -// Created by Mike Freno on 1/14/26. -// - -import Foundation - -/// Thread-safe configuration holder for eye tracking thresholds. -/// All properties are Sendable constants, safe for use in any concurrency context. -enum EyeTrackingConstants: Sendable { - // MARK: - Logging - /// Interval between log messages in seconds - static let logInterval: TimeInterval = 0.5 - - // MARK: - Eye Closure Detection - /// Threshold for eye closure (smaller value means eye must be more closed to trigger) - /// Range: 0.0 to 1.0 (approximate eye opening ratio) - static let eyeClosedThreshold: CGFloat = 0.02 - static let eyeClosedEnabled: Bool = true - - // MARK: - Face Pose Thresholds - /// Maximum yaw (left/right head turn) in radians before considering user looking away - /// 0.20 radians ≈ 11.5 degrees (Tightened from 0.35) - /// NOTE: Vision Framework often provides unreliable yaw/pitch on macOS - disabled by default - static let yawThreshold: Double = 0.3 - static let yawEnabled: Bool = false - - /// Pitch threshold for looking UP (above screen). - /// Since camera is at top, looking at screen is negative pitch. - /// Values > 0.1 imply looking straight ahead or up (away from screen). - /// NOTE: Vision Framework often doesn't provide pitch data on macOS - disabled by default - static let pitchUpThreshold: Double = 0.1 - static let pitchUpEnabled: Bool = false - - /// Pitch threshold for looking DOWN (at keyboard/lap). - /// Values < -0.45 imply looking too far down. - /// NOTE: Vision Framework often doesn't provide pitch data on macOS - disabled by default - static let pitchDownThreshold: Double = -0.45 - static let pitchDownEnabled: Bool = false - - // MARK: - Pupil Tracking Thresholds - /// Minimum horizontal pupil ratio (0.0 = right edge, 1.0 = left edge) - /// Values below this are considered looking right (camera view) - /// Tightened to 0.35 based on observed values (typically 0.31-0.47) - static let minPupilRatio: Double = 0.35 - static let minPupilEnabled: Bool = true - - /// Maximum horizontal pupil ratio - /// Values above this are considered looking left (camera view) - /// Tightened to 0.45 based on observed values (typically 0.31-0.47) - static let maxPupilRatio: Double = 0.45 - static let maxPupilEnabled: Bool = true - - // MARK: - Pixel-Based Gaze Detection Thresholds - /// Thresholds for pupil-based gaze detection - /// Based on video test data: - /// - Looking at screen (center): H ≈ 0.20-0.50 - /// - Looking left (away): H ≈ 0.50+ - /// - Looking right (away): H ≈ 0.20- - /// Coordinate system: Lower values = right, Higher values = left - static let pixelGazeMinRatio: Double = 0.20 // Below this = looking right (away) - static let pixelGazeMaxRatio: Double = 0.50 // Above this = looking left (away) - static let pixelGazeEnabled: Bool = true - - // MARK: - Screen Boundary Detection (New) - - /// Forgiveness margin for the "gray area" around the screen edge. - /// 0.05 means the safe zone is extended by 5% of the range on each side. - /// If in the gray area, we assume the user is Looking Away (success). - static let boundaryForgivenessMargin: Double = 0.05 - - /// Distance sensitivity factor. - /// 1.0 = Linear scaling (face width 50% smaller -> eye movement expected to be 50% smaller) - /// > 1.0 = More aggressive scaling - static let distanceSensitivity: Double = 1.0 - - /// Default reference face width for distance scaling when uncalibrated. - /// Measured from test videos at typical laptop distance (~60cm). - /// Face bounding box width as ratio of image width. - static let defaultReferenceFaceWidth: Double = 0.4566 - - /// Minimum confidence required for a valid pupil detection before updating the gaze average. - /// Helps filter out blinks or noisy frames. - static let minimumGazeConfidence: Int = 3 // consecutive valid frames -} diff --git a/Gaze/Models/CalibrationData.swift b/Gaze/Models/CalibrationData.swift deleted file mode 100644 index 417559a..0000000 --- a/Gaze/Models/CalibrationData.swift +++ /dev/null @@ -1,281 +0,0 @@ -// -// CalibrationData.swift -// Gaze -// -// Created by Mike Freno on 1/15/26. -// - -import Foundation - -// MARK: - Calibration Models - -enum CalibrationStep: String, Codable, CaseIterable { - case center - case farLeft - case left - case farRight - case right - case up - case down - case topLeft - case topRight - case bottomLeft - case bottomRight - - var displayName: String { - switch self { - case .center: return "Center" - case .farLeft: return "Far Left" - case .left: return "Left" - case .farRight: return "Far Right" - case .right: return "Right" - case .up: return "Up" - case .down: return "Down" - case .topLeft: return "Top Left" - case .topRight: return "Top Right" - case .bottomLeft: return "Bottom Left" - case .bottomRight: return "Bottom Right" - } - } - - var instructionText: String { - switch self { - case .center: - return "Look at the center of the screen" - case .farLeft: - return "Look as far left as comfortable" - case .left: - return "Look to the left edge of the screen" - case .farRight: - return "Look as far right as comfortable" - case .right: - return "Look to the right edge of the screen" - case .up: - return "Look to the top edge of the screen" - case .down: - return "Look to the bottom edge of the screen" - case .topLeft: - return "Look to the top left corner" - case .topRight: - return "Look to the top right corner" - case .bottomLeft: - return "Look to the bottom left corner" - case .bottomRight: - return "Look to the bottom right corner" - } - } -} - -struct GazeSample: Codable { - let leftRatio: Double? - let rightRatio: Double? - let averageRatio: Double - let leftVerticalRatio: Double? - let rightVerticalRatio: Double? - let averageVerticalRatio: Double - let faceWidthRatio: Double? // For distance scaling (face width / image width) - let timestamp: Date - - init( - leftRatio: Double?, - rightRatio: Double?, - leftVerticalRatio: Double? = nil, - rightVerticalRatio: Double? = nil, - faceWidthRatio: Double? = nil - ) { - self.leftRatio = leftRatio - self.rightRatio = rightRatio - self.leftVerticalRatio = leftVerticalRatio - self.rightVerticalRatio = rightVerticalRatio - self.faceWidthRatio = faceWidthRatio - - self.averageRatio = GazeSample.average(left: leftRatio, right: rightRatio, fallback: 0.5) - self.averageVerticalRatio = GazeSample.average( - left: leftVerticalRatio, - right: rightVerticalRatio, - fallback: 0.5 - ) - - self.timestamp = Date() - } - - private static func average(left: Double?, right: Double?, fallback: Double) -> Double { - switch (left, right) { - case let (left?, right?): - return (left + right) / 2.0 - case let (left?, nil): - return left - case let (nil, right?): - return right - default: - return fallback - } - } -} - -struct GazeThresholds: Codable { - // Horizontal Thresholds - let minLeftRatio: Double // Looking left (≥ value) - let maxRightRatio: Double // Looking right (≤ value) - - // Vertical Thresholds - let minUpRatio: Double // Looking up (≤ value, typically < 0.5) - let maxDownRatio: Double // Looking down (≥ value, typically > 0.5) - - // Screen Bounds (Calibration Zone) - // Defines the rectangle of pupil ratios that correspond to looking AT the screen - let screenLeftBound: Double - let screenRightBound: Double - let screenTopBound: Double - let screenBottomBound: Double - - // Reference Data for Distance Scaling - let referenceFaceWidth: Double // Average face width during calibration - - var isValid: Bool { - isFiniteValues([ - minLeftRatio, maxRightRatio, minUpRatio, maxDownRatio, - screenLeftBound, screenRightBound, screenTopBound, screenBottomBound, - ]) - } - - private func isFiniteValues(_ values: [Double]) -> Bool { - values.allSatisfy { $0.isFinite } - } - - /// Default thresholds based on video test data: - /// - Center (looking at screen): H ≈ 0.29-0.35 - /// - Screen left edge: H ≈ 0.45-0.50 - /// - Looking away left: H ≈ 0.55+ - /// - Screen right edge: H ≈ 0.20-0.25 - /// - Looking away right: H ≈ 0.15- - /// Coordinate system: Lower H = right, Higher H = left - static var defaultThresholds: GazeThresholds { - GazeThresholds( - minLeftRatio: 0.55, // Beyond this = looking left (away) - maxRightRatio: 0.15, // Below this = looking right (away) - minUpRatio: 0.30, // Below this = looking up (away) - maxDownRatio: 0.60, // Above this = looking down (away) - screenLeftBound: 0.50, // Left edge of screen - screenRightBound: 0.20, // Right edge of screen - screenTopBound: 0.35, // Top edge of screen - screenBottomBound: 0.55, // Bottom edge of screen - referenceFaceWidth: 0.4566 // Measured from test videos (avg of inner/outer) - ) - } -} - -struct CalibrationData: Codable { - var samples: [CalibrationStep: [GazeSample]] - var computedThresholds: GazeThresholds? - var calibrationDate: Date - var isComplete: Bool - private let thresholdCalculator = CalibrationThresholdCalculator() - - enum CodingKeys: String, CodingKey { - case samples - case computedThresholds - case calibrationDate - case isComplete - } - - init() { - self.samples = [:] - self.computedThresholds = nil - self.calibrationDate = Date() - self.isComplete = false - } - - mutating func addSample(_ sample: GazeSample, for step: CalibrationStep) { - if samples[step] == nil { - samples[step] = [] - } - samples[step]?.append(sample) - } - - func getSamples(for step: CalibrationStep) -> [GazeSample] { - return samples[step] ?? [] - } - - func averageRatio(for step: CalibrationStep) -> Double? { - let stepSamples = getSamples(for: step) - guard !stepSamples.isEmpty else { return nil } - return stepSamples.reduce(0.0) { $0 + $1.averageRatio } / Double(stepSamples.count) - } - - func averageVerticalRatio(for step: CalibrationStep) -> Double? { - let stepSamples = getSamples(for: step) - guard !stepSamples.isEmpty else { return nil } - return stepSamples.reduce(0.0) { $0 + $1.averageVerticalRatio } / Double(stepSamples.count) - } - - func averageFaceWidth(for step: CalibrationStep) -> Double? { - let stepSamples = getSamples(for: step) - let validSamples = stepSamples.compactMap { $0.faceWidthRatio } - guard !validSamples.isEmpty else { return nil } - return validSamples.reduce(0.0, +) / Double(validSamples.count) - } - - mutating func calculateThresholds() { - self.computedThresholds = thresholdCalculator.calculate(using: self) - logStepData() - } - - private func logStepData() { - print(" Per-step data:") - for step in CalibrationStep.allCases { - if let h = averageRatio(for: step) { - let v = averageVerticalRatio(for: step) ?? -1 - let fw = averageFaceWidth(for: step) ?? -1 - let count = getSamples(for: step).count - print( - " \(step.rawValue): H=\(String(format: "%.3f", h)), V=\(String(format: "%.3f", v)), FW=\(String(format: "%.3f", fw)), samples=\(count)" - ) - } - } - } -} - -/// Thread-safe storage for active calibration thresholds -/// Allows non-isolated code (video processing) to read thresholds without hitting MainActor -class CalibrationState: @unchecked Sendable { - static let shared = CalibrationState() - private let queue = DispatchQueue(label: "com.gaze.calibrationState", attributes: .concurrent) - private var _thresholds: GazeThresholds? - private var _isComplete: Bool = false - - var thresholds: GazeThresholds? { - get { queue.sync { _thresholds } } - set { queue.async(flags: .barrier) { self._thresholds = newValue } } - } - - var isComplete: Bool { - get { queue.sync { _isComplete } } - set { queue.async(flags: .barrier) { self._isComplete = newValue } } - } - - func reset() { - setState(thresholds: nil, isComplete: false) - } - - func setThresholds(_ thresholds: GazeThresholds?) { - setState(thresholds: thresholds, isComplete: nil) - } - - func setComplete(_ isComplete: Bool) { - setState(thresholds: nil, isComplete: isComplete) - } - - private func setState(thresholds: GazeThresholds?, isComplete: Bool?) { - queue.async(flags: .barrier) { - if let thresholds { - self._thresholds = thresholds - } else if isComplete == nil { - self._thresholds = nil - } - if let isComplete { - self._isComplete = isComplete - } - } - } -} diff --git a/Gaze/Models/CalibrationThresholdCalculator.swift b/Gaze/Models/CalibrationThresholdCalculator.swift deleted file mode 100644 index 7a1034e..0000000 --- a/Gaze/Models/CalibrationThresholdCalculator.swift +++ /dev/null @@ -1,158 +0,0 @@ -// -// CalibrationThresholdCalculator.swift -// Gaze -// -// Created by Mike Freno on 1/29/26. -// - -import Foundation - -struct CalibrationThresholdCalculator { - func calculate(using data: CalibrationData) -> GazeThresholds? { - let centerH = data.averageRatio(for: .center) - let centerV = data.averageVerticalRatio(for: .center) - - guard let cH = centerH else { - print("⚠️ No center calibration data, using defaults") - return GazeThresholds.defaultThresholds - } - - let cV = centerV ?? 0.45 - - print("📊 Calibration data collected:") - print(" Center H: \(String(format: "%.3f", cH)), V: \(String(format: "%.3f", cV))") - - let screenLeftH = data.averageRatio(for: .left) - ?? data.averageRatio(for: .topLeft) - ?? data.averageRatio(for: .bottomLeft) - let screenRightH = data.averageRatio(for: .right) - ?? data.averageRatio(for: .topRight) - ?? data.averageRatio(for: .bottomRight) - - let farLeftH = data.averageRatio(for: .farLeft) - let farRightH = data.averageRatio(for: .farRight) - - let (leftBound, lookLeftThreshold) = horizontalBounds( - center: cH, - screenEdge: screenLeftH, - farEdge: farLeftH, - direction: .left - ) - let (rightBound, lookRightThreshold) = horizontalBounds( - center: cH, - screenEdge: screenRightH, - farEdge: farRightH, - direction: .right - ) - - let screenTopV = data.averageVerticalRatio(for: .up) - ?? data.averageVerticalRatio(for: .topLeft) - ?? data.averageVerticalRatio(for: .topRight) - let screenBottomV = data.averageVerticalRatio(for: .down) - ?? data.averageVerticalRatio(for: .bottomLeft) - ?? data.averageVerticalRatio(for: .bottomRight) - - let (topBound, lookUpThreshold) = verticalBounds(center: cV, screenEdge: screenTopV, isUpperEdge: true) - let (bottomBound, lookDownThreshold) = verticalBounds( - center: cV, - screenEdge: screenBottomV, - isUpperEdge: false - ) - - let allFaceWidths = CalibrationStep.allCases.compactMap { data.averageFaceWidth(for: $0) } - let refFaceWidth = allFaceWidths.isEmpty ? 0.0 : allFaceWidths.average() - - let thresholds = GazeThresholds( - minLeftRatio: lookLeftThreshold, - maxRightRatio: lookRightThreshold, - minUpRatio: lookUpThreshold, - maxDownRatio: lookDownThreshold, - screenLeftBound: leftBound, - screenRightBound: rightBound, - screenTopBound: topBound, - screenBottomBound: bottomBound, - referenceFaceWidth: refFaceWidth - ) - - logThresholds( - thresholds: thresholds, - centerHorizontal: cH, - centerVertical: cV - ) - - return thresholds - } - - private enum HorizontalDirection { - case left - case right - } - - private func horizontalBounds( - center: Double, - screenEdge: Double?, - farEdge: Double?, - direction: HorizontalDirection - ) -> (bound: Double, threshold: Double) { - let defaultBoundOffset = direction == .left ? 0.15 : -0.15 - let defaultThresholdOffset = direction == .left ? 0.20 : -0.20 - - guard let screenEdge = screenEdge else { - return (center + defaultBoundOffset, center + defaultThresholdOffset) - } - - let bound = screenEdge - let threshold: Double - if let farEdge = farEdge { - threshold = (screenEdge + farEdge) / 2.0 - } else { - threshold = screenEdge + defaultThresholdOffset - } - - return (bound, threshold) - } - - private func verticalBounds(center: Double, screenEdge: Double?, isUpperEdge: Bool) -> (bound: Double, threshold: Double) { - let defaultBoundOffset = isUpperEdge ? -0.10 : 0.10 - let defaultThresholdOffset = isUpperEdge ? -0.15 : 0.15 - - guard let screenEdge = screenEdge else { - return (center + defaultBoundOffset, center + defaultThresholdOffset) - } - - let bound = screenEdge - let edgeDistance = isUpperEdge ? center - screenEdge : screenEdge - center - let threshold = isUpperEdge ? screenEdge - (edgeDistance * 0.5) : screenEdge + (edgeDistance * 0.5) - - return (bound, threshold) - } - - private func logThresholds( - thresholds: GazeThresholds, - centerHorizontal: Double, - centerVertical: Double - ) { - print("✓ Calibration thresholds calculated:") - print(" Center: H=\(String(format: "%.3f", centerHorizontal)), V=\(String(format: "%.3f", centerVertical))") - print( - " Screen H-Range: \(String(format: "%.3f", thresholds.screenRightBound)) to \(String(format: "%.3f", thresholds.screenLeftBound))" - ) - print( - " Screen V-Range: \(String(format: "%.3f", thresholds.screenTopBound)) to \(String(format: "%.3f", thresholds.screenBottomBound))" - ) - print( - " Away Thresholds: L≥\(String(format: "%.3f", thresholds.minLeftRatio)), R≤\(String(format: "%.3f", thresholds.maxRightRatio))" - ) - print( - " Away Thresholds: U≤\(String(format: "%.3f", thresholds.minUpRatio)), D≥\(String(format: "%.3f", thresholds.maxDownRatio))" - ) - print(" Ref Face Width: \(String(format: "%.3f", thresholds.referenceFaceWidth))") - } -} - -private extension Array where Element == Double { - func average() -> Double { - guard !isEmpty else { return 0 } - return reduce(0.0, +) / Double(count) - } -} diff --git a/Gaze/Services/Calibration/CalibrationFlowController.swift b/Gaze/Services/Calibration/CalibrationFlowController.swift deleted file mode 100644 index b66cd00..0000000 --- a/Gaze/Services/Calibration/CalibrationFlowController.swift +++ /dev/null @@ -1,83 +0,0 @@ -// -// CalibrationFlowController.swift -// Gaze -// -// Created by Mike Freno on 1/29/26. -// - -import Combine -import Foundation - -final class CalibrationFlowController: ObservableObject { - @Published private(set) var currentStep: CalibrationStep? - @Published private(set) var currentStepIndex = 0 - @Published private(set) var isCollectingSamples = false - @Published private(set) var samplesCollected = 0 - - private let samplesPerStep: Int - private let calibrationSteps: [CalibrationStep] - - init(samplesPerStep: Int, calibrationSteps: [CalibrationStep]) { - self.samplesPerStep = samplesPerStep - self.calibrationSteps = calibrationSteps - self.currentStep = calibrationSteps.first - } - - func start() { - isCollectingSamples = false - currentStepIndex = 0 - currentStep = calibrationSteps.first - samplesCollected = 0 - } - - func stop() { - isCollectingSamples = false - currentStep = nil - currentStepIndex = 0 - samplesCollected = 0 - } - - func startCollectingSamples() { - guard currentStep != nil else { return } - isCollectingSamples = true - } - - func resetSamples() { - isCollectingSamples = false - samplesCollected = 0 - } - - func markSampleCollected() -> Bool { - samplesCollected += 1 - return samplesCollected >= samplesPerStep - } - - func advanceToNextStep() -> Bool { - isCollectingSamples = false - currentStepIndex += 1 - - guard currentStepIndex < calibrationSteps.count else { - currentStep = nil - return false - } - - currentStep = calibrationSteps[currentStepIndex] - samplesCollected = 0 - return true - } - - func skipStep() -> Bool { - advanceToNextStep() - } - - var progress: Double { - let totalSteps = calibrationSteps.count - guard totalSteps > 0 else { return 0 } - let currentProgress = Double(samplesCollected) / Double(samplesPerStep) - return (Double(currentStepIndex) + currentProgress) / Double(totalSteps) - } - - var progressText: String { - "\(min(currentStepIndex + 1, calibrationSteps.count)) of \(calibrationSteps.count)" - } -} diff --git a/Gaze/Services/Calibration/CalibratorService.swift b/Gaze/Services/Calibration/CalibratorService.swift deleted file mode 100644 index c3f6612..0000000 --- a/Gaze/Services/Calibration/CalibratorService.swift +++ /dev/null @@ -1,292 +0,0 @@ -// -// CalibratorService.swift -// Gaze -// -// Created by Mike Freno on 1/29/26. -// - -import Combine -import Foundation -import AppKit -import SwiftUI - -final class CalibratorService: ObservableObject { - static let shared = CalibratorService() - - @Published var isCalibrating = false - @Published var isCollectingSamples = false - @Published var currentStep: CalibrationStep? - @Published var currentStepIndex = 0 - @Published var samplesCollected = 0 - @Published var calibrationData = CalibrationData() - - private let samplesPerStep = 30 - private let userDefaultsKey = "eyeTrackingCalibration" - private let flowController: CalibrationFlowController - private var windowController: NSWindowController? - - private init() { - self.flowController = CalibrationFlowController( - samplesPerStep: samplesPerStep, - calibrationSteps: [ - .center, - .left, - .right, - .farLeft, - .farRight, - .up, - .down, - .topLeft, - .topRight - ] - ) - loadCalibration() - bindFlowController() - } - - private func bindFlowController() { - flowController.$isCollectingSamples - .assign(to: &$isCollectingSamples) - flowController.$currentStep - .assign(to: &$currentStep) - flowController.$currentStepIndex - .assign(to: &$currentStepIndex) - flowController.$samplesCollected - .assign(to: &$samplesCollected) - } - - func startCalibration() { - print("🎯 Starting calibration...") - isCalibrating = true - flowController.start() - calibrationData = CalibrationData() - } - - func resetForNewCalibration() { - print("🔄 Resetting for new calibration...") - calibrationData = CalibrationData() - flowController.start() - } - - func startCollectingSamples() { - guard isCalibrating else { return } - print("📊 Started collecting samples for step: \(currentStep?.displayName ?? "unknown")") - flowController.startCollectingSamples() - } - - func collectSample( - leftRatio: Double?, - rightRatio: Double?, - leftVertical: Double? = nil, - rightVertical: Double? = nil, - faceWidthRatio: Double? = nil - ) { - guard isCalibrating, isCollectingSamples, let step = currentStep else { return } - - let sample = GazeSample( - leftRatio: leftRatio, - rightRatio: rightRatio, - leftVerticalRatio: leftVertical, - rightVerticalRatio: rightVertical, - faceWidthRatio: faceWidthRatio - ) - calibrationData.addSample(sample, for: step) - - if flowController.markSampleCollected() { - advanceToNextStep() - } - } - - private func advanceToNextStep() { - if flowController.advanceToNextStep() { - print("📍 Calibration step: \(currentStep?.displayName ?? "unknown")") - } else { - finishCalibration() - } - } - - func skipStep() { - guard isCalibrating, let step = currentStep else { return } - - print("⏭️ Skipping calibration step: \(step.displayName)") - advanceToNextStep() - } - - func showCalibrationOverlay() { - guard let screen = NSScreen.main else { return } - - let window = KeyableWindow( - contentRect: screen.frame, - styleMask: [.borderless, .fullSizeContentView], - backing: .buffered, - defer: false - ) - - window.level = .screenSaver - window.isOpaque = true - window.backgroundColor = .black - window.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary] - window.acceptsMouseMovedEvents = true - window.ignoresMouseEvents = false - - let overlayView = CalibrationOverlayView { - self.dismissCalibrationOverlay() - } - window.contentView = NSHostingView(rootView: overlayView) - - windowController = NSWindowController(window: window) - windowController?.showWindow(nil) - window.makeKeyAndOrderFront(nil) - NSApp.activate(ignoringOtherApps: true) - - print("🎯 Calibration overlay window opened") - } - - func dismissCalibrationOverlay() { - windowController?.close() - windowController = nil - print("🎯 Calibration overlay window closed") - } - - func finishCalibration() { - print("✓ Calibration complete, calculating thresholds...") - - calibrationData.calculateThresholds() - calibrationData.isComplete = true - calibrationData.calibrationDate = Date() - - saveCalibration() - applyCalibration() - - isCalibrating = false - flowController.stop() - - print("✓ Calibration saved and applied") - } - - func cancelCalibration() { - print("❌ Calibration cancelled") - isCalibrating = false - flowController.stop() - calibrationData = CalibrationData() - - CalibrationState.shared.reset() - } - - private func saveCalibration() { - do { - let encoder = JSONEncoder() - encoder.dateEncodingStrategy = .iso8601 - let data = try encoder.encode(calibrationData) - UserDefaults.standard.set(data, forKey: userDefaultsKey) - print("💾 Calibration data saved to UserDefaults") - } catch { - print("❌ Failed to save calibration: \(error)") - } - } - - func loadCalibration() { - guard let data = UserDefaults.standard.data(forKey: userDefaultsKey) else { - print("ℹ️ No existing calibration found") - return - } - - do { - let decoder = JSONDecoder() - decoder.dateDecodingStrategy = .iso8601 - calibrationData = try decoder.decode(CalibrationData.self, from: data) - - if isCalibrationValid() { - print("✓ Loaded valid calibration from \(calibrationData.calibrationDate)") - applyCalibration() - } else { - print("⚠️ Calibration expired, needs recalibration") - } - } catch { - print("❌ Failed to load calibration: \(error)") - } - } - - func clearCalibration() { - UserDefaults.standard.removeObject(forKey: userDefaultsKey) - calibrationData = CalibrationData() - CalibrationState.shared.reset() - print("🗑️ Calibration data cleared") - } - - func isCalibrationValid() -> Bool { - guard calibrationData.isComplete, - let thresholds = calibrationData.computedThresholds, - thresholds.isValid else { - return false - } - return true - } - - func needsRecalibration() -> Bool { - return !isCalibrationValid() - } - - private func applyCalibration() { - guard let thresholds = calibrationData.computedThresholds else { - print("⚠️ No thresholds to apply") - return - } - - CalibrationState.shared.setThresholds(thresholds) - CalibrationState.shared.setComplete(true) - - print("✓ Applied calibrated thresholds:") - print(" Looking left: ≥\(String(format: "%.3f", thresholds.minLeftRatio))") - print(" Looking right: ≤\(String(format: "%.3f", thresholds.maxRightRatio))") - print(" Looking up: ≤\(String(format: "%.3f", thresholds.minUpRatio))") - print(" Looking down: ≥\(String(format: "%.3f", thresholds.maxDownRatio))") - print(" Screen Bounds: [\(String(format: "%.2f", thresholds.screenRightBound))..\(String(format: "%.2f", thresholds.screenLeftBound))] x [\(String(format: "%.2f", thresholds.screenTopBound))..\(String(format: "%.2f", thresholds.screenBottomBound))]") - } - - func getCalibrationSummary() -> String { - guard calibrationData.isComplete else { - return "No calibration data" - } - - let dateFormatter = DateFormatter() - dateFormatter.dateStyle = .medium - dateFormatter.timeStyle = .short - - var summary = "Calibrated: \(dateFormatter.string(from: calibrationData.calibrationDate))\n" - - if let thresholds = calibrationData.computedThresholds { - summary += "H-Range: \(String(format: "%.3f", thresholds.screenRightBound)) to \(String(format: "%.3f", thresholds.screenLeftBound))\n" - summary += "V-Range: \(String(format: "%.3f", thresholds.screenTopBound)) to \(String(format: "%.3f", thresholds.screenBottomBound))\n" - summary += "Ref Face Width: \(String(format: "%.3f", thresholds.referenceFaceWidth))" - } - - return summary - } - - var progress: Double { - flowController.progress - } - - var progressText: String { - flowController.progressText - } - - func submitSampleToBridge( - leftRatio: Double, - rightRatio: Double, - leftVertical: Double? = nil, - rightVertical: Double? = nil, - faceWidthRatio: Double = 0 - ) { - Task { [weak self] in - self?.collectSample( - leftRatio: leftRatio, - rightRatio: rightRatio, - leftVertical: leftVertical, - rightVertical: rightVertical, - faceWidthRatio: faceWidthRatio - ) - } - } -} diff --git a/Gaze/Services/EnforceModeService.swift b/Gaze/Services/EnforceModeService.swift index 0964d19..73e79e4 100644 --- a/Gaze/Services/EnforceModeService.swift +++ b/Gaze/Services/EnforceModeService.swift @@ -60,14 +60,15 @@ class EnforceModeService: ObservableObject { } private func setupEyeTrackingObservers() { - eyeTrackingService.$userLookingAtScreen + eyeTrackingService.$trackingResult .sink { [weak self] _ in guard let self, self.isCameraActive else { return } self.checkUserCompliance() } .store(in: &cancellables) - eyeTrackingService.$faceDetected + eyeTrackingService.$trackingResult + .map { $0.faceDetected } .sink { [weak self] faceDetected in guard let self else { return } if faceDetected { @@ -166,11 +167,18 @@ class EnforceModeService: ObservableObject { } func evaluateCompliance( - isLookingAtScreen: Bool, + gazeState: GazeState, faceDetected: Bool ) -> ComplianceResult { guard faceDetected else { return .faceNotDetected } - return isLookingAtScreen ? .notCompliant : .compliant + switch gazeState { + case .lookingAway: + return .compliant + case .lookingAtScreen: + return .notCompliant + case .unknown: + return .notCompliant + } } // MARK: - Camera Control @@ -214,8 +222,8 @@ class EnforceModeService: ObservableObject { return } let compliance = evaluateCompliance( - isLookingAtScreen: eyeTrackingService.userLookingAtScreen, - faceDetected: eyeTrackingService.faceDetected + gazeState: eyeTrackingService.trackingResult.gazeState, + faceDetected: eyeTrackingService.trackingResult.faceDetected ) switch compliance { diff --git a/Gaze/Services/EyeTracking/EyeTrackingService.swift b/Gaze/Services/EyeTracking/EyeTrackingService.swift index 0ebd66c..1583f3c 100644 --- a/Gaze/Services/EyeTracking/EyeTrackingService.swift +++ b/Gaze/Services/EyeTracking/EyeTrackingService.swift @@ -5,7 +5,6 @@ // Created by Mike Freno on 1/13/26. // -import AppKit import AVFoundation import Combine import Foundation @@ -14,79 +13,24 @@ class EyeTrackingService: NSObject, ObservableObject { static let shared = EyeTrackingService() @Published var isEyeTrackingActive = false - @Published var isEyesClosed = false - @Published var userLookingAtScreen = true - @Published var faceDetected = false - @Published var debugLeftPupilRatio: Double? - @Published var debugRightPupilRatio: Double? - @Published var debugLeftVerticalRatio: Double? - @Published var debugRightVerticalRatio: Double? - @Published var debugYaw: Double? - @Published var debugPitch: Double? - @Published var enableDebugLogging: Bool = false { - didSet { - debugAdapter.enableDebugLogging = enableDebugLogging - } - } - @Published var debugLeftEyeInput: NSImage? - @Published var debugRightEyeInput: NSImage? - @Published var debugLeftEyeProcessed: NSImage? - @Published var debugRightEyeProcessed: NSImage? - @Published var debugLeftPupilPosition: PupilPosition? - @Published var debugRightPupilPosition: PupilPosition? - @Published var debugLeftEyeSize: CGSize? - @Published var debugRightEyeSize: CGSize? - @Published var debugLeftEyeRegion: EyeRegion? - @Published var debugRightEyeRegion: EyeRegion? - @Published var debugImageSize: CGSize? + @Published var trackingResult = TrackingResult.empty + @Published var debugState = EyeTrackingDebugState.empty private let cameraManager = CameraSessionManager() private let visionPipeline = VisionPipeline() - private let debugAdapter = EyeDebugStateAdapter() - private let gazeDetector: GazeDetector + private let processor: VisionGazeProcessor var previewLayer: AVCaptureVideoPreviewLayer? { cameraManager.previewLayer } - var gazeDirection: GazeDirection { - guard let leftH = debugLeftPupilRatio, - let rightH = debugRightPupilRatio, - let leftV = debugLeftVerticalRatio, - let rightV = debugRightVerticalRatio else { - return .center - } - - let avgHorizontal = (leftH + rightH) / 2.0 - let avgVertical = (leftV + rightV) / 2.0 - - return GazeDirection.from(horizontal: avgHorizontal, vertical: avgVertical) - } - var isInFrame: Bool { - faceDetected + trackingResult.faceDetected } private override init() { - let configuration = GazeDetector.Configuration( - thresholds: CalibrationState.shared.thresholds, - isCalibrationComplete: CalibrationState.shared.isComplete, - eyeClosedEnabled: EyeTrackingConstants.eyeClosedEnabled, - eyeClosedThreshold: EyeTrackingConstants.eyeClosedThreshold, - yawEnabled: EyeTrackingConstants.yawEnabled, - yawThreshold: EyeTrackingConstants.yawThreshold, - pitchUpEnabled: EyeTrackingConstants.pitchUpEnabled, - pitchUpThreshold: EyeTrackingConstants.pitchUpThreshold, - pitchDownEnabled: EyeTrackingConstants.pitchDownEnabled, - pitchDownThreshold: EyeTrackingConstants.pitchDownThreshold, - pixelGazeEnabled: EyeTrackingConstants.pixelGazeEnabled, - pixelGazeMinRatio: EyeTrackingConstants.pixelGazeMinRatio, - pixelGazeMaxRatio: EyeTrackingConstants.pixelGazeMaxRatio, - boundaryForgivenessMargin: EyeTrackingConstants.boundaryForgivenessMargin, - distanceSensitivity: EyeTrackingConstants.distanceSensitivity, - defaultReferenceFaceWidth: EyeTrackingConstants.defaultReferenceFaceWidth - ) - self.gazeDetector = GazeDetector(configuration: configuration) + let config = TrackingConfig.default + self.processor = VisionGazeProcessor(config: config) super.init() cameraManager.delegate = self } @@ -109,55 +53,10 @@ class EyeTrackingService: NSObject, ObservableObject { cameraManager.stop() Task { @MainActor in isEyeTrackingActive = false - isEyesClosed = false - userLookingAtScreen = true - faceDetected = false - debugAdapter.clear() - syncDebugState() + trackingResult = TrackingResult.empty + debugState = EyeTrackingDebugState.empty } } - - private func syncDebugState() { - debugLeftPupilRatio = debugAdapter.leftPupilRatio - debugRightPupilRatio = debugAdapter.rightPupilRatio - debugLeftVerticalRatio = debugAdapter.leftVerticalRatio - debugRightVerticalRatio = debugAdapter.rightVerticalRatio - debugYaw = debugAdapter.yaw - debugPitch = debugAdapter.pitch - debugLeftEyeInput = debugAdapter.leftEyeInput - debugRightEyeInput = debugAdapter.rightEyeInput - debugLeftEyeProcessed = debugAdapter.leftEyeProcessed - debugRightEyeProcessed = debugAdapter.rightEyeProcessed - debugLeftPupilPosition = debugAdapter.leftPupilPosition - debugRightPupilPosition = debugAdapter.rightPupilPosition - debugLeftEyeSize = debugAdapter.leftEyeSize - debugRightEyeSize = debugAdapter.rightEyeSize - debugLeftEyeRegion = debugAdapter.leftEyeRegion - debugRightEyeRegion = debugAdapter.rightEyeRegion - debugImageSize = debugAdapter.imageSize - } - - private func updateGazeConfiguration() { - let configuration = GazeDetector.Configuration( - thresholds: CalibrationState.shared.thresholds, - isCalibrationComplete: CalibratorService.shared.isCalibrating || CalibrationState.shared.isComplete, - eyeClosedEnabled: EyeTrackingConstants.eyeClosedEnabled, - eyeClosedThreshold: EyeTrackingConstants.eyeClosedThreshold, - yawEnabled: EyeTrackingConstants.yawEnabled, - yawThreshold: EyeTrackingConstants.yawThreshold, - pitchUpEnabled: EyeTrackingConstants.pitchUpEnabled, - pitchUpThreshold: EyeTrackingConstants.pitchUpThreshold, - pitchDownEnabled: EyeTrackingConstants.pitchDownEnabled, - pitchDownThreshold: EyeTrackingConstants.pitchDownThreshold, - pixelGazeEnabled: EyeTrackingConstants.pixelGazeEnabled, - pixelGazeMinRatio: EyeTrackingConstants.pixelGazeMinRatio, - pixelGazeMaxRatio: EyeTrackingConstants.pixelGazeMaxRatio, - boundaryForgivenessMargin: EyeTrackingConstants.boundaryForgivenessMargin, - distanceSensitivity: EyeTrackingConstants.distanceSensitivity, - defaultReferenceFaceWidth: EyeTrackingConstants.defaultReferenceFaceWidth - ) - gazeDetector.updateConfiguration(configuration) - } } extension EyeTrackingService: CameraSessionDelegate { @@ -166,31 +65,17 @@ extension EyeTrackingService: CameraSessionDelegate { didOutput pixelBuffer: CVPixelBuffer, imageSize: CGSize ) { - PupilDetector.advanceFrame() - let analysis = visionPipeline.analyze(pixelBuffer: pixelBuffer, imageSize: imageSize) - let result = gazeDetector.process(analysis: analysis, pixelBuffer: pixelBuffer) + let observation = processor.process(analysis: analysis) - if let leftRatio = result.leftPupilRatio, - let rightRatio = result.rightPupilRatio, - let faceWidth = result.faceWidthRatio { - guard CalibratorService.shared.isCalibrating else { return } - CalibratorService.shared.submitSampleToBridge( - leftRatio: leftRatio, - rightRatio: rightRatio, - leftVertical: result.leftVerticalRatio, - rightVertical: result.rightVerticalRatio, - faceWidthRatio: faceWidth - ) - } - - self.faceDetected = result.faceDetected - self.isEyesClosed = result.isEyesClosed - self.userLookingAtScreen = result.userLookingAtScreen - self.debugAdapter.update(from: result) - self.debugAdapter.updateEyeImages(from: PupilDetector.self) - self.syncDebugState() - self.updateGazeConfiguration() + trackingResult = TrackingResult( + faceDetected: observation.faceDetected, + gazeState: observation.gazeState, + eyesClosed: observation.eyesClosed, + confidence: observation.confidence, + timestamp: Date() + ) + debugState = observation.debugState } } @@ -215,96 +100,3 @@ enum EyeTrackingError: Error, LocalizedError { } } } - -// MARK: - Debug State Adapter - -final class EyeDebugStateAdapter { - var leftPupilRatio: Double? - var rightPupilRatio: Double? - var leftVerticalRatio: Double? - var rightVerticalRatio: Double? - var yaw: Double? - var pitch: Double? - var enableDebugLogging: Bool = false { - didSet { - PupilDetector.enableDiagnosticLogging = enableDebugLogging - } - } - - var leftEyeInput: NSImage? - var rightEyeInput: NSImage? - var leftEyeProcessed: NSImage? - var rightEyeProcessed: NSImage? - var leftPupilPosition: PupilPosition? - var rightPupilPosition: PupilPosition? - var leftEyeSize: CGSize? - var rightEyeSize: CGSize? - var leftEyeRegion: EyeRegion? - var rightEyeRegion: EyeRegion? - var imageSize: CGSize? - - var gazeDirection: GazeDirection { - guard let leftH = leftPupilRatio, - let rightH = rightPupilRatio, - let leftV = leftVerticalRatio, - let rightV = rightVerticalRatio else { - return .center - } - - let avgHorizontal = (leftH + rightH) / 2.0 - let avgVertical = (leftV + rightV) / 2.0 - - return GazeDirection.from(horizontal: avgHorizontal, vertical: avgVertical) - } - - func update(from result: EyeTrackingProcessingResult) { - leftPupilRatio = result.leftPupilRatio - rightPupilRatio = result.rightPupilRatio - leftVerticalRatio = result.leftVerticalRatio - rightVerticalRatio = result.rightVerticalRatio - yaw = result.yaw - pitch = result.pitch - } - - func updateEyeImages(from detector: PupilDetector.Type) { - if let leftInput = detector.debugLeftEyeInput { - leftEyeInput = NSImage(cgImage: leftInput, size: NSSize(width: leftInput.width, height: leftInput.height)) - } - if let rightInput = detector.debugRightEyeInput { - rightEyeInput = NSImage(cgImage: rightInput, size: NSSize(width: rightInput.width, height: rightInput.height)) - } - if let leftProcessed = detector.debugLeftEyeProcessed { - leftEyeProcessed = NSImage(cgImage: leftProcessed, size: NSSize(width: leftProcessed.width, height: leftProcessed.height)) - } - if let rightProcessed = detector.debugRightEyeProcessed { - rightEyeProcessed = NSImage(cgImage: rightProcessed, size: NSSize(width: rightProcessed.width, height: rightProcessed.height)) - } - leftPupilPosition = detector.debugLeftPupilPosition - rightPupilPosition = detector.debugRightPupilPosition - leftEyeSize = detector.debugLeftEyeSize - rightEyeSize = detector.debugRightEyeSize - leftEyeRegion = detector.debugLeftEyeRegion - rightEyeRegion = detector.debugRightEyeRegion - imageSize = detector.debugImageSize - } - - func clear() { - leftPupilRatio = nil - rightPupilRatio = nil - leftVerticalRatio = nil - rightVerticalRatio = nil - yaw = nil - pitch = nil - leftEyeInput = nil - rightEyeInput = nil - leftEyeProcessed = nil - rightEyeProcessed = nil - leftPupilPosition = nil - rightPupilPosition = nil - leftEyeSize = nil - rightEyeSize = nil - leftEyeRegion = nil - rightEyeRegion = nil - imageSize = nil - } -} diff --git a/Gaze/Services/EyeTracking/GazeBaselineModel.swift b/Gaze/Services/EyeTracking/GazeBaselineModel.swift new file mode 100644 index 0000000..21b3c06 --- /dev/null +++ b/Gaze/Services/EyeTracking/GazeBaselineModel.swift @@ -0,0 +1,54 @@ +// +// GazeBaselineModel.swift +// Gaze +// +// Created by Mike Freno on 1/31/26. +// + +import Foundation + +public final class GazeBaselineModel: @unchecked Sendable { + public struct Baseline: Sendable { + let horizontal: Double + let vertical: Double + let sampleCount: Int + } + + private let lock = NSLock() + private var horizontal: Double? + private var vertical: Double? + private var sampleCount: Int = 0 + + public func reset() { + lock.lock() + horizontal = nil + vertical = nil + sampleCount = 0 + lock.unlock() + } + + public func update(horizontal: Double, vertical: Double, smoothing: Double) { + lock.lock() + defer { lock.unlock() } + + if let existingH = self.horizontal, let existingV = self.vertical { + self.horizontal = existingH + (horizontal - existingH) * smoothing + self.vertical = existingV + (vertical - existingV) * smoothing + } else { + self.horizontal = horizontal + self.vertical = vertical + } + sampleCount += 1 + } + + public func current(defaultH: Double, defaultV: Double) -> Baseline { + lock.lock() + defer { lock.unlock() } + + return Baseline( + horizontal: horizontal ?? defaultH, + vertical: vertical ?? defaultV, + sampleCount: sampleCount + ) + } +} diff --git a/Gaze/Services/EyeTracking/GazeDetector.swift b/Gaze/Services/EyeTracking/GazeDetector.swift deleted file mode 100644 index 04bcdfd..0000000 --- a/Gaze/Services/EyeTracking/GazeDetector.swift +++ /dev/null @@ -1,344 +0,0 @@ -// -// GazeDetector.swift -// Gaze -// -// Gaze detection logic and pupil analysis. -// - -import Foundation -@preconcurrency import Vision -import simd - -struct EyeTrackingProcessingResult: Sendable { - let faceDetected: Bool - let isEyesClosed: Bool - let userLookingAtScreen: Bool - let leftPupilRatio: Double? - let rightPupilRatio: Double? - let leftVerticalRatio: Double? - let rightVerticalRatio: Double? - let yaw: Double? - let pitch: Double? - let faceWidthRatio: Double? -} - -final class GazeDetector: @unchecked Sendable { - struct GazeResult: Sendable { - let isLookingAway: Bool - let isEyesClosed: Bool - let leftPupilRatio: Double? - let rightPupilRatio: Double? - let leftVerticalRatio: Double? - let rightVerticalRatio: Double? - let yaw: Double? - let pitch: Double? - } - - struct Configuration: Sendable { - let thresholds: GazeThresholds? - let isCalibrationComplete: Bool - let eyeClosedEnabled: Bool - let eyeClosedThreshold: CGFloat - let yawEnabled: Bool - let yawThreshold: Double - let pitchUpEnabled: Bool - let pitchUpThreshold: Double - let pitchDownEnabled: Bool - let pitchDownThreshold: Double - let pixelGazeEnabled: Bool - let pixelGazeMinRatio: Double - let pixelGazeMaxRatio: Double - let boundaryForgivenessMargin: Double - let distanceSensitivity: Double - let defaultReferenceFaceWidth: Double - } - - private let lock = NSLock() - private nonisolated(unsafe) var configuration: Configuration - - nonisolated init(configuration: Configuration) { - self.configuration = configuration - } - - nonisolated func updateConfiguration(_ configuration: Configuration) { - lock.lock() - self.configuration = configuration - lock.unlock() - } - - func process( - analysis: VisionPipeline.FaceAnalysis, - pixelBuffer: CVPixelBuffer - ) -> EyeTrackingProcessingResult { - let config: Configuration - lock.lock() - config = configuration - lock.unlock() - - guard analysis.faceDetected, let face = analysis.face?.value else { - return EyeTrackingProcessingResult( - faceDetected: false, - isEyesClosed: false, - userLookingAtScreen: false, - leftPupilRatio: nil, - rightPupilRatio: nil, - leftVerticalRatio: nil, - rightVerticalRatio: nil, - yaw: analysis.debugYaw, - pitch: analysis.debugPitch, - faceWidthRatio: nil - ) - } - - let landmarks = face.landmarks - let yaw = face.yaw?.doubleValue ?? 0.0 - let pitch = face.pitch?.doubleValue ?? 0.0 - - var isEyesClosed = false - if let leftEye = landmarks?.leftEye, let rightEye = landmarks?.rightEye { - isEyesClosed = detectEyesClosed(leftEye: leftEye, rightEye: rightEye, configuration: config) - } - - let gazeResult = detectLookingAway( - face: face, - landmarks: landmarks, - imageSize: analysis.imageSize, - pixelBuffer: pixelBuffer, - configuration: config - ) - - let lookingAway = gazeResult.lookingAway - let userLookingAtScreen = !lookingAway - - return EyeTrackingProcessingResult( - faceDetected: true, - isEyesClosed: isEyesClosed, - userLookingAtScreen: userLookingAtScreen, - leftPupilRatio: gazeResult.leftPupilRatio, - rightPupilRatio: gazeResult.rightPupilRatio, - leftVerticalRatio: gazeResult.leftVerticalRatio, - rightVerticalRatio: gazeResult.rightVerticalRatio, - yaw: gazeResult.yaw ?? yaw, - pitch: gazeResult.pitch ?? pitch, - faceWidthRatio: face.boundingBox.width - ) - } - - private func detectEyesClosed( - leftEye: VNFaceLandmarkRegion2D, - rightEye: VNFaceLandmarkRegion2D, - configuration: Configuration - ) -> Bool { - guard configuration.eyeClosedEnabled else { return false } - guard leftEye.pointCount >= 2, rightEye.pointCount >= 2 else { return false } - - let leftEyeHeight = calculateEyeHeight(leftEye) - let rightEyeHeight = calculateEyeHeight(rightEye) - let closedThreshold = configuration.eyeClosedThreshold - - return leftEyeHeight < closedThreshold && rightEyeHeight < closedThreshold - } - - private func calculateEyeHeight(_ eye: VNFaceLandmarkRegion2D) -> CGFloat { - let points = eye.normalizedPoints - guard points.count >= 2 else { return 0 } - - let yValues = points.map { $0.y } - let maxY = yValues.max() ?? 0 - let minY = yValues.min() ?? 0 - - return abs(maxY - minY) - } - - private struct GazeDetectionResult: Sendable { - var lookingAway: Bool = false - var leftPupilRatio: Double? - var rightPupilRatio: Double? - var leftVerticalRatio: Double? - var rightVerticalRatio: Double? - var yaw: Double? - var pitch: Double? - } - - private func detectLookingAway( - face: VNFaceObservation, - landmarks: VNFaceLandmarks2D?, - imageSize: CGSize, - pixelBuffer: CVPixelBuffer, - configuration: Configuration - ) -> GazeDetectionResult { - var result = GazeDetectionResult() - - let yaw = face.yaw?.doubleValue ?? 0.0 - let pitch = face.pitch?.doubleValue ?? 0.0 - result.yaw = yaw - result.pitch = pitch - - var poseLookingAway = false - - if face.pitch != nil { - if configuration.yawEnabled { - let yawThreshold = configuration.yawThreshold - if abs(yaw) > yawThreshold { - poseLookingAway = true - } - } - - if !poseLookingAway { - var pitchLookingAway = false - - if configuration.pitchUpEnabled && pitch > configuration.pitchUpThreshold { - pitchLookingAway = true - } - - if configuration.pitchDownEnabled && pitch < configuration.pitchDownThreshold { - pitchLookingAway = true - } - - poseLookingAway = pitchLookingAway - } - } - - var eyesLookingAway = false - - if let landmarks, - let leftEye = landmarks.leftEye, - let rightEye = landmarks.rightEye, - configuration.pixelGazeEnabled { - var leftGazeRatio: Double? = nil - var rightGazeRatio: Double? = nil - var leftVerticalRatio: Double? = nil - var rightVerticalRatio: Double? = nil - - if let leftResult = PupilDetector.detectPupil( - in: pixelBuffer, - eyeLandmarks: leftEye, - faceBoundingBox: face.boundingBox, - imageSize: imageSize, - side: 0 - ) { - leftGazeRatio = calculateGazeRatio( - pupilPosition: leftResult.pupilPosition, - eyeRegion: leftResult.eyeRegion - ) - leftVerticalRatio = calculateVerticalRatio( - pupilPosition: leftResult.pupilPosition, - eyeRegion: leftResult.eyeRegion - ) - } - - if let rightResult = PupilDetector.detectPupil( - in: pixelBuffer, - eyeLandmarks: rightEye, - faceBoundingBox: face.boundingBox, - imageSize: imageSize, - side: 1 - ) { - rightGazeRatio = calculateGazeRatio( - pupilPosition: rightResult.pupilPosition, - eyeRegion: rightResult.eyeRegion - ) - rightVerticalRatio = calculateVerticalRatio( - pupilPosition: rightResult.pupilPosition, - eyeRegion: rightResult.eyeRegion - ) - } - - result.leftPupilRatio = leftGazeRatio - result.rightPupilRatio = rightGazeRatio - result.leftVerticalRatio = leftVerticalRatio - result.rightVerticalRatio = rightVerticalRatio - - if let leftRatio = leftGazeRatio, - let rightRatio = rightGazeRatio { - let avgH = (leftRatio + rightRatio) / 2.0 - let avgV = (leftVerticalRatio != nil && rightVerticalRatio != nil) - ? (leftVerticalRatio! + rightVerticalRatio!) / 2.0 - : 0.5 - - if configuration.isCalibrationComplete, - let thresholds = configuration.thresholds { - let currentFaceWidth = face.boundingBox.width - let refFaceWidth = thresholds.referenceFaceWidth - - var distanceScale = 1.0 - if refFaceWidth > 0 && currentFaceWidth > 0 { - let rawScale = refFaceWidth / currentFaceWidth - distanceScale = 1.0 + (rawScale - 1.0) * configuration.distanceSensitivity - distanceScale = max(0.5, min(2.0, distanceScale)) - } - - let centerH = (thresholds.screenLeftBound + thresholds.screenRightBound) / 2.0 - let centerV = (thresholds.screenTopBound + thresholds.screenBottomBound) / 2.0 - - let deltaH = (avgH - centerH) * distanceScale - let deltaV = (avgV - centerV) * distanceScale - - let normalizedH = centerH + deltaH - let normalizedV = centerV + deltaV - - let margin = configuration.boundaryForgivenessMargin - let isLookingLeft = normalizedH > (thresholds.screenLeftBound + margin) - let isLookingRight = normalizedH < (thresholds.screenRightBound - margin) - let isLookingUp = normalizedV < (thresholds.screenTopBound - margin) - let isLookingDown = normalizedV > (thresholds.screenBottomBound + margin) - - eyesLookingAway = isLookingLeft || isLookingRight || isLookingUp || isLookingDown - } else { - let currentFaceWidth = face.boundingBox.width - let refFaceWidth = configuration.defaultReferenceFaceWidth - - var distanceScale = 1.0 - if refFaceWidth > 0 && currentFaceWidth > 0 { - let rawScale = refFaceWidth / currentFaceWidth - distanceScale = 1.0 + (rawScale - 1.0) * configuration.distanceSensitivity - distanceScale = max(0.5, min(2.0, distanceScale)) - } - - let centerH = (configuration.pixelGazeMinRatio + configuration.pixelGazeMaxRatio) / 2.0 - let normalizedH = centerH + (avgH - centerH) * distanceScale - - let lookingRight = normalizedH <= configuration.pixelGazeMinRatio - let lookingLeft = normalizedH >= configuration.pixelGazeMaxRatio - eyesLookingAway = lookingRight || lookingLeft - } - } - } - - result.lookingAway = poseLookingAway || eyesLookingAway - return result - } - - private func calculateGazeRatio( - pupilPosition: PupilPosition, - eyeRegion: EyeRegion - ) -> Double { - let pupilX = Double(pupilPosition.x) - let eyeCenterX = Double(eyeRegion.center.x) - let denominator = (eyeCenterX * 2.0 - 10.0) - - guard denominator > 0 else { - let eyeLeft = Double(eyeRegion.frame.minX) - let eyeRight = Double(eyeRegion.frame.maxX) - let eyeWidth = eyeRight - eyeLeft - guard eyeWidth > 0 else { return 0.5 } - return (pupilX - eyeLeft) / eyeWidth - } - - let ratio = pupilX / denominator - return max(0.0, min(1.0, ratio)) - } - - private func calculateVerticalRatio( - pupilPosition: PupilPosition, - eyeRegion: EyeRegion - ) -> Double { - let pupilX = Double(pupilPosition.x) - let eyeWidth = Double(eyeRegion.frame.width) - - guard eyeWidth > 0 else { return 0.5 } - - let ratio = pupilX / eyeWidth - return max(0.0, min(1.0, ratio)) - } -} diff --git a/Gaze/Services/EyeTracking/PupilDetector.swift b/Gaze/Services/EyeTracking/PupilDetector.swift deleted file mode 100644 index 59ca1ff..0000000 --- a/Gaze/Services/EyeTracking/PupilDetector.swift +++ /dev/null @@ -1,1111 +0,0 @@ -// -// PupilDetector.swift -// Gaze -// -// Created by Mike Freno on 1/15/26. -// -// Pixel-based pupil detection translated from Python GazeTracking library -// Original: https://github.com/antoinelame/GazeTracking -// -// Optimized with: -// - Frame skipping (process every Nth frame) -// - vImage/Accelerate for grayscale conversion and erosion -// - Precomputed lookup tables for bilateral filter -// - Efficient contour detection with union-find -// - -import Accelerate -import CoreImage -import ImageIO -import UniformTypeIdentifiers -@preconcurrency import Vision - -struct PupilPosition: Equatable, Sendable { - let x: CGFloat - let y: CGFloat -} - -struct EyeRegion: Sendable { - let frame: CGRect - let center: CGPoint - let origin: CGPoint -} - -/// 9-point gaze direction grid -enum GazeDirection: String, Sendable, CaseIterable { - case upLeft = "↖" - case up = "↑" - case upRight = "↗" - case left = "←" - case center = "●" - case right = "→" - case downLeft = "↙" - case down = "↓" - case downRight = "↘" - - /// Thresholds for direction detection - /// Based on actual video test data: - /// - Looking at screen (center): H ≈ 0.29-0.35 - /// - Looking left (away): H ≈ 0.62-0.70 - /// Horizontal: Lower values = center/right, Higher values = left - /// Vertical: 0.0 = looking up, 1.0 = looking down - private static let horizontalLeftThreshold = 0.50 // Above this = looking left (away) - private static let horizontalRightThreshold = 0.20 // Below this = looking right - private static let verticalUpThreshold = 0.35 // Below this = looking up - private static let verticalDownThreshold = 0.55 // Above this = looking down - - static func from(horizontal: Double, vertical: Double) -> GazeDirection { - let isLeft = horizontal > horizontalLeftThreshold - let isRight = horizontal < horizontalRightThreshold - let isUp = vertical < verticalUpThreshold - let isDown = vertical > verticalDownThreshold - - if isUp { - if isLeft { return .upLeft } - if isRight { return .upRight } - return .up - } else if isDown { - if isLeft { return .downLeft } - if isRight { return .downRight } - return .down - } else { - if isLeft { return .left } - if isRight { return .right } - return .center - } - } - - /// Grid position (0-2 for x and y) - var gridPosition: (x: Int, y: Int) { - switch self { - case .upLeft: return (0, 0) - case .up: return (1, 0) - case .upRight: return (2, 0) - case .left: return (0, 1) - case .center: return (1, 1) - case .right: return (2, 1) - case .downLeft: return (0, 2) - case .down: return (1, 2) - case .downRight: return (2, 2) - } - } -} - -/// Calibration state for adaptive thresholding (matches Python Calibration class) -final class PupilCalibration: @unchecked Sendable { - private nonisolated let lock = NSLock() - private nonisolated let targetFrames = 20 - private nonisolated(unsafe) var thresholdsLeft: [Int] = [] - private nonisolated(unsafe) var thresholdsRight: [Int] = [] - - nonisolated func isComplete() -> Bool { - lock.lock() - defer { lock.unlock() } - return thresholdsLeft.count >= targetFrames && thresholdsRight.count >= targetFrames - } - - nonisolated func threshold(forSide side: Int) -> Int { - lock.lock() - defer { lock.unlock() } - let thresholds = side == 0 ? thresholdsLeft : thresholdsRight - // DEBUG: Use higher default threshold (was 50) - guard !thresholds.isEmpty else { return 90 } - return thresholds.reduce(0, +) / thresholds.count - } - - nonisolated func evaluate(eyeData: UnsafePointer, width: Int, height: Int, side: Int) { - let bestThreshold = findBestThreshold(eyeData: eyeData, width: width, height: height) - lock.lock() - defer { lock.unlock() } - if side == 0 { - thresholdsLeft.append(bestThreshold) - } else { - thresholdsRight.append(bestThreshold) - } - } - - private nonisolated func findBestThreshold( - eyeData: UnsafePointer, width: Int, height: Int - ) -> Int { - let averageIrisSize = 0.48 - var bestThreshold = 50 - var bestDiff = Double.greatestFiniteMagnitude - - let bufferSize = width * height - let tempBuffer = UnsafeMutablePointer.allocate(capacity: bufferSize) - defer { tempBuffer.deallocate() } - - for threshold in stride(from: 5, to: 100, by: 5) { - PupilDetector.imageProcessingOptimized( - input: eyeData, - output: tempBuffer, - width: width, - height: height, - threshold: threshold - ) - let irisSize = Self.irisSize(data: tempBuffer, width: width, height: height) - let diff = abs(irisSize - averageIrisSize) - if diff < bestDiff { - bestDiff = diff - bestThreshold = threshold - } - } - - return bestThreshold - } - - private nonisolated static func irisSize(data: UnsafePointer, width: Int, height: Int) - -> Double - { - let margin = 5 - guard width > margin * 2, height > margin * 2 else { return 0 } - - var blackCount = 0 - let innerWidth = width - margin * 2 - let innerHeight = height - margin * 2 - - for y in margin..<(height - margin) { - let rowStart = y * width + margin - for x in 0.. 0 ? Double(blackCount) / Double(totalCount) : 0 - } - - nonisolated func reset() { - lock.lock() - defer { lock.unlock() } - thresholdsLeft.removeAll() - thresholdsRight.removeAll() - } -} - -/// Performance metrics for pupil detection -struct PupilDetectorMetrics: Sendable { - var lastProcessingTimeMs: Double = 0 - var averageProcessingTimeMs: Double = 0 - var frameCount: Int = 0 - var processedFrameCount: Int = 0 - - nonisolated mutating func recordProcessingTime(_ ms: Double) { - lastProcessingTimeMs = ms - processedFrameCount += 1 - let alpha = 0.1 - averageProcessingTimeMs = averageProcessingTimeMs * (1 - alpha) + ms * alpha - } -} - -final class PupilDetector: @unchecked Sendable { - - // MARK: - Thread Safety - - private static let lock = NSLock() - - // MARK: - Configuration - - nonisolated(unsafe) static var enableDebugImageSaving: Bool = false // Disabled - causes sandbox errors - nonisolated(unsafe) static var enablePerformanceLogging = false - nonisolated(unsafe) static var enableDiagnosticLogging = false // Disabled - pupil detection now working - nonisolated(unsafe) static var enableDebugLogging: Bool { - #if DEBUG - return true - #else - return false - #endif - } - nonisolated(unsafe) static var frameSkipCount = 10 // Process every Nth frame - - // MARK: - State (protected by lock) - - private nonisolated(unsafe) static var _debugImageCounter = 0 - private nonisolated(unsafe) static var _frameCounter = 0 - private nonisolated(unsafe) static var _lastPupilPositions: - (left: PupilPosition?, right: PupilPosition?) = ( - nil, nil - ) - private nonisolated(unsafe) static var _metrics = PupilDetectorMetrics() - - // Debug images for UI display - nonisolated(unsafe) static var debugLeftEyeInput: CGImage? - nonisolated(unsafe) static var debugRightEyeInput: CGImage? - nonisolated(unsafe) static var debugLeftEyeProcessed: CGImage? - nonisolated(unsafe) static var debugRightEyeProcessed: CGImage? - nonisolated(unsafe) static var debugLeftPupilPosition: PupilPosition? - nonisolated(unsafe) static var debugRightPupilPosition: PupilPosition? - nonisolated(unsafe) static var debugLeftEyeSize: CGSize? - nonisolated(unsafe) static var debugRightEyeSize: CGSize? - - // Eye region positions in image coordinates (for drawing on video) - nonisolated(unsafe) static var debugLeftEyeRegion: EyeRegion? - nonisolated(unsafe) static var debugRightEyeRegion: EyeRegion? - nonisolated(unsafe) static var debugImageSize: CGSize? - - nonisolated static let calibration = PupilCalibration() - - // MARK: - Convenience Properties - - private nonisolated(unsafe) static var debugImageCounter: Int { - get { _debugImageCounter } - set { _debugImageCounter = newValue } - } - - private nonisolated(unsafe) static var frameCounter: Int { - get { _frameCounter } - set { _frameCounter = newValue } - } - - private nonisolated(unsafe) static var lastPupilPositions: (left: PupilPosition?, right: PupilPosition?) - { - get { _lastPupilPositions } - set { _lastPupilPositions = newValue } - } - - private nonisolated(unsafe) static var metrics: PupilDetectorMetrics { - get { _metrics } - set { _metrics = newValue } - } - - // MARK: - Precomputed Tables - - private nonisolated static let spatialWeightsLUT: [[Float]] = { - let d = 10 - let radius = d / 2 - let sigmaSpace: Float = 15.0 - var weights = [[Float]](repeating: [Float](repeating: 0, count: d), count: d) - for dy in 0..? - private nonisolated(unsafe) static var grayscaleBufferSize = 0 - private nonisolated(unsafe) static var eyeBuffer: UnsafeMutablePointer? - private nonisolated(unsafe) static var eyeBufferSize = 0 - private nonisolated(unsafe) static var tempBuffer: UnsafeMutablePointer? - private nonisolated(unsafe) static var tempBufferSize = 0 - - // MARK: - Public API - - /// Call once per video frame to enable proper frame skipping - nonisolated static func advanceFrame() { - frameCounter += 1 - } - - /// Detects pupil position with frame skipping for performance - /// Returns cached result on skipped frames - nonisolated static func detectPupil( - in pixelBuffer: CVPixelBuffer, - eyeLandmarks: VNFaceLandmarkRegion2D, - faceBoundingBox: CGRect, - imageSize: CGSize, - side: Int = 0, - threshold: Int? = nil - ) -> (pupilPosition: PupilPosition, eyeRegion: EyeRegion)? { - // Frame skipping - return cached result - if frameCounter % frameSkipCount != 0 { - let cachedPosition = side == 0 ? lastPupilPositions.left : lastPupilPositions.right - if let position = cachedPosition { - // Recreate eye region for consistency - let eyePoints = landmarksToPixelCoordinates( - landmarks: eyeLandmarks, - faceBoundingBox: faceBoundingBox, - imageSize: imageSize - ) - if let eyeRegion = createEyeRegion(from: eyePoints, imageSize: imageSize) { - return (position, eyeRegion) - } - } - return nil - } - - let startTime = CFAbsoluteTimeGetCurrent() - defer { - if enablePerformanceLogging { - let elapsed = (CFAbsoluteTimeGetCurrent() - startTime) * 1000 - metrics.recordProcessingTime(elapsed) - if metrics.processedFrameCount % 30 == 0 { - logDebug( - "👁 PupilDetector: \(String(format: "%.2f", elapsed))ms (avg: \(String(format: "%.2f", metrics.averageProcessingTimeMs))ms)" - ) - } - } - } - - // Step 1: Convert Vision landmarks to pixel coordinates - let eyePoints = landmarksToPixelCoordinates( - landmarks: eyeLandmarks, - faceBoundingBox: faceBoundingBox, - imageSize: imageSize - ) - - guard eyePoints.count >= 6 else { - if enableDiagnosticLogging { - logDebug("👁 PupilDetector: Failed - eyePoints.count=\(eyePoints.count) < 6") - } - return nil - } - - // Step 2: Create eye region bounding box with margin - guard let eyeRegion = createEyeRegion(from: eyePoints, imageSize: imageSize) else { - if enableDiagnosticLogging { - logDebug("👁 PupilDetector: Failed - createEyeRegion returned nil") - } - return nil - } - - // Store eye region for debug overlay - if side == 0 { - debugLeftEyeRegion = eyeRegion - } else { - debugRightEyeRegion = eyeRegion - } - debugImageSize = imageSize - - let frameWidth = CVPixelBufferGetWidth(pixelBuffer) - let frameHeight = CVPixelBufferGetHeight(pixelBuffer) - let frameSize = frameWidth * frameHeight - - // Step 3: Ensure buffers are allocated - ensureBufferCapacity( - frameSize: frameSize, eyeSize: Int(eyeRegion.frame.width * eyeRegion.frame.height)) - - guard let grayBuffer = grayscaleBuffer, - let eyeBuf = eyeBuffer, - let tmpBuf = tempBuffer - else { - if enableDiagnosticLogging { - logDebug("👁 PupilDetector: Failed - buffers not allocated") - } - return nil - } - - // Step 4: Extract grayscale data using vImage - guard - extractGrayscaleDataOptimized( - from: pixelBuffer, to: grayBuffer, width: frameWidth, height: frameHeight) - else { - if enableDiagnosticLogging { - logDebug("👁 PupilDetector: Failed - grayscale extraction failed") - } - return nil - } - - // Step 5: Isolate eye with polygon mask - let eyeWidth = Int(eyeRegion.frame.width) - let eyeHeight = Int(eyeRegion.frame.height) - - // Early exit for tiny regions (less than 10x10 pixels) - guard eyeWidth >= 10, eyeHeight >= 10 else { - if enableDiagnosticLogging { - logDebug( - "👁 PupilDetector: Failed - eye region too small (\(eyeWidth)x\(eyeHeight))") - } - return nil - } - - guard - isolateEyeWithMaskOptimized( - frameData: grayBuffer, - frameWidth: frameWidth, - frameHeight: frameHeight, - eyePoints: eyePoints, - region: eyeRegion, - output: eyeBuf - ) - else { - if enableDiagnosticLogging { - logDebug("👁 PupilDetector: Failed - isolateEyeWithMask failed") - } - return nil - } - - // Step 6: Get threshold (from calibration or parameter) - let effectiveThreshold: Int - if let manualThreshold = threshold { - effectiveThreshold = manualThreshold - } else if calibration.isComplete() { - effectiveThreshold = calibration.threshold(forSide: side) - } else { - calibration.evaluate(eyeData: eyeBuf, width: eyeWidth, height: eyeHeight, side: side) - effectiveThreshold = calibration.threshold(forSide: side) - } - - // Step 7: Process image (bilateral filter + erosion + threshold) - if enableDiagnosticLogging { - logDebug( - "👁 PupilDetector: Using threshold=\(effectiveThreshold) for \(eyeWidth)x\(eyeHeight) eye region" - ) - } - - // Debug: Save input eye image before processing - if enableDebugImageSaving && debugImageCounter < 20 { - NSLog( - "📸 Saving eye_input_%d - %dx%d, side=%d, region=(%.0f,%.0f,%.0f,%.0f)", - debugImageCounter, eyeWidth, eyeHeight, side, - eyeRegion.frame.origin.x, eyeRegion.frame.origin.y, - eyeRegion.frame.width, eyeRegion.frame.height) - - // Debug: Print pixel value statistics for input - var minVal: UInt8 = 255 - var maxVal: UInt8 = 0 - var sum: Int = 0 - var darkCount = 0 // pixels <= 90 - for i in 0..<(eyeWidth * eyeHeight) { - let v = eyeBuf[i] - if v < minVal { minVal = v } - if v > maxVal { maxVal = v } - sum += Int(v) - if v <= 90 { darkCount += 1 } - } - let avgVal = Double(sum) / Double(eyeWidth * eyeHeight) - NSLog( - "📊 Eye input stats: min=%d, max=%d, avg=%.1f, darkPixels(<=90)=%d", minVal, maxVal, - avgVal, darkCount) - - saveDebugImage( - data: eyeBuf, width: eyeWidth, height: eyeHeight, - name: "eye_input_\(debugImageCounter)") - } - - imageProcessingOptimized( - input: eyeBuf, - output: tmpBuf, - width: eyeWidth, - height: eyeHeight, - threshold: effectiveThreshold - ) - - // Capture debug images for UI display - let inputImage = createCGImage(from: eyeBuf, width: eyeWidth, height: eyeHeight) - let processedImage = createCGImage(from: tmpBuf, width: eyeWidth, height: eyeHeight) - let eyeSize = CGSize(width: eyeWidth, height: eyeHeight) - if side == 0 { - debugLeftEyeInput = inputImage - debugLeftEyeProcessed = processedImage - debugLeftEyeSize = eyeSize - } else { - debugRightEyeInput = inputImage - debugRightEyeProcessed = processedImage - debugRightEyeSize = eyeSize - } - - // Debug: Save processed images if enabled - if enableDebugImageSaving && debugImageCounter < 10 { - // Debug: Print pixel value statistics for output - var darkCount = 0 // pixels == 0 (black) - var whiteCount = 0 // pixels == 255 (white) - for i in 0..<(eyeWidth * eyeHeight) { - if tmpBuf[i] == 0 { darkCount += 1 } else if tmpBuf[i] == 255 { whiteCount += 1 } - } - NSLog("📊 Processed output stats: darkPixels=%d, whitePixels=%d", darkCount, whiteCount) - - saveDebugImage( - data: tmpBuf, width: eyeWidth, height: eyeHeight, - name: "processed_eye_\(debugImageCounter)") - debugImageCounter += 1 - } - - // Step 8: Find contours and compute centroid - guard - let (centroidX, centroidY) = findPupilFromContoursOptimized( - data: tmpBuf, - width: eyeWidth, - height: eyeHeight - ) - else { - if enableDiagnosticLogging { - logDebug( - "👁 PupilDetector: Failed - findPupilFromContours returned nil (not enough dark pixels) for side \(side)" - ) - } - return nil - } - - if enableDiagnosticLogging { - logDebug( - "👁 PupilDetector: Success side=\(side) - centroid at (\(String(format: "%.1f", centroidX)), \(String(format: "%.1f", centroidY))) in \(eyeWidth)x\(eyeHeight) region" - ) - } - - let pupilPosition = PupilPosition(x: CGFloat(centroidX), y: CGFloat(centroidY)) - - // Cache result and debug position - if side == 0 { - lastPupilPositions.left = pupilPosition - debugLeftPupilPosition = pupilPosition - } else { - lastPupilPositions.right = pupilPosition - debugRightPupilPosition = pupilPosition - } - - return (pupilPosition, eyeRegion) - } - - // MARK: - Buffer Management - - // MARK: - Buffer Management - - private nonisolated static func ensureBufferCapacity(frameSize: Int, eyeSize: Int) { - if grayscaleBufferSize < frameSize { - grayscaleBuffer?.deallocate() - grayscaleBuffer = UnsafeMutablePointer.allocate(capacity: frameSize) - grayscaleBufferSize = frameSize - } - - let requiredEyeSize = max(eyeSize, 10000) // Minimum size for safety - if eyeBufferSize < requiredEyeSize { - eyeBuffer?.deallocate() - tempBuffer?.deallocate() - eyeBuffer = UnsafeMutablePointer.allocate(capacity: requiredEyeSize) - tempBuffer = UnsafeMutablePointer.allocate(capacity: requiredEyeSize) - eyeBufferSize = requiredEyeSize - } - } - - // MARK: - Optimized Grayscale Conversion (vImage) - - private nonisolated static func extractGrayscaleDataOptimized( - from pixelBuffer: CVPixelBuffer, - to output: UnsafeMutablePointer, - width: Int, - height: Int - ) -> Bool { - CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly) - defer { CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly) } - - let pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer) - - switch pixelFormat { - case kCVPixelFormatType_32BGRA: - guard let baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer) else { return false } - let bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer) - - var srcBuffer = vImage_Buffer( - data: baseAddress, - height: vImagePixelCount(height), - width: vImagePixelCount(width), - rowBytes: bytesPerRow - ) - - var dstBuffer = vImage_Buffer( - data: output, - height: vImagePixelCount(height), - width: vImagePixelCount(width), - rowBytes: width - ) - - // BGRA to Planar8 grayscale using luminance coefficients - // Y = 0.299*R + 0.587*G + 0.114*B - let matrix: [Int16] = [ - 28, // B coefficient (0.114 * 256 ≈ 29, adjusted) - 150, // G coefficient (0.587 * 256 ≈ 150) - 77, // R coefficient (0.299 * 256 ≈ 77) - 0, // A coefficient - ] - let divisor: Int32 = 256 - - let error = vImageMatrixMultiply_ARGB8888ToPlanar8( - &srcBuffer, - &dstBuffer, - matrix, - divisor, - nil, - 0, - vImage_Flags(kvImageNoFlags) - ) - - return error == kvImageNoError - - case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, - kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: - guard let yPlane = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0) else { - return false - } - let yBytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0) - let yBuffer = yPlane.assumingMemoryBound(to: UInt8.self) - - // Direct copy of Y plane (already grayscale) - for y in 0.., - frameWidth: Int, - frameHeight: Int, - eyePoints: [CGPoint], - region: EyeRegion, - output: UnsafeMutablePointer - ) -> Bool { - let minX = Int(region.frame.origin.x) - let minY = Int(region.frame.origin.y) - let eyeWidth = Int(region.frame.width) - let eyeHeight = Int(region.frame.height) - - guard eyeWidth > 0, eyeHeight > 0 else { return false } - - // Initialize to WHITE (255) - masked pixels should be bright so they don't affect pupil detection - memset(output, 255, eyeWidth * eyeHeight) - - // Convert eye points to local coordinates - let localPoints = eyePoints.map { point in - (x: Float(point.x) - Float(minX), y: Float(point.y) - Float(minY)) - } - - // Precompute edge data for faster point-in-polygon - let edges = (0..= 0, frameX < frameWidth, frameY >= 0, frameY < frameHeight { - output[y * eyeWidth + x] = frameData[frameY * frameWidth + frameX] - } - } - } - } - - return true - } - - @inline(__always) - private nonisolated static func pointInPolygonFast( - px: Float, py: Float, edges: [(x1: Float, y1: Float, x2: Float, y2: Float)] - ) -> Bool { - var inside = false - for edge in edges { - if ((edge.y1 > py) != (edge.y2 > py)) - && (px < (edge.x2 - edge.x1) * (py - edge.y1) / (edge.y2 - edge.y1) + edge.x1) - { - inside = !inside - } - } - return inside - } - - // MARK: - Optimized Image Processing - - nonisolated static func imageProcessingOptimized( - input: UnsafePointer, - output: UnsafeMutablePointer, - width: Int, - height: Int, - threshold: Int - ) { - let size = width * height - guard size > 0 else { return } - - // 1. Apply Gaussian Blur (reduces noise) - // We reuse tempBuffer for intermediate steps if available, or just output - // Note: gaussianBlurOptimized writes from input -> output - gaussianBlurOptimized(input: input, output: output, width: width, height: height) - - // 2. Apply Erosion (expands dark regions) - // Python: cv2.erode(kernel, iterations=3) - // This helps connect broken parts of the pupil - // Note: erodeOptimized processes in-place on output if input==output - erodeOptimized(input: output, output: output, width: width, height: height, iterations: 3) - - // 3. Binary Threshold - for i in 0.. threshold become 255 (white), others 0 (black) - // So Pupil is BLACK (0) - output[i] = output[i] > UInt8(threshold) ? 255 : 0 - } - } - - private nonisolated static func gaussianBlurOptimized( - input: UnsafePointer, - output: UnsafeMutablePointer, - width: Int, - height: Int - ) { - // Use a more appropriate convolution for performance - // Using vImageTentConvolve_Planar8 with optimized parameters - - var srcBuffer = vImage_Buffer( - data: UnsafeMutableRawPointer(mutating: input), - height: vImagePixelCount(height), - width: vImagePixelCount(width), - rowBytes: width - ) - - var dstBuffer = vImage_Buffer( - data: UnsafeMutableRawPointer(output), - height: vImagePixelCount(height), - width: vImagePixelCount(width), - rowBytes: width - ) - - // Kernel size that provides good blur with minimal computational overhead - let kernelSize: UInt32 = 5 - - vImageTentConvolve_Planar8( - &srcBuffer, - &dstBuffer, - nil, - 0, 0, - kernelSize, - kernelSize, - 0, - vImage_Flags(kvImageEdgeExtend) - ) - } - - private nonisolated static func erodeOptimized( - input: UnsafePointer, - output: UnsafeMutablePointer, - width: Int, - height: Int, - iterations: Int - ) { - guard iterations > 0 else { - memcpy(output, input, width * height) - return - } - - // Copy input to output first so we can use output as working buffer - memcpy(output, input, width * height) - - var srcBuffer = vImage_Buffer( - data: UnsafeMutableRawPointer(output), - height: vImagePixelCount(height), - width: vImagePixelCount(width), - rowBytes: width - ) - - // Allocate temp buffer for ping-pong - let tempData = UnsafeMutablePointer.allocate(capacity: width * height) - defer { tempData.deallocate() } - - var dstBuffer = vImage_Buffer( - data: UnsafeMutableRawPointer(tempData), - height: vImagePixelCount(height), - width: vImagePixelCount(width), - rowBytes: width - ) - - // 3x3 erosion kernel (all ones) - let kernel: [UInt8] = [ - 1, 1, 1, - 1, 1, 1, - 1, 1, 1, - ] - - for i in 0.., - width: Int, - height: Int - ) -> (x: Double, y: Double)? { - let size = width * height - - // 1. Threshold pass: Identify all dark pixels (0) - // We use a visited array to track processed pixels for flood fill - // Using a flat bool array for performance - var visited = [Bool](repeating: false, count: size) - - var maxBlobSize = 0 - var maxBlobSumX = 0 - var maxBlobSumY = 0 - - // 2. Iterate through pixels to find connected components - for y in 0..= 0 { - let nIdx = currentIdx - 1 - if data[nIdx] == 0 && !visited[nIdx] { - visited[nIdx] = true - stack.append(nIdx) - } - } - // Down - if cy + 1 < height { - let nIdx = currentIdx + width - if data[nIdx] == 0 && !visited[nIdx] { - visited[nIdx] = true - stack.append(nIdx) - } - } - // Up - if cy - 1 >= 0 { - let nIdx = currentIdx - width - if data[nIdx] == 0 && !visited[nIdx] { - visited[nIdx] = true - stack.append(nIdx) - } - } - } - - // Check if this is the largest blob so far - if currentBlobSize > maxBlobSize { - maxBlobSize = currentBlobSize - maxBlobSumX = currentBlobSumX - maxBlobSumY = currentBlobSumY - } - } - } - } - - if enableDiagnosticLogging && maxBlobSize < 5 { - logDebug("👁 PupilDetector: Largest blob size = \(maxBlobSize) (need >= 5)") - } - - // Minimum 5 pixels for valid pupil - guard maxBlobSize >= 5 else { return nil } - - return ( - x: Double(maxBlobSumX) / Double(maxBlobSize), - y: Double(maxBlobSumY) / Double(maxBlobSize) - ) - } - - // MARK: - Helper Methods - - private nonisolated static func landmarksToPixelCoordinates( - landmarks: VNFaceLandmarkRegion2D, - faceBoundingBox: CGRect, - imageSize: CGSize - ) -> [CGPoint] { - // Vision uses bottom-left origin (normalized 0-1), CVPixelBuffer uses top-left - // We need to flip Y: flippedY = 1.0 - y - return landmarks.normalizedPoints.map { point in - let imageX = - (faceBoundingBox.origin.x + point.x * faceBoundingBox.width) * imageSize.width - // Flip Y coordinate for pixel buffer coordinate system - let flippedY = 1.0 - (faceBoundingBox.origin.y + point.y * faceBoundingBox.height) - let imageY = flippedY * imageSize.height - return CGPoint(x: imageX, y: imageY) - } - } - - private nonisolated static func createEyeRegion(from points: [CGPoint], imageSize: CGSize) - -> EyeRegion? - { - guard !points.isEmpty else { return nil } - - let margin: CGFloat = 5 - var minX = CGFloat.greatestFiniteMagnitude - var maxX = -CGFloat.greatestFiniteMagnitude - var minY = CGFloat.greatestFiniteMagnitude - var maxY = -CGFloat.greatestFiniteMagnitude - - for point in points { - minX = min(minX, point.x) - maxX = max(maxX, point.x) - minY = min(minY, point.y) - maxY = max(maxY, point.y) - } - - minX -= margin - maxX += margin - minY -= margin - maxY += margin - - let clampedMinX = max(0, minX) - let clampedMaxX = min(imageSize.width, maxX) - let clampedMinY = max(0, minY) - let clampedMaxY = min(imageSize.height, maxY) - - let frame = CGRect( - x: clampedMinX, - y: clampedMinY, - width: clampedMaxX - clampedMinX, - height: clampedMaxY - clampedMinY - ) - - let center = CGPoint(x: frame.width / 2, y: frame.height / 2) - let origin = CGPoint(x: clampedMinX, y: clampedMinY) - - return EyeRegion(frame: frame, center: center, origin: origin) - } - - // MARK: - Debug Helpers - - private nonisolated static func saveDebugImage( - data: UnsafePointer, width: Int, height: Int, name: String - ) { - guard let cgImage = createCGImage(from: data, width: width, height: height) else { - NSLog("⚠️ PupilDetector: createCGImage failed for %@ (%dx%d)", name, width, height) - return - } - - let url = URL(fileURLWithPath: "/tmp/gaze_debug/\(name).png") - guard - let destination = CGImageDestinationCreateWithURL( - url as CFURL, UTType.png.identifier as CFString, 1, nil) - else { - NSLog("⚠️ PupilDetector: CGImageDestinationCreateWithURL failed for %@", url.path) - return - } - - CGImageDestinationAddImage(destination, cgImage, nil) - let success = CGImageDestinationFinalize(destination) - if success { - NSLog("💾 Saved debug image: %@", url.path) - } else { - NSLog("⚠️ PupilDetector: CGImageDestinationFinalize failed for %@", url.path) - } - } - - private nonisolated static func createCGImage( - from data: UnsafePointer, width: Int, height: Int - ) - -> CGImage? - { - guard width > 0 && height > 0 else { - print("⚠️ PupilDetector: Invalid dimensions \(width)x\(height)") - return nil - } - - // Create a Data object that copies the pixel data - let pixelData = Data(bytes: data, count: width * height) - - // Create CGImage from the data using CGDataProvider - guard let provider = CGDataProvider(data: pixelData as CFData) else { - print("⚠️ PupilDetector: CGDataProvider creation failed") - return nil - } - - let cgImage = CGImage( - width: width, - height: height, - bitsPerComponent: 8, - bitsPerPixel: 8, - bytesPerRow: width, - space: CGColorSpaceCreateDeviceGray(), - bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.none.rawValue), - provider: provider, - decode: nil, - shouldInterpolate: false, - intent: .defaultIntent - ) - - if cgImage == nil { - print("⚠️ PupilDetector: CGImage creation failed") - } - - return cgImage - } - - /// Clean up allocated buffers (call on app termination if needed) - nonisolated static func cleanup() { - grayscaleBuffer?.deallocate() - grayscaleBuffer = nil - grayscaleBufferSize = 0 - - eyeBuffer?.deallocate() - eyeBuffer = nil - - tempBuffer?.deallocate() - tempBuffer = nil - eyeBufferSize = 0 - } -} diff --git a/Gaze/Services/EyeTracking/TrackingModels.swift b/Gaze/Services/EyeTracking/TrackingModels.swift new file mode 100644 index 0000000..5b2e153 --- /dev/null +++ b/Gaze/Services/EyeTracking/TrackingModels.swift @@ -0,0 +1,72 @@ +// +// TrackingModels.swift +// Gaze +// +// Created by Mike Freno on 1/31/26. +// + +import Foundation + +public enum GazeState: String, Sendable { + case lookingAtScreen + case lookingAway + case unknown +} + +public struct TrackingResult: Sendable { + public let faceDetected: Bool + public let gazeState: GazeState + public let eyesClosed: Bool + public let confidence: Double + public let timestamp: Date + + public static let empty = TrackingResult( + faceDetected: false, + gazeState: .unknown, + eyesClosed: false, + confidence: 0, + timestamp: Date() + ) +} + +public struct EyeTrackingDebugState: Sendable { + public let leftEyeRect: CGRect? + public let rightEyeRect: CGRect? + public let leftPupil: CGPoint? + public let rightPupil: CGPoint? + public let imageSize: CGSize? + + public static let empty = EyeTrackingDebugState( + leftEyeRect: nil, + rightEyeRect: nil, + leftPupil: nil, + rightPupil: nil, + imageSize: nil + ) +} + +public struct TrackingConfig: Sendable { + public let horizontalAwayThreshold: Double + public let verticalAwayThreshold: Double + public let minBaselineSamples: Int + public let baselineSmoothing: Double + public let baselineUpdateThreshold: Double + public let minConfidence: Double + public let eyeClosedThreshold: Double + public let baselineEnabled: Bool + public let defaultCenterHorizontal: Double + public let defaultCenterVertical: Double + + public static let `default` = TrackingConfig( + horizontalAwayThreshold: 0.12, + verticalAwayThreshold: 0.18, + minBaselineSamples: 8, + baselineSmoothing: 0.15, + baselineUpdateThreshold: 0.08, + minConfidence: 0.5, + eyeClosedThreshold: 0.18, + baselineEnabled: true, + defaultCenterHorizontal: 0.5, + defaultCenterVertical: 0.5 + ) +} diff --git a/Gaze/Services/EyeTracking/VisionGazeProcessor.swift b/Gaze/Services/EyeTracking/VisionGazeProcessor.swift new file mode 100644 index 0000000..8e51571 --- /dev/null +++ b/Gaze/Services/EyeTracking/VisionGazeProcessor.swift @@ -0,0 +1,297 @@ +// +// VisionGazeProcessor.swift +// Gaze +// +// Created by Mike Freno on 1/31/26. +// + +import Foundation +@preconcurrency import Vision + +final class VisionGazeProcessor: @unchecked Sendable { + struct EyeObservation: Sendable { + let center: CGPoint + let width: Double + let height: Double + let pupil: CGPoint? + let frame: CGRect + let normalizedPupil: CGPoint? + let hasPupilLandmarks: Bool + } + + struct ObservationResult: Sendable { + let faceDetected: Bool + let eyesClosed: Bool + let gazeState: GazeState + let confidence: Double + let horizontal: Double? + let vertical: Double? + let debugState: EyeTrackingDebugState + } + + private let baselineModel = GazeBaselineModel() + private var config: TrackingConfig + + init(config: TrackingConfig) { + self.config = config + } + + func updateConfig(_ config: TrackingConfig) { + self.config = config + } + + func resetBaseline() { + baselineModel.reset() + } + + func process(analysis: VisionPipeline.FaceAnalysis) -> ObservationResult { + guard analysis.faceDetected, let face = analysis.face?.value else { + return ObservationResult( + faceDetected: false, + eyesClosed: false, + gazeState: .unknown, + confidence: 0, + horizontal: nil, + vertical: nil, + debugState: .empty + ) + } + + guard let landmarks = face.landmarks else { + return ObservationResult( + faceDetected: true, + eyesClosed: false, + gazeState: .unknown, + confidence: 0.3, + horizontal: nil, + vertical: nil, + debugState: .empty + ) + } + + let leftEye = makeEyeObservation( + eye: landmarks.leftEye, + pupil: landmarks.leftPupil, + face: face, + imageSize: analysis.imageSize + ) + let rightEye = makeEyeObservation( + eye: landmarks.rightEye, + pupil: landmarks.rightPupil, + face: face, + imageSize: analysis.imageSize + ) + + let eyesClosed = detectEyesClosed(left: leftEye, right: rightEye) + let (horizontal, vertical) = normalizePupilPosition(left: leftEye, right: rightEye) + + let confidence = calculateConfidence(leftEye: leftEye, rightEye: rightEye) + let gazeState = decideGazeState( + horizontal: horizontal, + vertical: vertical, + confidence: confidence, + eyesClosed: eyesClosed + ) + + let debugState = EyeTrackingDebugState( + leftEyeRect: leftEye?.frame, + rightEyeRect: rightEye?.frame, + leftPupil: leftEye?.pupil, + rightPupil: rightEye?.pupil, + imageSize: analysis.imageSize + ) + + return ObservationResult( + faceDetected: true, + eyesClosed: eyesClosed, + gazeState: gazeState, + confidence: confidence, + horizontal: horizontal, + vertical: vertical, + debugState: debugState + ) + } + + private func makeEyeObservation( + eye: VNFaceLandmarkRegion2D?, + pupil: VNFaceLandmarkRegion2D?, + face: VNFaceObservation, + imageSize: CGSize + ) -> EyeObservation? { + guard let eye else { return nil } + + let eyePoints = normalizePoints(eye.normalizedPoints, face: face, imageSize: imageSize) + guard let bounds = boundingRect(points: eyePoints) else { return nil } + + let pupilPoint: CGPoint? + let hasPupilLandmarks = (pupil?.pointCount ?? 0) > 0 + if let pupil, pupil.pointCount > 0 { + let pupilPoints = normalizePoints(pupil.normalizedPoints, face: face, imageSize: imageSize) + pupilPoint = averagePoint(pupilPoints, fallback: bounds.center) + } else { + pupilPoint = bounds.center + } + + let normalizedPupil: CGPoint? + if let pupilPoint { + let nx = clamp((pupilPoint.x - bounds.minX) / bounds.size.width) + let ny = clamp((pupilPoint.y - bounds.minY) / bounds.size.height) + normalizedPupil = CGPoint(x: nx, y: ny) + } else { + normalizedPupil = nil + } + + return EyeObservation( + center: bounds.center, + width: bounds.size.width, + height: bounds.size.height, + pupil: pupilPoint, + frame: CGRect(x: bounds.minX, y: bounds.minY, width: bounds.size.width, height: bounds.size.height), + normalizedPupil: normalizedPupil, + hasPupilLandmarks: hasPupilLandmarks + ) + } + + private func normalizePoints( + _ points: [CGPoint], + face: VNFaceObservation, + imageSize: CGSize + ) -> [CGPoint] { + points.map { point in + let x = (face.boundingBox.origin.x + point.x * face.boundingBox.size.width) + * imageSize.width + let y = (1.0 - (face.boundingBox.origin.y + point.y * face.boundingBox.size.height)) + * imageSize.height + return CGPoint(x: x, y: y) + } + } + + private func boundingRect(points: [CGPoint]) -> (center: CGPoint, size: CGSize, minX: CGFloat, minY: CGFloat)? { + guard !points.isEmpty else { return nil } + var minX = CGFloat.greatestFiniteMagnitude + var maxX = -CGFloat.greatestFiniteMagnitude + var minY = CGFloat.greatestFiniteMagnitude + var maxY = -CGFloat.greatestFiniteMagnitude + + for point in points { + minX = min(minX, point.x) + maxX = max(maxX, point.x) + minY = min(minY, point.y) + maxY = max(maxY, point.y) + } + + let width = maxX - minX + let height = maxY - minY + guard width > 0, height > 0 else { return nil } + + return ( + center: CGPoint(x: minX + width / 2, y: minY + height / 2), + size: CGSize(width: width, height: height), + minX: minX, + minY: minY + ) + } + + private func averagePoint(_ points: [CGPoint], fallback: CGPoint) -> CGPoint { + guard !points.isEmpty else { return fallback } + let sum = points.reduce(CGPoint.zero) { partial, next in + CGPoint(x: partial.x + next.x, y: partial.y + next.y) + } + return CGPoint(x: sum.x / CGFloat(points.count), y: sum.y / CGFloat(points.count)) + } + + private func clamp(_ value: CGFloat) -> CGFloat { + min(1, max(0, value)) + } + + private func averageCoordinate(left: CGFloat?, right: CGFloat?, fallback: Double?) -> Double? { + switch (left, right) { + case let (left?, right?): + return Double((left + right) / 2) + case let (left?, nil): + return Double(left) + case let (nil, right?): + return Double(right) + default: + return fallback + } + } + + private func normalizePupilPosition( + left: EyeObservation?, + right: EyeObservation? + ) -> (horizontal: Double?, vertical: Double?) { + let leftPupil = left?.normalizedPupil + let rightPupil = right?.normalizedPupil + + let horizontal = averageCoordinate( + left: leftPupil?.x, + right: rightPupil?.x, + fallback: nil + ) + let vertical = averageCoordinate( + left: leftPupil?.y, + right: rightPupil?.y, + fallback: nil + ) + return (horizontal, vertical) + } + + private func detectEyesClosed(left: EyeObservation?, right: EyeObservation?) -> Bool { + guard let left, let right else { return false } + let leftRatio = left.height / max(left.width, 1) + let rightRatio = right.height / max(right.width, 1) + let avgRatio = (leftRatio + rightRatio) / 2 + return avgRatio < config.eyeClosedThreshold + } + + private func calculateConfidence(leftEye: EyeObservation?, rightEye: EyeObservation?) -> Double { + var score = 0.0 + if leftEye?.hasPupilLandmarks == true { score += 0.35 } + if rightEye?.hasPupilLandmarks == true { score += 0.35 } + if leftEye != nil { score += 0.15 } + if rightEye != nil { score += 0.15 } + return min(1.0, score) + } + + private func decideGazeState( + horizontal: Double?, + vertical: Double?, + confidence: Double, + eyesClosed: Bool + ) -> GazeState { + guard confidence >= config.minConfidence else { return .unknown } + guard let horizontal, let vertical else { return .unknown } + if eyesClosed { return .unknown } + + let baseline = baselineModel.current( + defaultH: config.defaultCenterHorizontal, + defaultV: config.defaultCenterVertical + ) + + let deltaH = abs(horizontal - baseline.horizontal) + let deltaV = abs(vertical - baseline.vertical) + let away = deltaH > config.horizontalAwayThreshold || deltaV > config.verticalAwayThreshold + + if config.baselineEnabled { + if baseline.sampleCount < config.minBaselineSamples { + baselineModel.update( + horizontal: horizontal, + vertical: vertical, + smoothing: config.baselineSmoothing + ) + } else if deltaH < config.baselineUpdateThreshold + && deltaV < config.baselineUpdateThreshold { + baselineModel.update( + horizontal: horizontal, + vertical: vertical, + smoothing: config.baselineSmoothing + ) + } + } + + let stable = baseline.sampleCount >= config.minBaselineSamples || !config.baselineEnabled + if !stable { return .unknown } + return away ? .lookingAway : .lookingAtScreen + } +} diff --git a/Gaze/Services/EyeTracking/VisionPipeline.swift b/Gaze/Services/EyeTracking/VisionPipeline.swift index 70210a2..e96f8b3 100644 --- a/Gaze/Services/EyeTracking/VisionPipeline.swift +++ b/Gaze/Services/EyeTracking/VisionPipeline.swift @@ -13,8 +13,6 @@ final class VisionPipeline: @unchecked Sendable { let faceDetected: Bool let face: NonSendableFaceObservation? let imageSize: CGSize - let debugYaw: Double? - let debugPitch: Double? } struct NonSendableFaceObservation: @unchecked Sendable { @@ -44,9 +42,7 @@ final class VisionPipeline: @unchecked Sendable { return FaceAnalysis( faceDetected: false, face: nil, - imageSize: imageSize, - debugYaw: nil, - debugPitch: nil + imageSize: imageSize ) } @@ -54,18 +50,14 @@ final class VisionPipeline: @unchecked Sendable { return FaceAnalysis( faceDetected: false, face: nil, - imageSize: imageSize, - debugYaw: nil, - debugPitch: nil + imageSize: imageSize ) } return FaceAnalysis( faceDetected: true, face: NonSendableFaceObservation(value: face), - imageSize: imageSize, - debugYaw: face.yaw?.doubleValue, - debugPitch: face.pitch?.doubleValue + imageSize: imageSize ) } } diff --git a/Gaze/Views/Components/CalibrationOverlayView.swift b/Gaze/Views/Components/CalibrationOverlayView.swift deleted file mode 100644 index 090ea6e..0000000 --- a/Gaze/Views/Components/CalibrationOverlayView.swift +++ /dev/null @@ -1,456 +0,0 @@ -// -// CalibrationOverlayView.swift -// Gaze -// -// Fullscreen overlay view for eye tracking calibration targets. -// - -import AVFoundation -import Combine -import SwiftUI - -struct CalibrationOverlayView: View { - @StateObject private var calibratorService = CalibratorService.shared - @StateObject private var eyeTrackingService = EyeTrackingService.shared - @StateObject private var viewModel = CalibrationOverlayViewModel() - - let onDismiss: () -> Void - - var body: some View { - GeometryReader { geometry in - ZStack { - Color.black.ignoresSafeArea() - - // Camera preview at 50% opacity (mirrored for natural feel) - if let previewLayer = eyeTrackingService.previewLayer { - CameraPreviewView(previewLayer: previewLayer, borderColor: .clear) - .scaleEffect(x: -1, y: 1) - .opacity(0.5) - .ignoresSafeArea() - } - - if let error = viewModel.showError { - errorView(error) - } else if !viewModel.cameraStarted { - startingCameraView - } else if calibratorService.isCalibrating { - calibrationContentView(screenSize: geometry.size) - } else if viewModel.calibrationStarted - && calibratorService.calibrationData.isComplete - { - // Only show completion if we started calibration this session AND it completed - completionView - } else if viewModel.calibrationStarted { - // Calibration was started but not yet complete - show content - calibrationContentView(screenSize: geometry.size) - } - } - } - .task { - await viewModel.startCamera( - eyeTrackingService: eyeTrackingService, calibratorService: calibratorService) - } - .onDisappear { - viewModel.cleanup( - eyeTrackingService: eyeTrackingService, calibratorService: calibratorService) - } - .onChange(of: calibratorService.currentStep) { oldStep, newStep in - if newStep != nil && oldStep != newStep { - viewModel.startStepCountdown(calibratorService: calibratorService) - } - } - } - - // MARK: - Starting Camera View - - private var startingCameraView: some View { - VStack(spacing: 20) { - ProgressView() - .scaleEffect(2) - .tint(.white) - - Text("Starting camera...") - .font(.title2) - .foregroundStyle(.white) - } - } - - // MARK: - Error View - - private func errorView(_ message: String) -> some View { - VStack(spacing: 20) { - Image(systemName: "exclamationmark.triangle.fill") - .font(.system(size: 60)) - .foregroundStyle(.orange) - - Text("Camera Error") - .font(.title) - .foregroundStyle(.white) - - Text(message) - .font(.body) - .foregroundStyle(.gray) - .multilineTextAlignment(.center) - - Button("Close") { - onDismiss() - } - .buttonStyle(.borderedProminent) - .padding(.top, 20) - } - .padding(40) - } - - // MARK: - Calibration Content - - private func calibrationContentView(screenSize: CGSize) -> some View { - ZStack { - VStack { - progressBar - Spacer() - } - - if let step = calibratorService.currentStep { - calibrationTarget(for: step, screenSize: screenSize) - } - - VStack { - Spacer() - HStack { - cancelButton - Spacer() - if !calibratorService.isCollectingSamples { - skipButton - } - } - .padding(.horizontal, 40) - .padding(.bottom, 40) - } - - // Face detection indicator - VStack { - HStack { - Spacer() - faceDetectionIndicator - } - Spacer() - } - } - } - - // MARK: - Progress Bar - - private var progressBar: some View { - VStack(spacing: 10) { - HStack { - Text("Calibrating...") - .foregroundStyle(.white) - Spacer() - Text(calibratorService.progressText) - .foregroundStyle(.white.opacity(0.7)) - } - - ProgressView(value: calibratorService.progress) - .progressViewStyle(.linear) - .tint(.blue) - } - .padding() - .background(Color.black.opacity(0.7)) - } - - // MARK: - Face Detection Indicator - - private var faceDetectionIndicator: some View { - HStack(spacing: 8) { - Circle() - .fill(viewModel.stableFaceDetected ? Color.green : Color.red) - .frame(width: 12, height: 12) - - Text(viewModel.stableFaceDetected ? "Face detected" : "No face detected") - .font(.caption) - .foregroundStyle(.white.opacity(0.8)) - } - .padding(.horizontal, 16) - .padding(.vertical, 10) - .background(Color.black.opacity(0.7)) - .cornerRadius(20) - .padding() - .animation(.easeInOut(duration: 0.3), value: viewModel.stableFaceDetected) - } - - // MARK: - Calibration Target - - @ViewBuilder - private func calibrationTarget(for step: CalibrationStep, screenSize: CGSize) -> some View { - let position = targetPosition(for: step, screenSize: screenSize) - - VStack(spacing: 20) { - ZStack { - // Outer ring (pulsing when counting down) - Circle() - .stroke(Color.blue.opacity(0.3), lineWidth: 3) - .frame(width: 100, height: 100) - .scaleEffect(viewModel.isCountingDown ? 1.2 : 1.0) - .animation( - viewModel.isCountingDown - ? .easeInOut(duration: 0.6).repeatForever(autoreverses: true) - : .default, - value: viewModel.isCountingDown) - - // Progress ring when collecting - if calibratorService.isCollectingSamples { - Circle() - .trim(from: 0, to: CGFloat(calibratorService.samplesCollected) / 30.0) - .stroke(Color.green, lineWidth: 4) - .frame(width: 90, height: 90) - .rotationEffect(.degrees(-90)) - .animation( - .linear(duration: 0.1), value: calibratorService.samplesCollected) - } - - // Inner circle - Circle() - .fill(calibratorService.isCollectingSamples ? Color.green : Color.blue) - .frame(width: 60, height: 60) - .animation( - .easeInOut(duration: 0.3), value: calibratorService.isCollectingSamples) - - // Countdown number or collecting indicator - if viewModel.isCountingDown && viewModel.countdownValue > 0 { - Text("\(viewModel.countdownValue)") - .font(.system(size: 36, weight: .bold)) - .foregroundStyle(.white) - } else if calibratorService.isCollectingSamples { - Image(systemName: "eye.fill") - .font(.system(size: 24, weight: .bold)) - .foregroundStyle(.white) - } - } - - Text(instructionText(for: step)) - .font(.title2) - .foregroundStyle(.white) - .padding(.horizontal, 40) - .padding(.vertical, 15) - .background(Color.black.opacity(0.7)) - .cornerRadius(10) - } - .position(position) - } - - private func instructionText(for step: CalibrationStep) -> String { - if viewModel.isCountingDown && viewModel.countdownValue > 0 { - return "Get ready..." - } else if calibratorService.isCollectingSamples { - return "Look at the target" - } else { - return step.instructionText - } - } - - // MARK: - Buttons - - private var skipButton: some View { - Button { - viewModel.skipCurrentStep(calibratorService: calibratorService) - } label: { - Text("Skip") - .foregroundStyle(.white) - .padding(.horizontal, 20) - .padding(.vertical, 10) - .background(Color.white.opacity(0.2)) - .cornerRadius(8) - } - .buttonStyle(.plain) - } - - private var cancelButton: some View { - Button { - viewModel.cleanup( - eyeTrackingService: eyeTrackingService, calibratorService: calibratorService) - onDismiss() - } label: { - HStack(spacing: 6) { - Image(systemName: "xmark") - Text("Cancel") - } - .foregroundStyle(.white.opacity(0.7)) - .padding(.horizontal, 20) - .padding(.vertical, 10) - .background(Color.white.opacity(0.1)) - .cornerRadius(8) - } - .buttonStyle(.plain) - .keyboardShortcut(.escape, modifiers: []) - } - - // MARK: - Completion View - - private var completionView: some View { - VStack(spacing: 30) { - Image(systemName: "checkmark.circle.fill") - .font(.system(size: 80)) - .foregroundStyle(.green) - - Text("Calibration Complete!") - .font(.largeTitle) - .foregroundStyle(.white) - .fontWeight(.bold) - - Text("Your eye tracking has been calibrated successfully.") - .font(.title3) - .foregroundStyle(.gray) - - Button("Done") { - onDismiss() - } - .buttonStyle(.borderedProminent) - .keyboardShortcut(.return, modifiers: []) - .padding(.top, 20) - } - .onAppear { - DispatchQueue.main.asyncAfter(deadline: .now() + 2.5) { - onDismiss() - } - } - } - - // MARK: - Helper Methods - - private func targetPosition(for step: CalibrationStep, screenSize: CGSize) -> CGPoint { - let width = screenSize.width - let height = screenSize.height - - let centerX = width / 2 - let centerY = height / 2 - let marginX: CGFloat = 150 - let marginY: CGFloat = 120 - - switch step { - case .center: - return CGPoint(x: centerX, y: centerY) - case .left: - return CGPoint(x: centerX - width / 4, y: centerY) - case .right: - return CGPoint(x: centerX + width / 4, y: centerY) - case .farLeft: - return CGPoint(x: marginX, y: centerY) - case .farRight: - return CGPoint(x: width - marginX, y: centerY) - case .up: - return CGPoint(x: centerX, y: marginY) - case .down: - return CGPoint(x: centerX, y: height - marginY) - case .topLeft: - return CGPoint(x: marginX, y: marginY) - case .topRight: - return CGPoint(x: width - marginX, y: marginY) - case .bottomLeft: - return CGPoint(x: marginX, y: height - marginY) - case .bottomRight: - return CGPoint(x: width - marginX, y: height - marginY) - } - } -} - -// MARK: - ViewModel - -@MainActor -class CalibrationOverlayViewModel: ObservableObject { - @Published var countdownValue = 1 - @Published var isCountingDown = false - @Published var cameraStarted = false - @Published var showError: String? - @Published var calibrationStarted = false - @Published var stableFaceDetected = false // Debounced face detection - - private var countdownTask: Task? - private var faceDetectionCancellable: AnyCancellable? - private var lastFaceDetectedTime: Date = .distantPast - private let faceDetectionDebounce: TimeInterval = 0.5 // 500ms debounce - - func startCamera(eyeTrackingService: EyeTrackingService, calibratorService: CalibratorService) - async - { - do { - try await eyeTrackingService.startEyeTracking() - cameraStarted = true - - // Set up debounced face detection - setupFaceDetectionObserver(eyeTrackingService: eyeTrackingService) - - try? await Task.sleep(for: .seconds(0.5)) - - // Reset any previous calibration data before starting fresh - calibratorService.resetForNewCalibration() - calibratorService.startCalibration() - calibrationStarted = true - startStepCountdown(calibratorService: calibratorService) - } catch { - showError = "Failed to start camera: \(error.localizedDescription)" - } - } - - private func setupFaceDetectionObserver(eyeTrackingService: EyeTrackingService) { - faceDetectionCancellable = eyeTrackingService.$faceDetected - .receive(on: DispatchQueue.main) - .sink { [weak self] detected in - guard let self = self else { return } - - if detected { - // Face detected - update immediately - self.lastFaceDetectedTime = Date() - self.stableFaceDetected = true - } else { - // Face lost - only update after debounce period - let timeSinceLastDetection = Date().timeIntervalSince(self.lastFaceDetectedTime) - if timeSinceLastDetection > self.faceDetectionDebounce { - self.stableFaceDetected = false - } - } - } - } - - func cleanup(eyeTrackingService: EyeTrackingService, calibratorService: CalibratorService) { - countdownTask?.cancel() - countdownTask = nil - faceDetectionCancellable?.cancel() - faceDetectionCancellable = nil - isCountingDown = false - - if calibratorService.isCalibrating { - calibratorService.cancelCalibration() - } - - eyeTrackingService.stopEyeTracking() - } - - func skipCurrentStep(calibratorService: CalibratorService) { - countdownTask?.cancel() - countdownTask = nil - isCountingDown = false - calibratorService.skipStep() - } - - func startStepCountdown(calibratorService: CalibratorService) { - countdownTask?.cancel() - countdownTask = nil - countdownValue = 1 - isCountingDown = true - - countdownTask = Task { @MainActor in - // Just 1 second countdown - try? await Task.sleep(for: .seconds(1)) - if Task.isCancelled { return } - - // Done counting, start collecting - isCountingDown = false - countdownValue = 0 - calibratorService.startCollectingSamples() - } - } -} - -#Preview { - CalibrationOverlayView(onDismiss: {}) -} diff --git a/Gaze/Views/Components/EnforceModeSetupContent.swift b/Gaze/Views/Components/EnforceModeSetupContent.swift index e7cb79c..e5894f1 100644 --- a/Gaze/Views/Components/EnforceModeSetupContent.swift +++ b/Gaze/Views/Components/EnforceModeSetupContent.swift @@ -5,6 +5,7 @@ // Created by Mike Freno on 1/30/26. // +import AppKit import AVFoundation import SwiftUI @@ -13,15 +14,11 @@ struct EnforceModeSetupContent: View { @ObservedObject var cameraService = CameraAccessService.shared @ObservedObject var eyeTrackingService = EyeTrackingService.shared @ObservedObject var enforceModeService = EnforceModeService.shared - @ObservedObject var calibratorService = CalibratorService.shared @Environment(\.isCompactLayout) private var isCompact let presentation: SetupPresentation @Binding var isTestModeActive: Bool @Binding var cachedPreviewLayer: AVCaptureVideoPreviewLayer? - @Binding var showAdvancedSettings: Bool - @Binding var showCalibrationWindow: Bool - @Binding var isViewActive: Bool let isProcessingToggle: Bool let handleEnforceModeToggle: (Bool) -> Void @@ -86,9 +83,6 @@ struct EnforceModeSetupContent: View { Spacer(minLength: 0) } } - .sheet(isPresented: $showCalibrationWindow) { - EyeTrackingCalibrationView() - } } private var testModeButton: some View { @@ -120,65 +114,10 @@ struct EnforceModeSetupContent: View { .controlSize(presentation.isCard ? .regular : .large) } - private var calibrationSection: some View { - VStack(alignment: .leading, spacing: 12) { - HStack { - Image(systemName: "target") - .font(.title3) - .foregroundStyle(.blue) - Text("Eye Tracking Calibration") - .font(.headline) - } - - if calibratorService.calibrationData.isComplete { - VStack(alignment: .leading, spacing: 8) { - Text(calibratorService.getCalibrationSummary()) - .font(.caption) - .foregroundStyle(.secondary) - - if calibratorService.needsRecalibration() { - Label( - "Calibration expired - recalibration recommended", - systemImage: "exclamationmark.triangle.fill" - ) - .font(.caption) - .foregroundStyle(.orange) - } else { - Label("Calibration active and valid", systemImage: "checkmark.circle.fill") - .font(.caption) - .foregroundStyle(.green) - } - } - } else { - Text("Not calibrated - using default thresholds") - .font(.caption) - .foregroundStyle(.secondary) - } - - Button(action: { - showCalibrationWindow = true - }) { - HStack { - Image(systemName: "target") - Text( - calibratorService.calibrationData.isComplete - ? "Recalibrate" : "Run Calibration") - } - .frame(maxWidth: .infinity) - .padding(.vertical, 8) - } - .buttonStyle(.bordered) - .controlSize(.regular) - } - .padding(sectionPadding) - .glassEffectIfAvailable( - GlassStyle.regular.tint(.blue.opacity(0.1)), in: .rect(cornerRadius: sectionCornerRadius) - ) - } private var testModePreviewView: some View { VStack(spacing: 16) { - let lookingAway = !eyeTrackingService.userLookingAtScreen + let lookingAway = eyeTrackingService.trackingResult.gazeState == .lookingAway let borderColor: NSColor = lookingAway ? .systemGreen : .systemRed let previewLayer = eyeTrackingService.previewLayer ?? cachedPreviewLayer @@ -186,14 +125,12 @@ struct EnforceModeSetupContent: View { if let layer = previewLayer { ZStack { CameraPreviewView(previewLayer: layer, borderColor: borderColor) - PupilOverlayView(eyeTrackingService: eyeTrackingService) - VStack { - HStack { - Spacer() - GazeOverlayView(eyeTrackingService: eyeTrackingService) - } - Spacer() + GeometryReader { geometry in + EyeTrackingDebugOverlayView( + debugState: eyeTrackingService.debugState, + viewSize: geometry.size + ) } } .frame(height: presentation.isCard ? 180 : (isCompact ? 200 : 300)) @@ -256,13 +193,13 @@ struct EnforceModeSetupContent: View { HStack(spacing: 20) { statusIndicator( title: "Face Detected", - isActive: eyeTrackingService.faceDetected, + isActive: eyeTrackingService.trackingResult.faceDetected, icon: "person.fill" ) statusIndicator( title: "Looking Away", - isActive: !eyeTrackingService.userLookingAtScreen, + isActive: eyeTrackingService.trackingResult.gazeState == .lookingAway, icon: "arrow.turn.up.right" ) } @@ -358,190 +295,41 @@ struct EnforceModeSetupContent: View { private var trackingConstantsView: some View { VStack(alignment: .leading, spacing: 16) { HStack { - Text("Tracking Sensitivity") + Text("Tracking Status") .font(headerFont) - Spacer() - Button(action: { - eyeTrackingService.enableDebugLogging.toggle() - }) { - Image( - systemName: eyeTrackingService.enableDebugLogging - ? "ant.circle.fill" : "ant.circle" - ) - .foregroundStyle(eyeTrackingService.enableDebugLogging ? .orange : .secondary) - } - .buttonStyle(.plain) - .help("Toggle console debug logging") - - Button(showAdvancedSettings ? "Hide Settings" : "Show Settings") { - withAnimation { - showAdvancedSettings.toggle() - } - } - .buttonStyle(.bordered) - .controlSize(.small) } + let gazeState = eyeTrackingService.trackingResult.gazeState + let stateLabel: String = { + switch gazeState { + case .lookingAway: + return "Looking Away" + case .lookingAtScreen: + return "Looking At Screen" + case .unknown: + return "Unknown" + } + }() + VStack(alignment: .leading, spacing: 8) { - Text("Live Values:") - .font(.caption) - .fontWeight(.semibold) - .foregroundStyle(.secondary) - if let leftRatio = eyeTrackingService.debugLeftPupilRatio, - let rightRatio = eyeTrackingService.debugRightPupilRatio - { - HStack(spacing: 16) { - VStack(alignment: .leading, spacing: 2) { - Text("Left Pupil: \(String(format: "%.3f", leftRatio))") - .font(.caption2) - .foregroundStyle( - !EyeTrackingConstants.minPupilEnabled - && !EyeTrackingConstants.maxPupilEnabled - ? .secondary - : (leftRatio < EyeTrackingConstants.minPupilRatio - || leftRatio > EyeTrackingConstants.maxPupilRatio) - ? Color.orange : Color.green - ) - Text("Right Pupil: \(String(format: "%.3f", rightRatio))") - .font(.caption2) - .foregroundStyle( - !EyeTrackingConstants.minPupilEnabled - && !EyeTrackingConstants.maxPupilEnabled - ? .secondary - : (rightRatio < EyeTrackingConstants.minPupilRatio - || rightRatio > EyeTrackingConstants.maxPupilRatio) - ? Color.orange : Color.green - ) - } + HStack(spacing: 12) { + Text("Gaze:") + .font(.caption2) + .foregroundStyle(.secondary) + Text(stateLabel) + .font(.caption2) + .foregroundStyle(gazeState == .lookingAway ? .green : .secondary) + } - Spacer() - - VStack(alignment: .trailing, spacing: 2) { - Text( - "Range: \(String(format: "%.2f", EyeTrackingConstants.minPupilRatio)) - \(String(format: "%.2f", EyeTrackingConstants.maxPupilRatio))" - ) - .font(.caption2) - .foregroundStyle(.secondary) - let bothEyesOut = - (leftRatio < EyeTrackingConstants.minPupilRatio - || leftRatio > EyeTrackingConstants.maxPupilRatio) - && (rightRatio < EyeTrackingConstants.minPupilRatio - || rightRatio > EyeTrackingConstants.maxPupilRatio) - Text(bothEyesOut ? "Both Out ⚠️" : "In Range ✓") - .font(.caption2) - .foregroundStyle(bothEyesOut ? .orange : .green) - } - } - } else { - Text("Pupil data unavailable") + HStack(spacing: 12) { + Text("Confidence:") + .font(.caption2) + .foregroundStyle(.secondary) + Text(String(format: "%.2f", eyeTrackingService.trackingResult.confidence)) .font(.caption2) .foregroundStyle(.secondary) } - - if let yaw = eyeTrackingService.debugYaw, - let pitch = eyeTrackingService.debugPitch - { - HStack(spacing: 16) { - VStack(alignment: .leading, spacing: 2) { - Text("Yaw: \(String(format: "%.3f", yaw))") - .font(.caption2) - .foregroundStyle( - !EyeTrackingConstants.yawEnabled - ? .secondary - : abs(yaw) > EyeTrackingConstants.yawThreshold - ? Color.orange : Color.green - ) - Text("Pitch: \(String(format: "%.3f", pitch))") - .font(.caption2) - .foregroundStyle( - !EyeTrackingConstants.pitchUpEnabled - && !EyeTrackingConstants.pitchDownEnabled - ? .secondary - : (pitch > EyeTrackingConstants.pitchUpThreshold - || pitch < EyeTrackingConstants.pitchDownThreshold) - ? Color.orange : Color.green - ) - } - - Spacer() - - VStack(alignment: .trailing, spacing: 2) { - Text( - "Yaw Max: \(String(format: "%.2f", EyeTrackingConstants.yawThreshold))" - ) - .font(.caption2) - .foregroundStyle(.secondary) - Text( - "Pitch: \(String(format: "%.2f", EyeTrackingConstants.pitchDownThreshold)) to \(String(format: "%.2f", EyeTrackingConstants.pitchUpThreshold))" - ) - .font(.caption2) - .foregroundStyle(.secondary) - } - } - } - } - .padding(.top, 4) - - if showAdvancedSettings { - VStack(spacing: 16) { - VStack(alignment: .leading, spacing: 8) { - Text("Current Threshold Values:") - .font(.caption) - .fontWeight(.semibold) - .foregroundStyle(.secondary) - - HStack { - Text("Yaw Threshold:") - Spacer() - Text("\(String(format: "%.2f", EyeTrackingConstants.yawThreshold)) rad") - .foregroundStyle(.secondary) - } - - HStack { - Text("Pitch Up Threshold:") - Spacer() - Text( - "\(String(format: "%.2f", EyeTrackingConstants.pitchUpThreshold)) rad" - ) - .foregroundStyle(.secondary) - } - - HStack { - Text("Pitch Down Threshold:") - Spacer() - Text( - "\(String(format: "%.2f", EyeTrackingConstants.pitchDownThreshold)) rad" - ) - .foregroundStyle(.secondary) - } - - HStack { - Text("Min Pupil Ratio:") - Spacer() - Text("\(String(format: "%.2f", EyeTrackingConstants.minPupilRatio))") - .foregroundStyle(.secondary) - } - - HStack { - Text("Max Pupil Ratio:") - Spacer() - Text("\(String(format: "%.2f", EyeTrackingConstants.maxPupilRatio))") - .foregroundStyle(.secondary) - } - - HStack { - Text("Eye Closed Threshold:") - Spacer() - Text( - "\(String(format: "%.3f", EyeTrackingConstants.eyeClosedThreshold))" - ) - .foregroundStyle(.secondary) - } - } - .padding(.top, 8) - } - .padding(.top, 8) } } .padding(sectionPadding) diff --git a/Gaze/Views/Components/EyeTrackingCalibrationView.swift b/Gaze/Views/Components/EyeTrackingCalibrationView.swift deleted file mode 100644 index dd05830..0000000 --- a/Gaze/Views/Components/EyeTrackingCalibrationView.swift +++ /dev/null @@ -1,116 +0,0 @@ -// -// EyeTrackingCalibrationView.swift -// Gaze -// -// Created by Mike Freno on 1/15/26. -// - -import SwiftUI - -struct EyeTrackingCalibrationView: View { - @StateObject private var calibratorService = CalibratorService.shared - @Environment(\.dismiss) private var dismiss - - var body: some View { - ZStack { - Color.black.ignoresSafeArea() - introductionScreenView - } - .frame(minWidth: 600, minHeight: 500) - } - - // MARK: - Introduction Screen - - private var introductionScreenView: some View { - VStack(spacing: 30) { - Image(systemName: "eye.circle.fill") - .font(.system(size: 80)) - .foregroundStyle(.blue) - - Text("Eye Tracking Calibration") - .font(.largeTitle) - .foregroundStyle(.white) - .fontWeight(.bold) - - Text("This calibration will help improve eye tracking accuracy.") - .font(.title3) - .multilineTextAlignment(.center) - .foregroundStyle(.gray) - - VStack(alignment: .leading, spacing: 15) { - InstructionRow(icon: "1.circle.fill", text: "Look at each target on the screen") - InstructionRow( - icon: "2.circle.fill", text: "Keep your head still, only move your eyes") - InstructionRow(icon: "3.circle.fill", text: "Follow the countdown at each position") - InstructionRow(icon: "4.circle.fill", text: "Takes about 30-45 seconds") - } - .padding(.vertical, 20) - - if calibratorService.calibrationData.isComplete { - VStack(spacing: 10) { - Text("Last calibration:") - .font(.caption) - .foregroundStyle(.gray) - Text(calibratorService.getCalibrationSummary()) - .font(.caption) - .multilineTextAlignment(.center) - .foregroundStyle(.gray) - } - .padding(.vertical) - } - - HStack(spacing: 20) { - Button("Cancel") { - dismiss() - } - .foregroundStyle(.white) - .buttonStyle(.plain) - .keyboardShortcut(.escape, modifiers: []) - - Button("Start Calibration") { - startFullscreenCalibration() - } - .keyboardShortcut(.return, modifiers: []) - .buttonStyle(.borderedProminent) - } - .padding(.top, 20) - } - .padding(60) - .frame(maxWidth: 600) - } - - // MARK: - Actions - - private func startFullscreenCalibration() { - dismiss() - - // Small delay to allow sheet dismissal animation - DispatchQueue.main.asyncAfter(deadline: .now() + 0.3) { - CalibratorService.shared.showCalibrationOverlay() - } - } -} - -// MARK: - Instruction Row - -struct InstructionRow: View { - let icon: String - let text: String - - var body: some View { - HStack(spacing: 15) { - Image(systemName: icon) - .font(.title2) - .foregroundStyle(.blue) - .frame(width: 30) - - Text(text) - .foregroundStyle(.white) - .font(.body) - } - } -} - -#Preview { - EyeTrackingCalibrationView() -} diff --git a/Gaze/Views/Components/EyeTrackingDebugOverlayView.swift b/Gaze/Views/Components/EyeTrackingDebugOverlayView.swift new file mode 100644 index 0000000..2f0cfb4 --- /dev/null +++ b/Gaze/Views/Components/EyeTrackingDebugOverlayView.swift @@ -0,0 +1,96 @@ +// +// EyeTrackingDebugOverlayView.swift +// Gaze +// +// Created by Mike Freno on 1/31/26. +// + +import SwiftUI + +struct EyeTrackingDebugOverlayView: View { + let debugState: EyeTrackingDebugState + let viewSize: CGSize + + var body: some View { + ZStack { + if let leftRect = debugState.leftEyeRect, + let imageSize = debugState.imageSize { + drawEyeRect(leftRect, imageSize: imageSize, color: .cyan) + } + + if let rightRect = debugState.rightEyeRect, + let imageSize = debugState.imageSize { + drawEyeRect(rightRect, imageSize: imageSize, color: .yellow) + } + + if let leftPupil = debugState.leftPupil, + let imageSize = debugState.imageSize { + drawPupil(leftPupil, imageSize: imageSize, color: .red) + } + + if let rightPupil = debugState.rightPupil, + let imageSize = debugState.imageSize { + drawPupil(rightPupil, imageSize: imageSize, color: .red) + } + } + } + + private func drawEyeRect(_ rect: CGRect, imageSize: CGSize, color: Color) -> some View { + let mapped = mapRect(rect, imageSize: imageSize) + return Rectangle() + .stroke(color, lineWidth: 2) + .frame(width: mapped.size.width, height: mapped.size.height) + .position(x: mapped.midX, y: mapped.midY) + } + + private func drawPupil(_ point: CGPoint, imageSize: CGSize, color: Color) -> some View { + let mapped = mapPoint(point, imageSize: imageSize) + return Circle() + .fill(color) + .frame(width: 6, height: 6) + .position(x: mapped.x, y: mapped.y) + } + + private func mapRect(_ rect: CGRect, imageSize: CGSize) -> CGRect { + let mappedOrigin = mapPoint(rect.origin, imageSize: imageSize) + let mappedMax = mapPoint(CGPoint(x: rect.maxX, y: rect.maxY), imageSize: imageSize) + + let width = abs(mappedMax.x - mappedOrigin.x) + let height = abs(mappedMax.y - mappedOrigin.y) + + return CGRect( + x: min(mappedOrigin.x, mappedMax.x), + y: min(mappedOrigin.y, mappedMax.y), + width: width, + height: height + ) + } + + private func mapPoint(_ point: CGPoint, imageSize: CGSize) -> CGPoint { + let rawImageWidth = imageSize.width + let rawImageHeight = imageSize.height + + let imageAspect = rawImageWidth / rawImageHeight + let viewAspect = viewSize.width / viewSize.height + + let scale: CGFloat + let offsetX: CGFloat + let offsetY: CGFloat + + if imageAspect > viewAspect { + scale = viewSize.height / rawImageHeight + offsetX = (viewSize.width - rawImageWidth * scale) / 2 + offsetY = 0 + } else { + scale = viewSize.width / rawImageWidth + offsetX = 0 + offsetY = (viewSize.height - rawImageHeight * scale) / 2 + } + + let mirroredX = rawImageWidth - point.x + let screenX = mirroredX * scale + offsetX + let screenY = point.y * scale + offsetY + + return CGPoint(x: screenX, y: screenY) + } +} diff --git a/Gaze/Views/Components/GazeOverlayView.swift b/Gaze/Views/Components/GazeOverlayView.swift deleted file mode 100644 index f138cfb..0000000 --- a/Gaze/Views/Components/GazeOverlayView.swift +++ /dev/null @@ -1,253 +0,0 @@ -// -// GazeOverlayView.swift -// Gaze -// -// Created by Mike Freno on 1/16/26. -// - -import SwiftUI - -struct GazeOverlayView: View { - @ObservedObject var eyeTrackingService: EyeTrackingService - - var body: some View { - VStack(spacing: 8) { - inFrameIndicator - gazeDirectionGrid - ratioDebugView - eyeImagesDebugView - } - .padding(12) - } - - private var inFrameIndicator: some View { - HStack(spacing: 6) { - Circle() - .fill(eyeTrackingService.isInFrame ? Color.green : Color.red) - .frame(width: 10, height: 10) - Text(eyeTrackingService.isInFrame ? "In Frame" : "No Face") - .font(.caption2) - .fontWeight(.semibold) - .foregroundStyle(.white) - } - .padding(.horizontal, 10) - .padding(.vertical, 6) - .background( - Capsule() - .fill(Color.black.opacity(0.6)) - ) - } - - private var gazeDirectionGrid: some View { - let currentDirection = eyeTrackingService.gazeDirection - let currentPos = currentDirection.gridPosition - - return VStack(spacing: 2) { - ForEach(0..<3, id: \.self) { row in - HStack(spacing: 2) { - ForEach(0..<3, id: \.self) { col in - let isActive = - currentPos.x == col && currentPos.y == row - && eyeTrackingService.isInFrame - gridCell(row: row, col: col, isActive: isActive) - } - } - } - } - .padding(8) - .background( - RoundedRectangle(cornerRadius: 8) - .fill(Color.black.opacity(0.5)) - ) - } - - private func gridCell(row: Int, col: Int, isActive: Bool) -> some View { - let direction = directionFor(row: row, col: col) - - return ZStack { - RoundedRectangle(cornerRadius: 4) - .fill(isActive ? Color.green : Color.white.opacity(0.2)) - - Text(direction.rawValue) - .font(.system(size: 14, weight: .bold)) - .foregroundStyle(isActive ? .white : .white.opacity(0.6)) - } - .frame(width: 28, height: 28) - } - - private func directionFor(row: Int, col: Int) -> GazeDirection { - switch (col, row) { - case (0, 0): return .upLeft - case (1, 0): return .up - case (2, 0): return .upRight - case (0, 1): return .left - case (1, 1): return .center - case (2, 1): return .right - case (0, 2): return .downLeft - case (1, 2): return .down - case (2, 2): return .downRight - default: return .center - } - } - - private var ratioDebugView: some View { - VStack(alignment: .leading, spacing: 2) { - // Show individual L/R ratios - HStack(spacing: 8) { - if let leftH = eyeTrackingService.debugLeftPupilRatio { - Text("L.H: \(String(format: "%.2f", leftH))") - .font(.system(size: 9, weight: .medium, design: .monospaced)) - .foregroundStyle(.white) - } - if let rightH = eyeTrackingService.debugRightPupilRatio { - Text("R.H: \(String(format: "%.2f", rightH))") - .font(.system(size: 9, weight: .medium, design: .monospaced)) - .foregroundStyle(.white) - } - } - - HStack(spacing: 8) { - if let leftV = eyeTrackingService.debugLeftVerticalRatio { - Text("L.V: \(String(format: "%.2f", leftV))") - .font(.system(size: 9, weight: .medium, design: .monospaced)) - .foregroundStyle(.white) - } - if let rightV = eyeTrackingService.debugRightVerticalRatio { - Text("R.V: \(String(format: "%.2f", rightV))") - .font(.system(size: 9, weight: .medium, design: .monospaced)) - .foregroundStyle(.white) - } - } - - // Show averaged ratios - if let leftH = eyeTrackingService.debugLeftPupilRatio, - let rightH = eyeTrackingService.debugRightPupilRatio, - let leftV = eyeTrackingService.debugLeftVerticalRatio, - let rightV = eyeTrackingService.debugRightVerticalRatio - { - let avgH = (leftH + rightH) / 2.0 - let avgV = (leftV + rightV) / 2.0 - Text("Avg H:\(String(format: "%.2f", avgH)) V:\(String(format: "%.2f", avgV))") - .font(.system(size: 9, weight: .bold, design: .monospaced)) - .foregroundStyle(.yellow) - } - } - .padding(.horizontal, 8) - .padding(.vertical, 4) - .background( - RoundedRectangle(cornerRadius: 4) - .fill(Color.black.opacity(0.5)) - ) - } - - private var eyeImagesDebugView: some View { - HStack(spacing: 12) { - // Left eye - VStack(spacing: 4) { - Text("Left") - .font(.system(size: 8, weight: .bold)) - .foregroundStyle(.white) - - HStack(spacing: 4) { - eyeImageView( - image: eyeTrackingService.debugLeftEyeInput, - pupilPosition: eyeTrackingService.debugLeftPupilPosition, - eyeSize: eyeTrackingService.debugLeftEyeSize, - label: "Input" - ) - eyeImageView( - image: eyeTrackingService.debugLeftEyeProcessed, - pupilPosition: eyeTrackingService.debugLeftPupilPosition, - eyeSize: eyeTrackingService.debugLeftEyeSize, - label: "Proc" - ) - } - } - - // Right eye - VStack(spacing: 4) { - Text("Right") - .font(.system(size: 8, weight: .bold)) - .foregroundStyle(.white) - - HStack(spacing: 4) { - eyeImageView( - image: eyeTrackingService.debugRightEyeInput, - pupilPosition: eyeTrackingService.debugRightPupilPosition, - eyeSize: eyeTrackingService.debugRightEyeSize, - label: "Input" - ) - eyeImageView( - image: eyeTrackingService.debugRightEyeProcessed, - pupilPosition: eyeTrackingService.debugRightPupilPosition, - eyeSize: eyeTrackingService.debugRightEyeSize, - label: "Proc" - ) - } - } - } - .padding(8) - .background( - RoundedRectangle(cornerRadius: 8) - .fill(Color.black.opacity(0.5)) - ) - } - - private func eyeImageView( - image: NSImage?, pupilPosition: PupilPosition?, eyeSize: CGSize?, label: String - ) -> some View { - let displaySize: CGFloat = 50 - - return VStack(spacing: 2) { - ZStack { - if let nsImage = image { - Image(nsImage: nsImage) - .resizable() - .interpolation(.none) - .aspectRatio(contentMode: .fit) - .frame(width: displaySize, height: displaySize) - - // Draw pupil position marker - if let pupil = pupilPosition, let size = eyeSize, size.width > 0, - size.height > 0 - { - let scaleX = displaySize / size.width - let scaleY = displaySize / size.height - let scale = min(scaleX, scaleY) - let scaledWidth = size.width * scale - let scaledHeight = size.height * scale - - Circle() - .fill(Color.red) - .frame(width: 4, height: 4) - .offset( - x: (pupil.x * scale) - (scaledWidth / 2), - y: (pupil.y * scale) - (scaledHeight / 2) - ) - } - } else { - RoundedRectangle(cornerRadius: 4) - .fill(Color.gray.opacity(0.3)) - .frame(width: displaySize, height: displaySize) - Text("--") - .font(.system(size: 10)) - .foregroundStyle(.white.opacity(0.5)) - } - } - .frame(width: displaySize, height: displaySize) - .clipShape(RoundedRectangle(cornerRadius: 4)) - - Text(label) - .font(.system(size: 7)) - .foregroundStyle(.white.opacity(0.7)) - } - } -} - -#Preview { - ZStack { - Color.gray - GazeOverlayView(eyeTrackingService: EyeTrackingService.shared) - } - .frame(width: 400, height: 400) -} diff --git a/Gaze/Views/Components/PupilOverlayView.swift b/Gaze/Views/Components/PupilOverlayView.swift deleted file mode 100644 index 4c0bd29..0000000 --- a/Gaze/Views/Components/PupilOverlayView.swift +++ /dev/null @@ -1,184 +0,0 @@ -// -// PupilOverlayView.swift -// Gaze -// -// Created by Mike Freno on 1/16/26. -// - -import SwiftUI - -/// Draws pupil detection markers directly on top of the camera preview -struct PupilOverlayView: View { - @ObservedObject var eyeTrackingService: EyeTrackingService - - var body: some View { - GeometryReader { geometry in - let viewSize = geometry.size - - // Draw eye regions and pupil markers - ZStack { - // Left eye - if let leftRegion = eyeTrackingService.debugLeftEyeRegion, - let leftPupil = eyeTrackingService.debugLeftPupilPosition, - let imageSize = eyeTrackingService.debugImageSize - { - EyeOverlayShape( - eyeRegion: leftRegion, - pupilPosition: leftPupil, - imageSize: imageSize, - viewSize: viewSize, - color: .cyan, - label: "L" - ) - } - - // Right eye - if let rightRegion = eyeTrackingService.debugRightEyeRegion, - let rightPupil = eyeTrackingService.debugRightPupilPosition, - let imageSize = eyeTrackingService.debugImageSize - { - EyeOverlayShape( - eyeRegion: rightRegion, - pupilPosition: rightPupil, - imageSize: imageSize, - viewSize: viewSize, - color: .yellow, - label: "R" - ) - } - } - } - } -} - -/// Helper view for drawing eye overlay -private struct EyeOverlayShape: View { - let eyeRegion: EyeRegion - let pupilPosition: PupilPosition - let imageSize: CGSize - let viewSize: CGSize - let color: Color - let label: String - - private var transformedCoordinates: (eyeRect: CGRect, pupilPoint: CGPoint) { - // Standard macOS Camera Coordinate System (Landscape): - // Raw Buffer: - // - Origin (0,0) is Top-Left - // - X increases Right - // - Y increases Down - // - // Preview Layer (Mirrored): - // - Appears like a mirror - // - Screen X increases Right - // - Screen Y increases Down - // - BUT the image content is flipped horizontally - // (Raw Left is Screen Right, Raw Right is Screen Left) - - // Use dimensions directly (no rotation swap) - let rawImageWidth = imageSize.width - let rawImageHeight = imageSize.height - - // Calculate aspect-fill scaling - // We compare the raw aspect ratio to the view aspect ratio - let imageAspect = rawImageWidth / rawImageHeight - let viewAspect = viewSize.width / viewSize.height - - let scale: CGFloat - let offsetX: CGFloat - let offsetY: CGFloat - - if imageAspect > viewAspect { - // Image is wider than view - crop sides (pillarbox behavior in aspect fill) - // Wait, aspect fill means we fill the view, so we crop the excess. - // If image is wider, we scale by height to fill height, and crop width. - scale = viewSize.height / rawImageHeight - offsetX = (viewSize.width - rawImageWidth * scale) / 2 - offsetY = 0 - } else { - // Image is taller than view (or view is wider) - scale by width, crop height - scale = viewSize.width / rawImageWidth - offsetX = 0 - offsetY = (viewSize.height - rawImageHeight * scale) / 2 - } - - // Transform Eye Region - // Mirroring X: The 'left' of the raw image becomes the 'right' of the screen - // Raw Rect: x, y, w, h - // Mirrored X = ImageWidth - (x + w) - let eyeRawX = eyeRegion.frame.origin.x - let eyeRawY = eyeRegion.frame.origin.y - let eyeRawW = eyeRegion.frame.width - let eyeRawH = eyeRegion.frame.height - - // Calculate Screen Coordinates - let eyeScreenX = (rawImageWidth - (eyeRawX + eyeRawW)) * scale + offsetX - let eyeScreenY = eyeRawY * scale + offsetY - let eyeScreenW = eyeRawW * scale - let eyeScreenH = eyeRawH * scale - - // Transform Pupil Position - // Global Raw Pupil X = eyeRawX + pupilPosition.x - // Global Raw Pupil Y = eyeRawY + pupilPosition.y - let pupilGlobalRawX = eyeRawX + pupilPosition.x - let pupilGlobalRawY = eyeRawY + pupilPosition.y - - // Mirror X for Pupil - let pupilScreenX = (rawImageWidth - pupilGlobalRawX) * scale + offsetX - let pupilScreenY = pupilGlobalRawY * scale + offsetY - - return ( - eyeRect: CGRect(x: eyeScreenX, y: eyeScreenY, width: eyeScreenW, height: eyeScreenH), - pupilPoint: CGPoint(x: pupilScreenX, y: pupilScreenY) - ) - } - - var body: some View { - let coords = transformedCoordinates - let eyeRect = coords.eyeRect - let pupilPoint = coords.pupilPoint - - ZStack { - // Eye region rectangle - Rectangle() - .stroke(color, lineWidth: 2) - .frame(width: eyeRect.width, height: eyeRect.height) - .position(x: eyeRect.midX, y: eyeRect.midY) - - // Pupil marker (red dot) - Circle() - .fill(Color.red) - .frame(width: 8, height: 8) - .position(x: pupilPoint.x, y: pupilPoint.y) - - // Crosshair at pupil position - Path { path in - path.move(to: CGPoint(x: pupilPoint.x - 6, y: pupilPoint.y)) - path.addLine(to: CGPoint(x: pupilPoint.x + 6, y: pupilPoint.y)) - path.move(to: CGPoint(x: pupilPoint.x, y: pupilPoint.y - 6)) - path.addLine(to: CGPoint(x: pupilPoint.x, y: pupilPoint.y + 6)) - } - .stroke(Color.red, lineWidth: 1) - - // Label - Text(label) - .font(.system(size: 10, weight: .bold)) - .foregroundStyle(color) - .position(x: eyeRect.minX + 8, y: eyeRect.minY - 8) - - // Debug: Show raw coordinates - Text("\(label): (\(Int(pupilPosition.x)), \(Int(pupilPosition.y)))") - .font(.system(size: 8, design: .monospaced)) - .foregroundStyle(.white) - .background(.black.opacity(0.7)) - .position(x: eyeRect.midX, y: eyeRect.maxY + 10) - } - } -} - -#Preview { - ZStack { - Color.black - PupilOverlayView(eyeTrackingService: EyeTrackingService.shared) - } - .frame(width: 400, height: 300) -} diff --git a/Gaze/Views/Containers/AdditionalModifiersView.swift b/Gaze/Views/Containers/AdditionalModifiersView.swift index f2c3d84..c5f753d 100644 --- a/Gaze/Views/Containers/AdditionalModifiersView.swift +++ b/Gaze/Views/Containers/AdditionalModifiersView.swift @@ -15,9 +15,6 @@ struct AdditionalModifiersView: View { @State private var isDragging: Bool = false @State private var isTestModeActive = false @State private var cachedPreviewLayer: AVCaptureVideoPreviewLayer? - @State private var showAdvancedSettings = false - @State private var showCalibrationWindow = false - @State private var isViewActive = false @State private var isProcessingToggle = false @ObservedObject var cameraService = CameraAccessService.shared @Environment(\.isCompactLayout) private var isCompact @@ -66,9 +63,6 @@ struct AdditionalModifiersView: View { presentation: .card, isTestModeActive: $isTestModeActive, cachedPreviewLayer: $cachedPreviewLayer, - showAdvancedSettings: $showAdvancedSettings, - showCalibrationWindow: $showCalibrationWindow, - isViewActive: $isViewActive, isProcessingToggle: isProcessingToggle, handleEnforceModeToggle: { enabled in if enabled { diff --git a/Gaze/Views/Setup/EnforceModeSetupView.swift b/Gaze/Views/Setup/EnforceModeSetupView.swift index 5370264..362ae43 100644 --- a/Gaze/Views/Setup/EnforceModeSetupView.swift +++ b/Gaze/Views/Setup/EnforceModeSetupView.swift @@ -16,10 +16,6 @@ struct EnforceModeSetupView: View { @State private var isProcessingToggle = false @State private var isTestModeActive = false @State private var cachedPreviewLayer: AVCaptureVideoPreviewLayer? - @State private var showDebugView = false - @State private var isViewActive = false - @State private var showAdvancedSettings = false - @State private var showCalibrationWindow = false private var cameraHardwareAvailable: Bool { cameraService.hasCameraHardware @@ -34,9 +30,6 @@ struct EnforceModeSetupView: View { presentation: .window, isTestModeActive: $isTestModeActive, cachedPreviewLayer: $cachedPreviewLayer, - showAdvancedSettings: $showAdvancedSettings, - showCalibrationWindow: $showCalibrationWindow, - isViewActive: $isViewActive, isProcessingToggle: isProcessingToggle, handleEnforceModeToggle: { enabled in print("🎛️ Toggle changed to: \(enabled)") @@ -54,12 +47,7 @@ struct EnforceModeSetupView: View { .frame(maxWidth: .infinity, maxHeight: .infinity) .padding() .background(.clear) - .onAppear { - isViewActive = true - } .onDisappear { - isViewActive = false - // If the view disappeared and camera is still active, stop it if enforceModeService.isCameraActive { print("👁️ EnforceModeSetupView disappeared, stopping camera preview") enforceModeService.stopCamera() diff --git a/GazeTests/Services/PupilDetectorTests.swift b/GazeTests/Services/PupilDetectorTests.swift deleted file mode 100644 index 7cee871..0000000 --- a/GazeTests/Services/PupilDetectorTests.swift +++ /dev/null @@ -1,177 +0,0 @@ -// -// PupilDetectorTests.swift -// GazeTests -// -// Created by Mike Freno on 1/16/26. -// - -import CoreVideo -import Vision -import XCTest - -@testable import Gaze - -final class PupilDetectorTests: XCTestCase { - - override func setUp() async throws { - // Reset the detector state - PupilDetector.cleanup() - } - - func testCreateCGImageFromData() throws { - // Test basic image creation - let width = 50 - let height = 50 - var pixels = [UInt8](repeating: 128, count: width * height) - - // Add some dark pixels for a "pupil" - for y in 20..<30 { - for x in 20..<30 { - pixels[y * width + x] = 10 // Very dark - } - } - - // Save test image to verify - let pixelData = Data(pixels) - guard let provider = CGDataProvider(data: pixelData as CFData) else { - XCTFail("Failed to create CGDataProvider") - return - } - - let cgImage = CGImage( - width: width, - height: height, - bitsPerComponent: 8, - bitsPerPixel: 8, - bytesPerRow: width, - space: CGColorSpaceCreateDeviceGray(), - bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.none.rawValue), - provider: provider, - decode: nil, - shouldInterpolate: false, - intent: .defaultIntent - ) - - XCTAssertNotNil(cgImage, "Should create CGImage from pixel data") - } - - func testImageProcessingWithDarkPixels() throws { - // Test that imageProcessingOptimized produces dark pixels - let width = 60 - let height = 40 - - // Create input with a dark circle (simulating pupil) - var input = [UInt8](repeating: 200, count: width * height) // Light background (like eye white) - - // Add a dark ellipse in center (pupil) - let centerX = width / 2 - let centerY = height / 2 - for y in 0..50%)" - ) - log( - " H-range: \(String(format: "%.3f", stats.minH)) to \(String(format: "%.3f", stats.maxH))" - ) - log( - " V-range: \(String(format: "%.3f", stats.minV)) to \(String(format: "%.3f", stats.maxV))" - ) - log( - " Face width: \(String(format: "%.3f", stats.avgFaceWidth)) (range: \(String(format: "%.3f", stats.minFaceWidth))-\(String(format: "%.3f", stats.maxFaceWidth)))" - ) - - attachLogs() - - // At least 50% should be detected as non-center when looking away - XCTAssertGreaterThan( - nonCenterRatio, 0.5, - "Looking away video should have >50% non-center detections. Log:\n\(logLines.joined(separator: "\n"))" - ) - } - - /// Process the inner video (looking at screen) - should detect "looking at screen" - func testInnerVideoGazeDetection() async throws { - logLines = [] - - let projectPath = "/Users/mike/Code/Gaze/GazeTests/video-test-inner.mp4" - guard FileManager.default.fileExists(atPath: projectPath) else { - XCTFail("Video file not found at: \(projectPath)") - return - } - let stats = try await processVideo( - at: URL(fileURLWithPath: projectPath), expectLookingAway: false) - - // For inner video, most frames should detect gaze at center - let centerRatio = Double(stats.centerFrames) / Double(max(1, stats.pupilDetectedFrames)) - log( - "🎯 INNER video: \(String(format: "%.1f%%", centerRatio * 100)) frames detected as center (expected: >50%)" - ) - log( - " H-range: \(String(format: "%.3f", stats.minH)) to \(String(format: "%.3f", stats.maxH))" - ) - log( - " V-range: \(String(format: "%.3f", stats.minV)) to \(String(format: "%.3f", stats.maxV))" - ) - log( - " Face width: \(String(format: "%.3f", stats.avgFaceWidth)) (range: \(String(format: "%.3f", stats.minFaceWidth))-\(String(format: "%.3f", stats.maxFaceWidth)))" - ) - - attachLogs() - - // At least 50% should be detected as center when looking at screen - XCTAssertGreaterThan( - centerRatio, 0.5, - "Looking at screen video should have >50% center detections. Log:\n\(logLines.joined(separator: "\n"))" - ) - } - - struct VideoStats { - var totalFrames = 0 - var faceDetectedFrames = 0 - var pupilDetectedFrames = 0 - var centerFrames = 0 - var nonCenterFrames = 0 - var minH = Double.greatestFiniteMagnitude - var maxH = -Double.greatestFiniteMagnitude - var minV = Double.greatestFiniteMagnitude - var maxV = -Double.greatestFiniteMagnitude - var minFaceWidth = Double.greatestFiniteMagnitude - var maxFaceWidth = -Double.greatestFiniteMagnitude - var totalFaceWidth = 0.0 - var faceWidthCount = 0 - - var avgFaceWidth: Double { - faceWidthCount > 0 ? totalFaceWidth / Double(faceWidthCount) : 0 - } - } - - private func processVideo(at url: URL, expectLookingAway: Bool) async throws -> VideoStats { - var stats = VideoStats() - - log("\n" + String(repeating: "=", count: 60)) - log("Processing video: \(url.lastPathComponent)") - log( - "Expected behavior: \(expectLookingAway ? "LOOKING AWAY (non-center)" : "LOOKING AT SCREEN (center)")" - ) - log(String(repeating: "=", count: 60)) - - let asset = AVURLAsset(url: url) - let duration = try await asset.load(.duration) - let durationSeconds = CMTimeGetSeconds(duration) - log("Duration: \(String(format: "%.2f", durationSeconds)) seconds") - - guard let track = try await asset.loadTracks(withMediaType: .video).first else { - XCTFail("No video track found") - return stats - } - - let size = try await track.load(.naturalSize) - let frameRate = try await track.load(.nominalFrameRate) - log( - "Size: \(Int(size.width))x\(Int(size.height)), FPS: \(String(format: "%.1f", frameRate))" - ) - - let reader = try AVAssetReader(asset: asset) - let outputSettings: [String: Any] = [ - kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA - ] - let trackOutput = AVAssetReaderTrackOutput(track: track, outputSettings: outputSettings) - reader.add(trackOutput) - reader.startReading() - - var frameIndex = 0 - let sampleInterval = max(1, Int(frameRate / 2)) // Sample ~2 frames per second - - log("\nFrame | Time | Face | H-Ratio L/R | V-Ratio L/R | Direction") - log(String(repeating: "-", count: 75)) - - // Reset calibration for fresh test - PupilDetector.calibration.reset() - - // Disable frame skipping for video testing - let originalFrameSkip = PupilDetector.frameSkipCount - PupilDetector.frameSkipCount = 1 - defer { PupilDetector.frameSkipCount = originalFrameSkip } - - while let sampleBuffer = trackOutput.copyNextSampleBuffer() { - defer { - frameIndex += 1 - PupilDetector.advanceFrame() - } - - // Only process every Nth frame - if frameIndex % sampleInterval != 0 { - continue - } - - stats.totalFrames += 1 - - guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { - continue - } - - let timestamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) - let timeSeconds = CMTimeGetSeconds(timestamp) - - // Run face detection - let request = VNDetectFaceLandmarksRequest() - request.revision = VNDetectFaceLandmarksRequestRevision3 - - let handler = VNImageRequestHandler( - cvPixelBuffer: pixelBuffer, - orientation: .leftMirrored, - options: [:] - ) - - try handler.perform([request]) - - guard let observations = request.results, !observations.isEmpty, - let face = observations.first, - let landmarks = face.landmarks, - let leftEye = landmarks.leftEye, - let rightEye = landmarks.rightEye - else { - log( - String( - format: "%5d | %5.1fs | NO | - | - | -", - frameIndex, timeSeconds)) - continue - } - - stats.faceDetectedFrames += 1 - - // Track face width (bounding box width as ratio of image width) - let faceWidth = face.boundingBox.width - stats.minFaceWidth = min(stats.minFaceWidth, faceWidth) - stats.maxFaceWidth = max(stats.maxFaceWidth, faceWidth) - stats.totalFaceWidth += faceWidth - stats.faceWidthCount += 1 - - let imageSize = CGSize( - width: CVPixelBufferGetWidth(pixelBuffer), - height: CVPixelBufferGetHeight(pixelBuffer) - ) - - // Detect pupils - var leftHRatio: Double? - var rightHRatio: Double? - var leftVRatio: Double? - var rightVRatio: Double? - - if let leftResult = PupilDetector.detectPupil( - in: pixelBuffer, - eyeLandmarks: leftEye, - faceBoundingBox: face.boundingBox, - imageSize: imageSize, - side: 0 - ) { - leftHRatio = calculateHorizontalRatio( - pupilPosition: leftResult.pupilPosition, eyeRegion: leftResult.eyeRegion) - leftVRatio = calculateVerticalRatio( - pupilPosition: leftResult.pupilPosition, eyeRegion: leftResult.eyeRegion) - } - - if let rightResult = PupilDetector.detectPupil( - in: pixelBuffer, - eyeLandmarks: rightEye, - faceBoundingBox: face.boundingBox, - imageSize: imageSize, - side: 1 - ) { - rightHRatio = calculateHorizontalRatio( - pupilPosition: rightResult.pupilPosition, eyeRegion: rightResult.eyeRegion) - rightVRatio = calculateVerticalRatio( - pupilPosition: rightResult.pupilPosition, eyeRegion: rightResult.eyeRegion) - } - - if let lh = leftHRatio, let rh = rightHRatio, - let lv = leftVRatio, let rv = rightVRatio - { - stats.pupilDetectedFrames += 1 - let avgH = (lh + rh) / 2.0 - let avgV = (lv + rv) / 2.0 - - // Track min/max ranges - stats.minH = min(stats.minH, avgH) - stats.maxH = max(stats.maxH, avgH) - stats.minV = min(stats.minV, avgV) - stats.maxV = max(stats.maxV, avgV) - - let direction = GazeDirection.from(horizontal: avgH, vertical: avgV) - if direction == .center { - stats.centerFrames += 1 - } else { - stats.nonCenterFrames += 1 - } - log( - String( - format: "%5d | %5.1fs | YES | %.2f / %.2f | %.2f / %.2f | %@ %@", - frameIndex, timeSeconds, lh, rh, lv, rv, direction.rawValue, - String(describing: direction))) - } else { - log( - String( - format: "%5d | %5.1fs | YES | PUPIL FAIL | PUPIL FAIL | -", - frameIndex, timeSeconds)) - } - } - - log(String(repeating: "=", count: 75)) - log( - "Summary: \(stats.totalFrames) frames sampled, \(stats.faceDetectedFrames) with face, \(stats.pupilDetectedFrames) with pupils" - ) - log("Center frames: \(stats.centerFrames), Non-center: \(stats.nonCenterFrames)") - log( - "Face width: avg=\(String(format: "%.3f", stats.avgFaceWidth)), range=\(String(format: "%.3f", stats.minFaceWidth)) to \(String(format: "%.3f", stats.maxFaceWidth))" - ) - log("Processing complete\n") - - return stats - } - - private func calculateHorizontalRatio(pupilPosition: PupilPosition, eyeRegion: EyeRegion) - -> Double - { - // pupilPosition.y controls horizontal gaze due to image orientation - let pupilY = Double(pupilPosition.y) - let eyeHeight = Double(eyeRegion.frame.height) - - guard eyeHeight > 0 else { return 0.5 } - - let ratio = pupilY / eyeHeight - return max(0.0, min(1.0, ratio)) - } - - private func calculateVerticalRatio(pupilPosition: PupilPosition, eyeRegion: EyeRegion) - -> Double - { - // pupilPosition.x controls vertical gaze due to image orientation - let pupilX = Double(pupilPosition.x) - let eyeWidth = Double(eyeRegion.frame.width) - - guard eyeWidth > 0 else { return 0.5 } - - let ratio = pupilX / eyeWidth - return max(0.0, min(1.0, ratio)) - } -}