oh baby boy what is happening
This commit is contained in:
Michael Freno
205
GazeTests/VideoGazeTests.swift
Normal file
205
GazeTests/VideoGazeTests.swift
Normal file
@@ -0,0 +1,205 @@
|
||||
//
|
||||
// VideoGazeTests.swift
|
||||
// GazeTests
|
||||
//
|
||||
// Created by Claude on 1/16/26.
|
||||
//
|
||||
|
||||
import XCTest
|
||||
import AVFoundation
|
||||
import Vision
|
||||
@testable import Gaze
|
||||
|
||||
final class VideoGazeTests: XCTestCase {
|
||||
|
||||
var logLines: [String] = []
|
||||
|
||||
private func log(_ message: String) {
|
||||
logLines.append(message)
|
||||
}
|
||||
|
||||
/// Process the outer video and log gaze detection results
|
||||
func testOuterVideoGazeDetection() async throws {
|
||||
logLines = []
|
||||
|
||||
let projectPath = "/Users/mike/Code/Gaze/GazeTests/video-test-outer.mp4"
|
||||
guard FileManager.default.fileExists(atPath: projectPath) else {
|
||||
XCTFail("Video file not found at: \(projectPath)")
|
||||
return
|
||||
}
|
||||
try await processVideo(at: URL(fileURLWithPath: projectPath))
|
||||
}
|
||||
|
||||
/// Process the inner video and log gaze detection results
|
||||
func testInnerVideoGazeDetection() async throws {
|
||||
logLines = []
|
||||
|
||||
let projectPath = "/Users/mike/Code/Gaze/GazeTests/video-test-inner.mp4"
|
||||
guard FileManager.default.fileExists(atPath: projectPath) else {
|
||||
XCTFail("Video file not found at: \(projectPath)")
|
||||
return
|
||||
}
|
||||
try await processVideo(at: URL(fileURLWithPath: projectPath))
|
||||
}
|
||||
|
||||
private func processVideo(at url: URL) async throws {
|
||||
log("\n" + String(repeating: "=", count: 60))
|
||||
log("Processing video: \(url.lastPathComponent)")
|
||||
log(String(repeating: "=", count: 60))
|
||||
|
||||
let asset = AVURLAsset(url: url)
|
||||
let duration = try await asset.load(.duration)
|
||||
let durationSeconds = CMTimeGetSeconds(duration)
|
||||
log("Duration: \(String(format: "%.2f", durationSeconds)) seconds")
|
||||
|
||||
guard let track = try await asset.loadTracks(withMediaType: .video).first else {
|
||||
XCTFail("No video track found")
|
||||
return
|
||||
}
|
||||
|
||||
let size = try await track.load(.naturalSize)
|
||||
let frameRate = try await track.load(.nominalFrameRate)
|
||||
log("Size: \(Int(size.width))x\(Int(size.height)), FPS: \(String(format: "%.1f", frameRate))")
|
||||
|
||||
let reader = try AVAssetReader(asset: asset)
|
||||
let outputSettings: [String: Any] = [
|
||||
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA
|
||||
]
|
||||
let trackOutput = AVAssetReaderTrackOutput(track: track, outputSettings: outputSettings)
|
||||
reader.add(trackOutput)
|
||||
reader.startReading()
|
||||
|
||||
var frameIndex = 0
|
||||
let sampleInterval = max(1, Int(frameRate / 2)) // Sample ~2 frames per second
|
||||
|
||||
log("\nFrame | Time | Face | H-Ratio L/R | V-Ratio L/R | Direction")
|
||||
log(String(repeating: "-", count: 75))
|
||||
|
||||
// Reset calibration for fresh test
|
||||
PupilDetector.calibration.reset()
|
||||
|
||||
// Disable frame skipping for video testing
|
||||
let originalFrameSkip = PupilDetector.frameSkipCount
|
||||
PupilDetector.frameSkipCount = 1
|
||||
defer { PupilDetector.frameSkipCount = originalFrameSkip }
|
||||
|
||||
var totalFrames = 0
|
||||
var faceDetectedFrames = 0
|
||||
var pupilDetectedFrames = 0
|
||||
|
||||
while let sampleBuffer = trackOutput.copyNextSampleBuffer() {
|
||||
defer {
|
||||
frameIndex += 1
|
||||
PupilDetector.advanceFrame()
|
||||
}
|
||||
|
||||
// Only process every Nth frame
|
||||
if frameIndex % sampleInterval != 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
totalFrames += 1
|
||||
|
||||
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
|
||||
continue
|
||||
}
|
||||
|
||||
let timestamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
|
||||
let timeSeconds = CMTimeGetSeconds(timestamp)
|
||||
|
||||
// Run face detection
|
||||
let request = VNDetectFaceLandmarksRequest()
|
||||
request.revision = VNDetectFaceLandmarksRequestRevision3
|
||||
|
||||
let handler = VNImageRequestHandler(
|
||||
cvPixelBuffer: pixelBuffer,
|
||||
orientation: .leftMirrored,
|
||||
options: [:]
|
||||
)
|
||||
|
||||
try handler.perform([request])
|
||||
|
||||
guard let observations = request.results, !observations.isEmpty,
|
||||
let face = observations.first,
|
||||
let landmarks = face.landmarks,
|
||||
let leftEye = landmarks.leftEye,
|
||||
let rightEye = landmarks.rightEye else {
|
||||
log(String(format: "%5d | %5.1fs | NO | - | - | -", frameIndex, timeSeconds))
|
||||
continue
|
||||
}
|
||||
|
||||
faceDetectedFrames += 1
|
||||
|
||||
let imageSize = CGSize(
|
||||
width: CVPixelBufferGetWidth(pixelBuffer),
|
||||
height: CVPixelBufferGetHeight(pixelBuffer)
|
||||
)
|
||||
|
||||
// Detect pupils
|
||||
var leftHRatio: Double?
|
||||
var rightHRatio: Double?
|
||||
var leftVRatio: Double?
|
||||
var rightVRatio: Double?
|
||||
|
||||
if let leftResult = PupilDetector.detectPupil(
|
||||
in: pixelBuffer,
|
||||
eyeLandmarks: leftEye,
|
||||
faceBoundingBox: face.boundingBox,
|
||||
imageSize: imageSize,
|
||||
side: 0
|
||||
) {
|
||||
leftHRatio = calculateHorizontalRatio(pupilPosition: leftResult.pupilPosition, eyeRegion: leftResult.eyeRegion)
|
||||
leftVRatio = calculateVerticalRatio(pupilPosition: leftResult.pupilPosition, eyeRegion: leftResult.eyeRegion)
|
||||
}
|
||||
|
||||
if let rightResult = PupilDetector.detectPupil(
|
||||
in: pixelBuffer,
|
||||
eyeLandmarks: rightEye,
|
||||
faceBoundingBox: face.boundingBox,
|
||||
imageSize: imageSize,
|
||||
side: 1
|
||||
) {
|
||||
rightHRatio = calculateHorizontalRatio(pupilPosition: rightResult.pupilPosition, eyeRegion: rightResult.eyeRegion)
|
||||
rightVRatio = calculateVerticalRatio(pupilPosition: rightResult.pupilPosition, eyeRegion: rightResult.eyeRegion)
|
||||
}
|
||||
|
||||
if let lh = leftHRatio, let rh = rightHRatio,
|
||||
let lv = leftVRatio, let rv = rightVRatio {
|
||||
pupilDetectedFrames += 1
|
||||
let avgH = (lh + rh) / 2.0
|
||||
let avgV = (lv + rv) / 2.0
|
||||
let direction = GazeDirection.from(horizontal: avgH, vertical: avgV)
|
||||
log(String(format: "%5d | %5.1fs | YES | %.2f / %.2f | %.2f / %.2f | %@ %@",
|
||||
frameIndex, timeSeconds, lh, rh, lv, rv, direction.rawValue, String(describing: direction)))
|
||||
} else {
|
||||
log(String(format: "%5d | %5.1fs | YES | PUPIL FAIL | PUPIL FAIL | -", frameIndex, timeSeconds))
|
||||
}
|
||||
}
|
||||
|
||||
log(String(repeating: "=", count: 75))
|
||||
log("Summary: \(totalFrames) frames sampled, \(faceDetectedFrames) with face, \(pupilDetectedFrames) with pupils")
|
||||
log("Processing complete\n")
|
||||
}
|
||||
|
||||
private func calculateHorizontalRatio(pupilPosition: PupilPosition, eyeRegion: EyeRegion) -> Double {
|
||||
// pupilPosition.y controls horizontal gaze due to image orientation
|
||||
let pupilY = Double(pupilPosition.y)
|
||||
let eyeHeight = Double(eyeRegion.frame.height)
|
||||
|
||||
guard eyeHeight > 0 else { return 0.5 }
|
||||
|
||||
let ratio = pupilY / eyeHeight
|
||||
return max(0.0, min(1.0, ratio))
|
||||
}
|
||||
|
||||
private func calculateVerticalRatio(pupilPosition: PupilPosition, eyeRegion: EyeRegion) -> Double {
|
||||
// pupilPosition.x controls vertical gaze due to image orientation
|
||||
let pupilX = Double(pupilPosition.x)
|
||||
let eyeWidth = Double(eyeRegion.frame.width)
|
||||
|
||||
guard eyeWidth > 0 else { return 0.5 }
|
||||
|
||||
let ratio = pupilX / eyeWidth
|
||||
return max(0.0, min(1.0, ratio))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user