18#import <AVFoundation/AVFoundation.h>
19#import <VideoToolbox/VideoToolbox.h>
21#include <winpr/assert.h>
26#define TAG CHANNELS_TAG("rdpecam-avfoundation.client")
28#define CAM_AVFOUNDATION_MEDIA_FORMAT CAM_MEDIA_FORMAT_NV12
32static BOOL cam_avfoundation_request_access(
void)
34 const AVAuthorizationStatus status =
35 [AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeVideo];
36 if (status == AVAuthorizationStatusAuthorized)
38 if (status == AVAuthorizationStatusNotDetermined)
41 [[NSBundle mainBundle] objectForInfoDictionaryKey:@"NSCameraUsageDescription"];
42 if (![usage isKindOfClass:[NSString class]] || (usage.length == 0))
46 "Camera access requires NSCameraUsageDescription in the application's Info.plist");
50 static atomic_bool pending;
51 bool expected =
false;
52 if (atomic_compare_exchange_strong(&pending, &expected,
true))
54 WLog_INFO(TAG,
"Requesting camera access");
55 [AVCaptureDevice requestAccessForMediaType:AVMediaTypeVideo
56 completionHandler:^(BOOL granted) {
58 WLog_WARN(TAG, "Camera access denied by user");
59 atomic_store(&pending, false);
64 const char *reason =
"unknown";
67 case AVAuthorizationStatusNotDetermined:
68 reason =
"not determined";
70 case AVAuthorizationStatusRestricted:
71 reason =
"restricted";
73 case AVAuthorizationStatusDenied:
76 case AVAuthorizationStatusAuthorized:
79 WLog_WARN(TAG,
"Camera access is %s", reason);
83#if TARGET_OS_OSX && (__MAC_OS_X_VERSION_MIN_REQUIRED < 140000)
86static AVCaptureDeviceType cam_avfoundation_legacy_external_type(
void)
87 API_DEPRECATED_WITH_REPLACEMENT(
"AVCaptureDeviceTypeExternal", macos(10.15, 14.0));
89static AVCaptureDeviceType cam_avfoundation_legacy_external_type(
void)
91 return AVCaptureDeviceTypeExternalUnknown;
96static BOOL cam_avfoundation_configure_camera(AVCaptureDevice *camera,
97 AVCaptureDeviceFormat *format, CMTime duration)
100 if (![camera lockForConfiguration:&error])
102 WLog_ERR(TAG,
"Unable to configure camera: %s", error.localizedDescription.UTF8String);
105 camera.activeFormat = format;
106 camera.activeVideoMinFrameDuration = duration;
107 camera.activeVideoMaxFrameDuration = duration;
108 [camera unlockForConfiguration];
112@interface FreeRDPCameraAVFoundationStream : NSObject <AVCaptureVideoDataOutputSampleBufferDelegate>
116 dispatch_queue_t _sampleQueue;
117 dispatch_queue_t _controlQueue;
119 ICamHalSampleCapturedCallback _callback;
121 CVPixelBufferRef _rawImage;
127 atomic_bool _drained;
129@property(nonatomic, retain) AVCaptureSession *session;
130@property(nonatomic, retain) AVCaptureVideoDataOutput *output;
134static void cam_avfoundation_deliver(FreeRDPCameraAVFoundationStream *stream,
const BYTE *data,
137 WINPR_ASSERT(stream);
139 if (!stream->_sampleLogged)
141 WLog_INFO(TAG,
"First camera sample: format %u, %" PRIu32
"x%" PRIu32
", %zu bytes",
142 CAM_AVFOUNDATION_MEDIA_FORMAT, stream->_width, stream->_height, length);
143 stream->_sampleLogged = TRUE;
145 const UINT rc = stream->_callback(stream->_device, stream->_streamIndex, data, length);
146 if (rc != CHANNEL_RC_OK)
148 atomic_store(&stream->_drained,
true);
149 if (rc != ECAM_SAMPLE_CAPTURE_DRAINED)
150 WLog_WARN(TAG,
"Sample callback failed: %" PRIu32, rc);
155static CAM_ERROR_CODE cam_avfoundation_stream_start(FreeRDPCameraAVFoundationStream *stream,
156 AVCaptureDevice *camera,
158 AVCaptureDeviceFormat *selected,
161 WINPR_ASSERT(stream);
162 WINPR_ASSERT(camera);
163 WINPR_ASSERT(mediaType);
164 WINPR_ASSERT(selected);
167 atomic_init(&stream->_drained,
true);
168 stream->_width = mediaType->Width;
169 stream->_height = mediaType->Height;
171 "Starting camera %s: format %u, %" PRIu32
"x%" PRIu32
", %" PRIu32
"/%" PRIu32
" fps",
172 camera.localizedName.UTF8String, CAM_AVFOUNDATION_MEDIA_FORMAT, stream->_width,
173 stream->_height, mediaType->FrameRateNumerator, mediaType->FrameRateDenominator);
174 stream->_sampleQueue =
175 dispatch_queue_create(
"org.freerdp.camera.samples", DISPATCH_QUEUE_SERIAL);
176 stream->_controlQueue =
177 dispatch_queue_create(
"org.freerdp.camera.control", DISPATCH_QUEUE_SERIAL);
178 stream.session = [[[AVCaptureSession alloc] init] autorelease];
179 stream.output = [[[AVCaptureVideoDataOutput alloc] init] autorelease];
180 if (!stream->_sampleQueue || !stream->_controlQueue || !stream.session || !stream.output)
181 return CAM_ERROR_CODE_OutOfMemory;
183 NSError *error = nil;
184 AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:camera error:&error];
187 WLog_ERR(TAG,
"Unable to open camera: %s", error.localizedDescription.UTF8String);
188 return CAM_ERROR_CODE_UnexpectedError;
190 [stream.session beginConfiguration];
192 stream.session.sessionPreset = AVCaptureSessionPresetInputPriority;
194 if (![stream.session canAddInput:input])
196 [stream.session commitConfiguration];
197 return CAM_ERROR_CODE_OperationNotSupported;
199 [stream.session addInput:input];
200 if (![stream.session canAddOutput:stream.output])
202 [stream.session commitConfiguration];
203 return CAM_ERROR_CODE_OperationNotSupported;
205 [stream.session addOutput:stream.output];
206 const OSType pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
207 if (![stream.output.availableVideoCVPixelFormatTypes containsObject:@(pixelFormat)])
209 [stream.session commitConfiguration];
210 return CAM_ERROR_CODE_InvalidMediaType;
212 stream.output.videoSettings = @{(NSString *)kCVPixelBufferPixelFormatTypeKey: @(pixelFormat)};
213 stream.output.alwaysDiscardsLateVideoFrames = YES;
215 if (!cam_avfoundation_configure_camera(camera, selected, duration))
217 [stream.session commitConfiguration];
218 return CAM_ERROR_CODE_UnexpectedError;
221 [stream.session commitConfiguration];
223#if (TARGET_OS_OSX && (__MAC_OS_X_VERSION_MAX_ALLOWED >= 140000)) || \
224 (!TARGET_OS_OSX && (__IPHONE_OS_VERSION_MAX_ALLOWED >= 170000))
225 if (@available(macOS 14.0, iOS 17.0, *))
227 AVCaptureConnection *connection = [stream.output connectionWithMediaType:AVMediaTypeVideo];
229 return CAM_ERROR_CODE_OperationNotSupported;
230 AVCaptureDeviceRotationCoordinator *coordinator =
231 [[AVCaptureDeviceRotationCoordinator alloc] initWithDevice:camera previewLayer:nil];
233 return CAM_ERROR_CODE_OutOfMemory;
234 const CGFloat angle = coordinator.videoRotationAngleForHorizonLevelCapture;
235 [coordinator release];
238 if ([connection isVideoRotationAngleSupported:angle])
240 connection.videoRotationAngle = angle;
241 WLog_INFO(TAG,
"Camera capture rotation: %.0f degrees", angle);
244 WLog_WARN(TAG,
"Camera does not support rotation by %.0f degrees", angle);
248 stream->_rawSample = Stream_New(NULL, (
size_t)stream->_width * stream->_height * 3 / 2);
249 if (!stream->_rawSample)
250 return CAM_ERROR_CODE_OutOfMemory;
251 OSStatus status = CVPixelBufferCreate(NULL, stream->_width, stream->_height, pixelFormat, NULL,
255 if (@available(iOS 16.0, *))
257 status = cam_avfoundation_pixel_transfer_create(&stream->_transfer);
262 WLog_ERR(TAG,
"Unable to create NV12 output buffer: %d", (
int)status);
263 return CAM_ERROR_CODE_UnexpectedError;
265 [stream.output setSampleBufferDelegate:stream queue:stream->_sampleQueue];
267 dispatch_sync(stream->_controlQueue, ^{
270 [stream.session startRunning];
277 if (stream.session.running && !cam_avfoundation_configure_camera(camera, selected, duration))
278 return CAM_ERROR_CODE_UnexpectedError;
280 if (!stream.session.running)
281 return CAM_ERROR_CODE_UnexpectedError;
282 atomic_store(&stream->_drained,
false);
283 return CAM_ERROR_CODE_None;
286@implementation FreeRDPCameraAVFoundationStream
287- (void)captureOutput:(AVCaptureOutput *)output
288 didOutputSampleBuffer:(CMSampleBufferRef)sample
289 fromConnection:(AVCaptureConnection *)connection
293 WINPR_ASSERT(output);
294 WINPR_ASSERT(sample);
295 WINPR_ASSERT(connection);
297 if (atomic_load(&_drained) || !_device->streams[_streamIndex].streaming)
299 CVImageBufferRef image = CMSampleBufferGetImageBuffer(sample);
304 WLog_INFO(TAG,
"Camera capture input: %zux%zu, pixel format 0x%08" PRIx32,
305 CVPixelBufferGetWidth(image), CVPixelBufferGetHeight(image),
306 CVPixelBufferGetPixelFormatType(image));
307 _captureLogged = TRUE;
309 CVPixelBufferRef raw = image;
310 if ((CVPixelBufferGetWidth(image) != _width) || (CVPixelBufferGetHeight(image) != _height))
312 OSStatus status = kVTParameterErr;
313 if (@available(iOS 16.0, *))
316 status = VTPixelTransferSessionTransferImage(
317 (VTPixelTransferSessionRef)_transfer, image, _rawImage);
321 WLog_WARN(TAG,
"Unable to resize NV12 camera sample: %d", (
int)status);
326 if (!cam_avfoundation_pixel_buffer_to_nv12(raw, _rawSample))
328 WLog_WARN(TAG,
"Unable to pack native NV12 camera sample");
331 cam_avfoundation_deliver(
self, Stream_Buffer(_rawSample), Stream_Length(_rawSample));
332 if (atomic_load(&_drained))
334 AVCaptureSession *session =
self.session;
335 dispatch_async(_controlQueue, ^{
338 [session stopRunning];
347 [
self.output setSampleBufferDelegate:nil queue:NULL];
348 AVCaptureSession *session =
self.session;
350 dispatch_sync(_controlQueue, ^{
353 [session stopRunning];
358 dispatch_sync(_sampleQueue, ^{
362 if (@available(iOS 16.0, *))
363 VTPixelTransferSessionInvalidate((VTPixelTransferSessionRef)_transfer);
364 CFRelease(_transfer);
369 CFRelease(_rawImage);
372 Stream_Free(_rawSample, TRUE);
383 dispatch_release(_sampleQueue);
385 dispatch_release(_controlQueue);
390@interface FreeRDPCameraAVFoundation : NSObject
391@property(nonatomic, retain) NSMutableDictionary<NSString *, AVCaptureDevice *> *cameras;
392@property(nonatomic, retain)
393 NSMutableDictionary<NSString *, FreeRDPCameraAVFoundationStream *> *streams;
394@property(nonatomic, retain)
395 NSMutableDictionary<NSString *, FreeRDPCameraAVFoundationModes *> *modes;
397@implementation FreeRDPCameraAVFoundation
414static FreeRDPCameraAVFoundation *cam_avfoundation_context(ICamHal *hal)
417 return (FreeRDPCameraAVFoundation *)((CamAVFoundationHal *)hal)->context;
421static UINT cam_avfoundation_enumerate(ICamHal *hal, ICamHalEnumCallback callback,
425 WINPR_ASSERT(callback);
427 WINPR_ASSERT(channel);
431 if (!cam_avfoundation_request_access())
433 NSMutableArray<AVCaptureDeviceType> *types =
434 [NSMutableArray arrayWithObject:AVCaptureDeviceTypeBuiltInWideAngleCamera];
436#if __MAC_OS_X_VERSION_MIN_REQUIRED >= 140000
437 [types addObject:AVCaptureDeviceTypeExternal];
439#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 140000
440 if (@available(macOS 14.0, *))
441 [types addObject:AVCaptureDeviceTypeExternal];
444 [types addObject:cam_avfoundation_legacy_external_type()];
447#if __IPHONE_OS_VERSION_MAX_ALLOWED >= 170000
448 if (@available(iOS 17.0, *))
449 [types addObject:AVCaptureDeviceTypeExternal];
452 AVCaptureDeviceDiscoverySession *discovery = [AVCaptureDeviceDiscoverySession
453 discoverySessionWithDeviceTypes:types
454 mediaType:AVMediaTypeVideo
455 position:AVCaptureDevicePositionUnspecified];
456 FreeRDPCameraAVFoundation *context = cam_avfoundation_context(hal);
458 for (AVCaptureDevice *camera in discovery.devices)
460 NSString *deviceId = nil;
461 @
synchronized(context)
463 for (NSString *key in context.cameras)
465 if ([context.cameras[key].uniqueID isEqualToString:camera.uniqueID])
474 stringWithFormat:@"avfoundation-%lu", (unsigned long)context.cameras.count];
475 context.cameras[deviceId] = camera;
477 [context.modes removeObjectForKey:deviceId];
479 if (callback(ecam, channel, deviceId.UTF8String, camera.localizedName.UTF8String) !=
489static BOOL cam_avfoundation_activate(ICamHal *hal,
const char *deviceId, CAM_ERROR_CODE *error)
492 WINPR_ASSERT(deviceId);
496 const BOOL found = cam_avfoundation_context(hal).cameras[@(deviceId)] != nil;
497 *error = found ? CAM_ERROR_CODE_None : CAM_ERROR_CODE_ItemNotFound;
503static INT16 cam_avfoundation_media_types(ICamHal *hal,
const char *deviceId,
size_t streamIndex,
509 WINPR_ASSERT(deviceId);
510 WINPR_ASSERT(formats);
511 WINPR_ASSERT(mediaTypes);
512 WINPR_ASSERT(mediaCount);
515 const size_t capacity = *mediaCount;
517 if (streamIndex != 0)
522 for (
size_t i = 0; i < formatCount; i++)
524 if ((formats[i].inputFormat == CAM_AVFOUNDATION_MEDIA_FORMAT) &&
525 ((formats[i].outputFormat == CAM_MEDIA_FORMAT_H264) ||
526 (formats[i].outputFormat == CAM_AVFOUNDATION_MEDIA_FORMAT)))
534 FreeRDPCameraAVFoundation *context = cam_avfoundation_context(hal);
536 NSString *key = @(deviceId);
537 @
synchronized(context)
539 modes = context.modes[key];
540 AVCaptureDevice *camera = context.cameras[key];
541 if (!modes && camera)
546 context.modes[key] = modes;
549 modes = [[modes retain] autorelease];
553 *mediaCount = [modes copyMediaTypes:mediaTypes capacity:capacity];
559static CAM_ERROR_CODE cam_avfoundation_stop(ICamHal *hal,
const char *deviceId,
size_t streamIndex)
562 WINPR_ASSERT(deviceId);
565 if (streamIndex != 0)
566 return CAM_ERROR_CODE_InvalidStreamNumber;
567 FreeRDPCameraAVFoundation *context = cam_avfoundation_context(hal);
568 NSString *key = @(deviceId);
569 [context.streams[key] stop];
570 [context.streams removeObjectForKey:key];
571 return CAM_ERROR_CODE_None;
576static BOOL cam_avfoundation_deactivate(ICamHal *hal,
const char *deviceId, CAM_ERROR_CODE *error)
579 WINPR_ASSERT(deviceId);
581 *error = cam_avfoundation_stop(hal, deviceId, 0);
582 return *error == CAM_ERROR_CODE_None;
586static CAM_ERROR_CODE cam_avfoundation_start(ICamHal *hal,
CameraDevice *device,
size_t streamIndex,
588 ICamHalSampleCapturedCallback callback)
591 WINPR_ASSERT(device);
592 WINPR_ASSERT(mediaType);
593 WINPR_ASSERT(callback);
596 if (streamIndex != 0)
597 return CAM_ERROR_CODE_InvalidStreamNumber;
598 if ((mediaType->Format != CAM_AVFOUNDATION_MEDIA_FORMAT) ||
599 !mediaType->FrameRateNumerator || !mediaType->FrameRateDenominator)
600 return CAM_ERROR_CODE_InvalidMediaType;
601 if (!cam_avfoundation_request_access())
602 return CAM_ERROR_CODE_OperationNotSupported;
603 FreeRDPCameraAVFoundation *context = cam_avfoundation_context(hal);
604 NSString *key = @(device->deviceId);
605 AVCaptureDevice *camera = nil;
608 @
synchronized(context)
610 camera = [[context.cameras[key] retain] autorelease];
611 modes = [[context.modes[key] retain] autorelease];
613 if (!camera || !camera.connected)
614 return CAM_ERROR_CODE_ItemNotFound;
615 CMTime duration = kCMTimeInvalid;
616 AVCaptureDeviceFormat *selected = [modes formatForMediaType:mediaType
617 frameDuration:&duration];
619 return CAM_ERROR_CODE_InvalidMediaType;
621 const CAM_ERROR_CODE stopError = cam_avfoundation_stop(hal, device->deviceId, streamIndex);
622 if (stopError != CAM_ERROR_CODE_None)
624 FreeRDPCameraAVFoundationStream *stream = [[FreeRDPCameraAVFoundationStream alloc] init];
626 return CAM_ERROR_CODE_OutOfMemory;
627 stream->_device = device;
628 stream->_streamIndex = streamIndex;
629 stream->_callback = callback;
630 const CAM_ERROR_CODE error =
631 cam_avfoundation_stream_start(stream, camera, mediaType, selected, duration);
632 if (error != CAM_ERROR_CODE_None)
637 context.streams[key] = stream;
639 return CAM_ERROR_CODE_None;
644static CAM_ERROR_CODE cam_avfoundation_free(ICamHal *hal)
649 CamAVFoundationHal *av = (CamAVFoundationHal *)hal;
650 FreeRDPCameraAVFoundation *context = (FreeRDPCameraAVFoundation *)av->context;
651 for (FreeRDPCameraAVFoundationStream *stream in context.streams.allValues)
656 return CAM_ERROR_CODE_None;
661FREERDP_ENTRY_POINT(UINT VCAPITYPE avfoundation_freerdp_rdpecam_client_subsystem_entry(
666 WINPR_ASSERT(entryPoints);
667 WINPR_ASSERT(entryPoints->plugin);
668 WINPR_ASSERT(entryPoints->pRegisterCameraHal);
669 CamAVFoundationHal *hal = calloc(1,
sizeof(CamAVFoundationHal));
671 return CHANNEL_RC_NO_MEMORY;
672 FreeRDPCameraAVFoundation *context = [[FreeRDPCameraAVFoundation alloc] init];
676 return CHANNEL_RC_NO_MEMORY;
678 context.cameras = [NSMutableDictionary dictionary];
679 context.streams = [NSMutableDictionary dictionary];
680 context.modes = [NSMutableDictionary dictionary];
681 hal->context = context;
682 hal->iface.RequestDrivenCapture = TRUE;
683 hal->iface.Enumerate = cam_avfoundation_enumerate;
684 hal->iface.Activate = cam_avfoundation_activate;
685 hal->iface.Deactivate = cam_avfoundation_deactivate;
686 hal->iface.GetMediaTypeDescriptions = cam_avfoundation_media_types;
687 hal->iface.StartStream = cam_avfoundation_start;
688 hal->iface.StopStream = cam_avfoundation_stop;
689 hal->iface.Free = cam_avfoundation_free;
690 const UINT rc = entryPoints->pRegisterCameraHal(entryPoints->plugin, &hal->iface);
691 if (rc != CHANNEL_RC_OK)
693 if (cam_avfoundation_free(&hal->iface) != CAM_ERROR_CODE_None)
694 WLog_ERR(TAG,
"Unable to release unregistered camera HAL");