23#include <camera/NdkCameraDevice.h>
24#include <camera/NdkCameraManager.h>
25#include <camera/NdkCameraMetadata.h>
26#include <camera/NdkCameraCaptureSession.h>
27#include <media/NdkImage.h>
28#include <media/NdkImageReader.h>
32#define TAG CHANNELS_TAG("rdpecam-android.client")
34#define ANDROID_CAMERA_PREFIX "android-camera-"
35#define ANDROID_CAMERA_PREFIX_LEN (sizeof(ANDROID_CAMERA_PREFIX) - 1)
37#define CAM_ANDROID_FPS 30
42 ACameraManager* manager;
50 ACameraDevice* device;
51 ACameraCaptureSession* session;
52 ACaptureSessionOutputContainer* outputContainer;
53 ACaptureSessionOutput* sessionOutput;
54 ACameraOutputTarget* outputTarget;
55 ACaptureRequest* captureRequest;
56 AImageReader* imageReader;
57 ANativeWindow* window;
59 ICamHalSampleCapturedCallback sampleCallback;
66 size_t nv12BufferSize;
69static const char* cam_android_extract_camera_id(
const char* deviceId)
71 WINPR_ASSERT(deviceId);
73 if (strncmp(deviceId, ANDROID_CAMERA_PREFIX, ANDROID_CAMERA_PREFIX_LEN) != 0)
75 return deviceId + ANDROID_CAMERA_PREFIX_LEN;
79static void cam_android_close_session(CamAndroidStream* stream)
85 ACameraCaptureSession_stopRepeating(stream->session);
86 ACameraCaptureSession_close(stream->session);
87 stream->session = NULL;
89 if (stream->captureRequest)
91 if (stream->outputTarget)
92 ACaptureRequest_removeTarget(stream->captureRequest, stream->outputTarget);
93 ACaptureRequest_free(stream->captureRequest);
94 stream->captureRequest = NULL;
96 if (stream->outputTarget)
98 ACameraOutputTarget_free(stream->outputTarget);
99 stream->outputTarget = NULL;
101 if (stream->outputContainer)
103 ACaptureSessionOutputContainer_free(stream->outputContainer);
104 stream->outputContainer = NULL;
106 if (stream->sessionOutput)
108 ACaptureSessionOutput_free(stream->sessionOutput);
109 stream->sessionOutput = NULL;
114static void cam_android_yuv420_888_to_nv12(BYTE* dst,
int width,
int height,
const uint8_t* yData,
115 int yRowStride,
const uint8_t* uData,
int uRowStride,
116 int uPixelStride,
const uint8_t* vData,
int vRowStride,
124 const size_t ySize = (size_t)width * height;
127 if (yRowStride == width)
128 memcpy(dst, yData, ySize);
131 for (
int row = 0; row < height; row++)
132 memcpy(dst + (
size_t)row * width, yData + (
size_t)row * yRowStride, (
size_t)width);
136 BYTE* uv = dst + ySize;
137 const int uvHeight = height / 2;
138 const int uvWidth = width / 2;
139 const size_t uvRowBytes = (size_t)(uvWidth * 2);
141 if (uPixelStride == 2 && vPixelStride == 2)
147 if (uRowStride == (
int)uvRowBytes)
148 memcpy(uv, uData, (
size_t)uvHeight * uvRowBytes);
151 for (
int row = 0; row < uvHeight; row++)
152 memcpy(uv + (
size_t)row * uvRowBytes, uData + (size_t)row * uRowStride,
159 for (
int row = 0; row < uvHeight; row++)
161 const uint8_t* u = uData + (size_t)row * uRowStride;
162 const uint8_t* v = vData + (size_t)row * vRowStride;
163 BYTE* d = uv + (size_t)row * uvRowBytes;
164 for (
int col = 0; col < uvWidth; col++, u += 2, v += 2, d += 2)
175 for (
int row = 0; row < uvHeight; row++)
177 const uint8_t* uRow = uData + (size_t)row * uRowStride;
178 const uint8_t* vRow = vData + (size_t)row * vRowStride;
179 BYTE* dstRow = uv + (size_t)row * uvRowBytes;
180 for (
int col = 0; col < uvWidth; col++)
182 dstRow[col * 2] = uRow[col * uPixelStride];
183 dstRow[col * 2 + 1] = vRow[col * vPixelStride];
189static void cam_android_deliver_frame(CamAndroidStream* stream, AImage* image)
191 WINPR_ASSERT(stream);
196 if (AImage_getWidth(image, &width) != AMEDIA_OK ||
197 AImage_getHeight(image, &height) != AMEDIA_OK)
200 const size_t ySize = (size_t)(width * height);
201 const size_t nv12Size = ySize + ySize / 2;
203 if (stream->nv12BufferSize < nv12Size)
205 BYTE* tmp = (BYTE*)realloc(stream->nv12Buffer, nv12Size);
208 stream->nv12Buffer = tmp;
209 stream->nv12BufferSize = nv12Size;
214 uint8_t* yData = NULL;
215 uint8_t* uData = NULL;
216 uint8_t* vData = NULL;
217 if (AImage_getPlaneData(image, 0, &yData, &dataLength) != AMEDIA_OK ||
218 AImage_getPlaneData(image, 1, &uData, &dataLength) != AMEDIA_OK ||
219 AImage_getPlaneData(image, 2, &vData, &dataLength) != AMEDIA_OK)
224 int uPixelStride = 0;
226 int vPixelStride = 0;
227 AImage_getPlaneRowStride(image, 0, &yRowStride);
228 AImage_getPlaneRowStride(image, 1, &uRowStride);
229 AImage_getPlanePixelStride(image, 1, &uPixelStride);
230 AImage_getPlaneRowStride(image, 2, &vRowStride);
231 AImage_getPlanePixelStride(image, 2, &vPixelStride);
233 cam_android_yuv420_888_to_nv12(stream->nv12Buffer, width, height, yData, yRowStride, uData,
234 uRowStride, uPixelStride, vData, vRowStride, vPixelStride);
236 stream->sampleCallback(stream->dev, stream->streamIndex, stream->nv12Buffer, nv12Size);
239static void cam_android_on_image_available(
void* context, AImageReader* reader)
241 CamAndroidStream* stream = (CamAndroidStream*)context;
242 WINPR_ASSERT(stream);
243 WINPR_ASSERT(reader);
245 EnterCriticalSection(&stream->lock);
246 const BOOL streaming = stream->streaming;
247 LeaveCriticalSection(&stream->lock);
252 AImage* image = NULL;
253 if (AImageReader_acquireLatestImage(reader, &image) == AMEDIA_OK && image)
255 cam_android_deliver_frame(stream, image);
256 AImage_delete(image);
261static void cam_android_mark_device_error(CamAndroidStream* stream)
265 EnterCriticalSection(&stream->lock);
266 stream->deviceError = TRUE;
267 stream->streaming = FALSE;
268 LeaveCriticalSection(&stream->lock);
271static void cam_android_device_disconnected(
void* context, ACameraDevice* device)
273 WINPR_UNUSED(device);
274 WLog_WARN(TAG,
"Camera device disconnected");
275 cam_android_mark_device_error((CamAndroidStream*)context);
278static void cam_android_device_error(
void* context, ACameraDevice* device,
int error)
280 WINPR_UNUSED(device);
281 WLog_ERR(TAG,
"Camera device error %d", error);
282 cam_android_mark_device_error((CamAndroidStream*)context);
285static void cam_android_session_active(
void* context, ACameraCaptureSession* session)
287 WINPR_UNUSED(context);
288 WINPR_UNUSED(session);
289 WLog_DBG(TAG,
"Capture session active");
292static void cam_android_session_closed(
void* context, ACameraCaptureSession* session)
294 WINPR_UNUSED(context);
295 WINPR_UNUSED(session);
296 WLog_DBG(TAG,
"Capture session closed");
299static void cam_android_session_ready(
void* context, ACameraCaptureSession* session)
301 WINPR_UNUSED(context);
302 WINPR_UNUSED(session);
303 WLog_DBG(TAG,
"Capture session ready");
308static BOOL cam_android_is_backward_compatible(ACameraMetadata* characteristics)
310 WINPR_ASSERT(characteristics);
312 ACameraMetadata_const_entry caps;
313 if (ACameraMetadata_getConstEntry(characteristics, ACAMERA_REQUEST_AVAILABLE_CAPABILITIES,
314 &caps) != ACAMERA_OK)
317 for (uint32_t i = 0; i < caps.count; i++)
319 if (caps.data.u8[i] == ACAMERA_REQUEST_AVAILABLE_CAPABILITIES_BACKWARD_COMPATIBLE)
325static UINT cam_android_enumerate(ICamHal* ihal, ICamHalEnumCallback callback,
CameraPlugin* ecam,
328 CamAndroidHal* hal = (CamAndroidHal*)ihal;
334 if (!freerdp_video_conversion_supported(FREERDP_VIDEO_FORMAT_NV12,
335 FREERDP_VIDEO_FORMAT_YUV420P))
337 WLog_WARN(TAG,
"Camera redirection unavailable: video conversion (FFmpeg/swscale) missing");
341 ACameraIdList* idList = NULL;
342 if (ACameraManager_getCameraIdList(hal->manager, &idList) != ACAMERA_OK || !idList)
348 BOOL seenFacing[ACAMERA_LENS_FACING_EXTERNAL + 1] = WINPR_C_ARRAY_INIT;
350 for (
int i = 0; i < idList->numCameras; i++)
352 const char* cameraId = idList->cameraIds[i];
354 ACameraMetadata* characteristics = NULL;
355 if (ACameraManager_getCameraCharacteristics(hal->manager, cameraId, &characteristics) !=
360 if (!cam_android_is_backward_compatible(characteristics))
362 WLog_DBG(TAG,
"Skipping camera %s: not BACKWARD_COMPATIBLE", cameraId);
363 ACameraMetadata_free(characteristics);
367 ACameraMetadata_const_entry facingEntry;
368 const char* facingName =
"Camera";
369 uint8_t facing = ACAMERA_LENS_FACING_EXTERNAL;
370 if (ACameraMetadata_getConstEntry(characteristics, ACAMERA_LENS_FACING, &facingEntry) ==
373 facing = facingEntry.data.u8[0];
376 case ACAMERA_LENS_FACING_FRONT:
377 facingName =
"Front Camera";
379 case ACAMERA_LENS_FACING_BACK:
380 facingName =
"Back Camera";
383 facingName =
"External Camera";
389 if (facing <= ACAMERA_LENS_FACING_EXTERNAL)
391 if (seenFacing[facing])
393 WLog_DBG(TAG,
"Skipping camera %s: already have a %s", cameraId, facingName);
394 ACameraMetadata_free(characteristics);
397 seenFacing[facing] = TRUE;
401 (void)_snprintf(deviceId,
sizeof(deviceId), ANDROID_CAMERA_PREFIX
"%s", cameraId);
403 IFCALL(callback, ecam, hchannel, deviceId, facingName);
406 ACameraMetadata_free(characteristics);
409 ACameraManager_deleteCameraIdList(idList);
415static void cam_android_close_device(CamAndroidStream* stream)
417 WINPR_ASSERT(stream);
419 EnterCriticalSection(&stream->lock);
420 stream->streaming = FALSE;
421 LeaveCriticalSection(&stream->lock);
423 if (stream->imageReader)
424 AImageReader_setImageListener(stream->imageReader, NULL);
426 cam_android_close_session(stream);
430 ACameraDevice_close(stream->device);
431 stream->device = NULL;
434 if (stream->imageReader)
436 AImageReader_delete(stream->imageReader);
437 stream->imageReader = NULL;
438 stream->window = NULL;
440 free(stream->nv12Buffer);
441 stream->nv12Buffer = NULL;
442 stream->nv12BufferSize = 0;
443 stream->deviceError = FALSE;
448static void cam_android_close_other_devices(CamAndroidHal* hal,
const char* keepDeviceId)
451 WINPR_ASSERT(keepDeviceId);
453 ULONG_PTR* keys = NULL;
454 const size_t count = HashTable_GetKeys(hal->streams, &keys);
455 for (
size_t i = 0; i < count; i++)
457 const char*
id = (
const char*)keys[i];
458 if (strcmp(
id, keepDeviceId) == 0)
461 CamAndroidStream* other = (CamAndroidStream*)HashTable_GetItemValue(hal->streams,
id);
462 if (!other || !other->device)
465 WLog_DBG(TAG,
"Closing camera %s to free resources for %s",
id, keepDeviceId);
466 cam_android_close_device(other);
471static CamAndroidStream* cam_android_get_or_create_stream(CamAndroidHal* hal,
const char* deviceId,
472 CAM_ERROR_CODE* errorCode)
475 WINPR_ASSERT(deviceId);
476 WINPR_ASSERT(errorCode);
478 CamAndroidStream* stream = (CamAndroidStream*)HashTable_GetItemValue(hal->streams, deviceId);
482 stream = (CamAndroidStream*)calloc(1,
sizeof(CamAndroidStream));
485 *errorCode = CAM_ERROR_CODE_OutOfMemory;
488 if (!InitializeCriticalSectionEx(&stream->lock, 0, 0))
491 *errorCode = CAM_ERROR_CODE_UnexpectedError;
494 if (!HashTable_Insert(hal->streams, deviceId, stream))
496 DeleteCriticalSection(&stream->lock);
498 *errorCode = CAM_ERROR_CODE_UnexpectedError;
506static BOOL cam_android_open_device(CamAndroidHal* hal, CamAndroidStream* stream,
507 const char* deviceId,
const char* cameraId,
508 CAM_ERROR_CODE* errorCode)
511 WINPR_ASSERT(stream);
512 WINPR_ASSERT(deviceId);
513 WINPR_ASSERT(cameraId);
514 WINPR_ASSERT(errorCode);
517 EnterCriticalSection(&stream->lock);
518 const BOOL hadError = stream->deviceError;
519 LeaveCriticalSection(&stream->lock);
520 if (stream->device && hadError)
521 cam_android_close_device(stream);
526 cam_android_close_other_devices(hal, deviceId);
528 ACameraDevice_StateCallbacks deviceCallbacks = {
530 .onDisconnected = cam_android_device_disconnected,
531 .onError = cam_android_device_error,
534 camera_status_t status =
535 ACameraManager_openCamera(hal->manager, cameraId, &deviceCallbacks, &stream->device);
536 if (status != ACAMERA_OK)
538 WLog_ERR(TAG,
"ACameraManager_openCamera failed: %d", status);
539 *errorCode = CAM_ERROR_CODE_InvalidRequest;
543 WLog_DBG(TAG,
"Opened camera %s", cameraId);
547static BOOL cam_android_activate(ICamHal* ihal,
const char* deviceId, CAM_ERROR_CODE* errorCode)
549 CamAndroidHal* hal = (CamAndroidHal*)ihal;
551 WINPR_ASSERT(deviceId);
552 WINPR_ASSERT(errorCode);
554 *errorCode = CAM_ERROR_CODE_None;
556 const char* cameraId = cam_android_extract_camera_id(deviceId);
559 *errorCode = CAM_ERROR_CODE_InvalidRequest;
564 return cam_android_get_or_create_stream(hal, deviceId, errorCode) != NULL;
567static BOOL cam_android_deactivate(ICamHal* ihal,
const char* deviceId, CAM_ERROR_CODE* errorCode)
569 CamAndroidHal* hal = (CamAndroidHal*)ihal;
571 WINPR_ASSERT(deviceId);
572 WINPR_ASSERT(errorCode);
574 *errorCode = CAM_ERROR_CODE_None;
576 CamAndroidStream* stream = (CamAndroidStream*)HashTable_GetItemValue(hal->streams, deviceId);
577 if (!stream || !stream->device)
580 cam_android_close_device(stream);
584static INT16 cam_android_get_media_type_descriptions(ICamHal* ihal,
const char* deviceId,
587 size_t nSupportedFormats,
591 WINPR_UNUSED(streamIndex);
592 CamAndroidHal* hal = (CamAndroidHal*)ihal;
594 WINPR_ASSERT(deviceId);
595 WINPR_ASSERT(supportedFormats || (nSupportedFormats == 0));
596 WINPR_ASSERT(mediaTypes);
597 WINPR_ASSERT(nMediaTypes);
599 size_t maxMediaTypes = *nMediaTypes;
602 const char* cameraId = cam_android_extract_camera_id(deviceId);
607 INT16 formatIndex = -1;
608 for (
size_t i = 0; i < nSupportedFormats; i++)
610 if (supportedFormats[i].inputFormat == CAM_MEDIA_FORMAT_NV12)
612 formatIndex = (INT16)i;
618 WLog_WARN(TAG,
"Server does not offer NV12 format");
622 ACameraMetadata* characteristics = NULL;
623 if (ACameraManager_getCameraCharacteristics(hal->manager, cameraId, &characteristics) !=
628 ACameraMetadata_const_entry entry;
629 if (ACameraMetadata_getConstEntry(
630 characteristics, ACAMERA_SCALER_AVAILABLE_STREAM_CONFIGURATIONS, &entry) != ACAMERA_OK)
632 ACameraMetadata_free(characteristics);
637 for (uint32_t i = 0; i + 3 < entry.count; i += 4)
639 const int32_t fmt = entry.data.i32[i];
640 const int32_t w = entry.data.i32[i + 1];
641 const int32_t h = entry.data.i32[i + 2];
642 const int32_t isInput = entry.data.i32[i + 3];
646 if (fmt != AIMAGE_FORMAT_YUV_420_888)
649 if (w > 1920 || h > 1080)
652 mediaTypes[nTypes].Format = CAM_MEDIA_FORMAT_NV12;
653 mediaTypes[nTypes].Width = (UINT32)w;
654 mediaTypes[nTypes].Height = (UINT32)h;
655 mediaTypes[nTypes].FrameRateNumerator = CAM_ANDROID_FPS;
656 mediaTypes[nTypes].FrameRateDenominator = 1;
657 mediaTypes[nTypes].PixelAspectRatioNumerator = 1;
658 mediaTypes[nTypes].PixelAspectRatioDenominator = 1;
660 WLog_DBG(TAG,
"Camera format NV12 %dx%d @ %dfps", w, h, CAM_ANDROID_FPS);
662 if (++nTypes >= maxMediaTypes)
666 *nMediaTypes = nTypes;
667 ACameraMetadata_free(characteristics);
671 WLog_ERR(TAG,
"No supported YUV resolutions found for camera %s", cameraId);
678static CAM_ERROR_CODE cam_android_start_stream(ICamHal* ihal,
CameraDevice* dev,
size_t streamIndex,
680 ICamHalSampleCapturedCallback callback)
682 CamAndroidHal* hal = (CamAndroidHal*)ihal;
685 WINPR_ASSERT(mediaType);
686 WINPR_ASSERT(callback);
688 const char* deviceId = dev->deviceId;
690 const char* cameraId = cam_android_extract_camera_id(deviceId);
692 return CAM_ERROR_CODE_InvalidRequest;
694 CAM_ERROR_CODE errorCode = CAM_ERROR_CODE_None;
695 CamAndroidStream* stream = cam_android_get_or_create_stream(hal, deviceId, &errorCode);
699 if (!cam_android_open_device(hal, stream, deviceId, cameraId, &errorCode))
703 stream->streamIndex = streamIndex;
704 stream->sampleCallback = callback;
706 const int32_t width = (int32_t)mediaType->Width;
707 const int32_t height = (int32_t)mediaType->Height;
709 if (AImageReader_new(width, height, AIMAGE_FORMAT_YUV_420_888, 4, &stream->imageReader) !=
712 WLog_ERR(TAG,
"AImageReader_new failed");
713 return CAM_ERROR_CODE_UnexpectedError;
716 AImageReader_ImageListener listener = {
718 .onImageAvailable = cam_android_on_image_available,
720 AImageReader_setImageListener(stream->imageReader, &listener);
722 if (AImageReader_getWindow(stream->imageReader, &stream->window) != AMEDIA_OK)
724 WLog_ERR(TAG,
"AImageReader_getWindow failed");
728 ACaptureSessionOutput_create(stream->window, &stream->sessionOutput);
729 ACaptureSessionOutputContainer_create(&stream->outputContainer);
730 ACaptureSessionOutputContainer_add(stream->outputContainer, stream->sessionOutput);
731 ACameraOutputTarget_create(stream->window, &stream->outputTarget);
732 ACameraDevice_createCaptureRequest(stream->device, TEMPLATE_RECORD, &stream->captureRequest);
733 ACaptureRequest_addTarget(stream->captureRequest, stream->outputTarget);
735 ACameraCaptureSession_stateCallbacks sessionCallbacks = {
737 .onActive = cam_android_session_active,
738 .onClosed = cam_android_session_closed,
739 .onReady = cam_android_session_ready,
742 if (ACameraDevice_createCaptureSession(stream->device, stream->outputContainer,
743 &sessionCallbacks, &stream->session) != ACAMERA_OK)
745 WLog_ERR(TAG,
"ACameraDevice_createCaptureSession failed");
749 if (ACameraCaptureSession_setRepeatingRequest(stream->session, NULL, 1, &stream->captureRequest,
752 WLog_ERR(TAG,
"ACameraCaptureSession_setRepeatingRequest failed");
756 EnterCriticalSection(&stream->lock);
757 stream->streaming = TRUE;
758 LeaveCriticalSection(&stream->lock);
760 WLog_DBG(TAG,
"Camera stream started: %dx%d", width, height);
761 return CAM_ERROR_CODE_None;
764 cam_android_close_device(stream);
765 return CAM_ERROR_CODE_UnexpectedError;
768static CAM_ERROR_CODE cam_android_stop_stream(ICamHal* ihal,
const char* deviceId,
771 WINPR_UNUSED(streamIndex);
772 CamAndroidHal* hal = (CamAndroidHal*)ihal;
774 WINPR_ASSERT(deviceId);
776 CamAndroidStream* stream = (CamAndroidStream*)HashTable_GetItemValue(hal->streams, deviceId);
778 return CAM_ERROR_CODE_None;
780 cam_android_close_device(stream);
782 WLog_DBG(TAG,
"Camera stream stopped for %s", deviceId);
783 return CAM_ERROR_CODE_None;
786static void cam_android_stream_free(
void* obj)
788 CamAndroidStream* stream = (CamAndroidStream*)obj;
792 cam_android_close_device(stream);
793 DeleteCriticalSection(&stream->lock);
797static CAM_ERROR_CODE cam_android_free(ICamHal* ihal)
799 CamAndroidHal* hal = (CamAndroidHal*)ihal;
801 return CAM_ERROR_CODE_NotInitialized;
803 HashTable_Free(hal->streams);
805 ACameraManager_delete(hal->manager);
807 return CAM_ERROR_CODE_None;
810FREERDP_ENTRY_POINT(UINT VCAPITYPE android_freerdp_rdpecam_client_subsystem_entry(
813 WINPR_ASSERT(pEntryPoints);
815 CamAndroidHal* hal = (CamAndroidHal*)calloc(1,
sizeof(CamAndroidHal));
817 return CHANNEL_RC_NO_MEMORY;
819 hal->manager = ACameraManager_create();
823 return ERROR_INTERNAL_ERROR;
826 hal->streams = HashTable_New(FALSE);
830 if (!HashTable_SetupForStringData(hal->streams, FALSE))
833 wObject* obj = HashTable_ValueObject(hal->streams);
837 hal->iHal.Enumerate = cam_android_enumerate;
838 hal->iHal.Activate = cam_android_activate;
839 hal->iHal.Deactivate = cam_android_deactivate;
840 hal->iHal.GetMediaTypeDescriptions = cam_android_get_media_type_descriptions;
841 hal->iHal.StartStream = cam_android_start_stream;
842 hal->iHal.StopStream = cam_android_stop_stream;
843 hal->iHal.Free = cam_android_free;
845 UINT ret = pEntryPoints->pRegisterCameraHal(pEntryPoints->plugin, &hal->iHal);
846 if (ret != CHANNEL_RC_OK)
848 WLog_ERR(TAG,
"RegisterCameraHal failed: %" PRIu32, ret);
855 cam_android_free(&hal->iHal);
856 return ERROR_INTERNAL_ERROR;
This struct contains function pointer to initialize/free objects.
OBJECT_FREE_FN fnObjectFree