24#define __ANDROID_UNAVAILABLE_SYMBOLS_ARE_WEAK__
26#include <freerdp/config.h>
28#include <winpr/wlog.h>
29#include <winpr/assert.h>
30#include <winpr/cast.h>
31#include <winpr/library.h>
33#include <freerdp/log.h>
34#include <freerdp/codec/h264.h>
36#include <media/NdkMediaCodec.h>
37#include <media/NdkMediaFormat.h>
41static const char* CODEC_NAME =
"video/avc";
43static const int COLOR_FormatYUV420Planar = 19;
44static const int COLOR_FormatYUV420SemiPlanar = 21;
45static const int COLOR_FormatYUV420Flexible = 0x7f420888;
49static const char* MEDIACODEC_KEY_PREPEND_HEADER =
"prepend-sps-pps-to-idr-frames";
52static const char* MEDIACODEC_KEY_LOW_LATENCY =
"low-latency";
55static const char* MEDIACODEC_KEY_CROP_LEFT =
"crop-left";
56static const char* MEDIACODEC_KEY_CROP_RIGHT =
"crop-right";
57static const char* MEDIACODEC_KEY_CROP_TOP =
"crop-top";
58static const char* MEDIACODEC_KEY_CROP_BOTTOM =
"crop-bottom";
61static const int64_t MEDIACODEC_INPUT_DEQUEUE_TIMEOUT = 10000;
62static const int64_t MEDIACODEC_OUTPUT_DEQUEUE_TIMEOUT = 5000;
65static const int MEDIACODEC_DECODE_MAX_RETRIES = 100;
68static const int MEDIACODEC_MINIMUM_WIDTH = 320;
69static const int MEDIACODEC_MINIMUM_HEIGHT = 240;
70static const int32_t MEDIACODEC_MAXIMUM_DIMENSION = 16384;
73static const int MEDIACODEC_IFRAME_INTERVAL = 1;
75static const int MEDIACODEC_DEFAULT_FRAMERATE = 30;
76static const int MEDIACODEC_DEFAULT_BITRATE = 4000000;
79static const int MEDIACODEC_BITRATE_MODE_VBR = 1;
82static const char* MEDIACODEC_PARAM_VIDEO_BITRATE =
"video-bitrate";
88 AMediaFormat* inputFormat;
89 AMediaFormat* outputFormat;
95 int32_t outputSliceHeight;
99 ssize_t currentOutputBufferIndex;
104 BYTE* decodeI420Buffer;
105 size_t decodeI420Capacity;
109 int32_t encoderWidth;
110 int32_t encoderHeight;
111 int32_t encoderStride;
112 int32_t encoderSliceHeight;
113 int32_t encoderBitrate;
117 size_t outputCapacity;
118 int32_t encoderFailedWidth;
119 int32_t encoderFailedHeight;
124} H264_CONTEXT_MEDIACODEC;
126#define mediacodec_warn(log, status, what) \
127 mediacodec_warn_((log), (status), (what), __FILE__, __func__, __LINE__)
129static void mediacodec_warn_(wLog* log, media_status_t status,
const char* what,
const char* file,
130 const char* fkt,
size_t line)
132 if ((status != AMEDIA_OK) && WLog_IsLevelActive(log, WLOG_WARN))
133 WLog_PrintTextMessage(log, WLOG_WARN, line, file, fkt,
"%s failed: %d", what, status);
137static AMediaFormat* mediacodec_format_new(wLog* log,
int width,
int height)
139 const char* media_format =
nullptr;
140 AMediaFormat* format = AMediaFormat_new();
141 if (format ==
nullptr)
143 WLog_Print(log, WLOG_ERROR,
"AMediaFormat_new failed");
147 AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, CODEC_NAME);
148 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_WIDTH, width);
149 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, height);
150 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, COLOR_FormatYUV420Planar);
152 AMediaFormat_setInt32(format, MEDIACODEC_KEY_LOW_LATENCY, 1);
153 if (__builtin_available(android 28, *))
154 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_PRIORITY, 0);
156 media_format = AMediaFormat_toString(format);
157 if (media_format ==
nullptr)
159 WLog_Print(log, WLOG_ERROR,
"AMediaFormat_toString failed");
160 AMediaFormat_delete(format);
164 WLog_Print(log, WLOG_DEBUG,
"MediaCodec configuring with desired output format [%s]",
170static void set_mediacodec_format(H264_CONTEXT* h264, AMediaFormat** formatVariable,
171 AMediaFormat* newFormat)
173 media_status_t status = AMEDIA_OK;
174 H264_CONTEXT_MEDIACODEC* sys =
nullptr;
177 WINPR_ASSERT(formatVariable);
179 sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
182 if (*formatVariable == newFormat)
185 if (*formatVariable !=
nullptr)
187 status = AMediaFormat_delete(*formatVariable);
188 if (status != AMEDIA_OK)
190 WLog_Print(h264->log, WLOG_ERROR,
"Error AMediaFormat_delete %d", status);
194 *formatVariable = newFormat;
198static int update_mediacodec_inputformat(H264_CONTEXT* h264)
200 H264_CONTEXT_MEDIACODEC* sys =
nullptr;
201 AMediaFormat* inputFormat =
nullptr;
202 const char* mediaFormatName =
nullptr;
206 sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
209 if (__builtin_available(android 28, *))
211 inputFormat = AMediaCodec_getInputFormat(sys->decoder);
212 if (inputFormat ==
nullptr)
214 WLog_Print(h264->log, WLOG_ERROR,
"AMediaCodec_getInputFormat failed");
220 inputFormat = sys->inputFormat;
221 set_mediacodec_format(h264, &sys->inputFormat, inputFormat);
223 mediaFormatName = AMediaFormat_toString(sys->inputFormat);
224 if (mediaFormatName ==
nullptr)
226 WLog_Print(h264->log, WLOG_ERROR,
"AMediaFormat_toString failed");
229 WLog_Print(h264->log, WLOG_DEBUG,
"Using MediaCodec with input MediaFormat [%s]",
236static int update_mediacodec_outputformat(H264_CONTEXT* h264)
238 H264_CONTEXT_MEDIACODEC* sys =
nullptr;
239 AMediaFormat* outputFormat =
nullptr;
240 const char* mediaFormatName =
nullptr;
241 int32_t outputWidth = 0;
242 int32_t outputHeight = 0;
246 sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
249 outputFormat = AMediaCodec_getOutputFormat(sys->decoder);
250 if (outputFormat ==
nullptr)
252 WLog_Print(h264->log, WLOG_ERROR,
"AMediaCodec_getOutputFormat failed");
255 set_mediacodec_format(h264, &sys->outputFormat, outputFormat);
257 mediaFormatName = AMediaFormat_toString(sys->outputFormat);
258 if (mediaFormatName ==
nullptr)
260 WLog_Print(h264->log, WLOG_ERROR,
"AMediaFormat_toString failed");
263 WLog_Print(h264->log, WLOG_DEBUG,
"Using MediaCodec with output MediaFormat [%s]",
266 if (!AMediaFormat_getInt32(sys->outputFormat, AMEDIAFORMAT_KEY_WIDTH, &outputWidth))
268 WLog_Print(h264->log, WLOG_ERROR,
"fnAMediaFormat_getInt32 failed getting width");
272 if (!AMediaFormat_getInt32(sys->outputFormat, AMEDIAFORMAT_KEY_HEIGHT, &outputHeight))
274 WLog_Print(h264->log, WLOG_ERROR,
"fnAMediaFormat_getInt32 failed getting height");
279 int32_t outputStride = 0;
280 int32_t outputSliceHeight = 0;
281 if (!AMediaFormat_getInt32(sys->outputFormat, AMEDIAFORMAT_KEY_STRIDE, &outputStride) ||
282 outputStride < outputWidth)
283 outputStride = outputWidth;
284 outputSliceHeight = outputHeight;
285 if (__builtin_available(android 28, *))
287 int32_t reportedSliceHeight = 0;
288 if (AMediaFormat_getInt32(sys->outputFormat, AMEDIAFORMAT_KEY_SLICE_HEIGHT,
289 &reportedSliceHeight) &&
290 reportedSliceHeight >= outputHeight)
291 outputSliceHeight = reportedSliceHeight;
293 if ((outputWidth <= 0) || (outputHeight <= 0) ||
294 (outputStride > MEDIACODEC_MAXIMUM_DIMENSION) ||
295 (outputSliceHeight > MEDIACODEC_MAXIMUM_DIMENSION))
297 WLog_Print(h264->log, WLOG_ERROR,
298 "MediaCodec output layout out of range: %dx%d stride %d slice-height %d",
299 outputWidth, outputHeight, outputStride, outputSliceHeight);
302 sys->outputStride = outputStride;
303 sys->outputSliceHeight = outputSliceHeight;
307 sys->outputWidth = outputWidth;
308 sys->outputHeight = outputHeight;
312 int32_t cropLeft = 0;
313 int32_t cropRight = 0;
315 int32_t cropBottom = 0;
317 if (!AMediaFormat_getInt32(sys->outputFormat, MEDIACODEC_KEY_CROP_LEFT, &cropLeft))
319 if (!AMediaFormat_getInt32(sys->outputFormat, MEDIACODEC_KEY_CROP_TOP, &cropTop))
322 if (AMediaFormat_getInt32(sys->outputFormat, MEDIACODEC_KEY_CROP_RIGHT, &cropRight) &&
323 AMediaFormat_getInt32(sys->outputFormat, MEDIACODEC_KEY_CROP_BOTTOM, &cropBottom) &&
324 (cropLeft >= 0) && (cropTop >= 0) && (cropRight >= cropLeft) && (cropBottom >= cropTop) &&
325 (cropRight < outputWidth) && (cropBottom < outputHeight))
327 sys->outputWidth = cropRight - cropLeft + 1;
328 sys->outputHeight = cropBottom - cropTop + 1;
329 sys->cropLeft = cropLeft;
330 sys->cropTop = cropTop;
334 int32_t colorFormat = 0;
335 if (!AMediaFormat_getInt32(sys->outputFormat, AMEDIAFORMAT_KEY_COLOR_FORMAT, &colorFormat))
336 colorFormat = COLOR_FormatYUV420Planar;
337 sys->colorFormat = colorFormat;
339 if (colorFormat == COLOR_FormatYUV420Flexible)
340 WLog_Print(h264->log, WLOG_DEBUG,
"MediaCodec output is flexible YUV, assuming NV12");
343 h264->YUVWidth = (UINT32)sys->outputWidth;
344 h264->YUVHeight = (UINT32)sys->outputHeight;
346 WLog_Print(h264->log, WLOG_DEBUG,
347 "MediaCodec output %dx%d stride %d slice-height %d crop [%d,%d] color 0x%x",
348 sys->outputWidth, sys->outputHeight, sys->outputStride, sys->outputSliceHeight,
349 sys->cropLeft, sys->cropTop, colorFormat);
354static void release_current_outputbuffer(H264_CONTEXT* h264)
356 media_status_t status = AMEDIA_OK;
357 H264_CONTEXT_MEDIACODEC* sys =
nullptr;
360 sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
363 if (sys->currentOutputBufferIndex < 0)
368 status = AMediaCodec_releaseOutputBuffer(sys->decoder, sys->currentOutputBufferIndex, FALSE);
369 if (status != AMEDIA_OK)
371 WLog_Print(h264->log, WLOG_ERROR,
"Error AMediaCodec_releaseOutputBuffer %d", status);
374 sys->currentOutputBufferIndex = -1;
378static void mediacodec_encoder_release(wLog* log, H264_CONTEXT_MEDIACODEC* sys)
380 if (!sys || !sys->encoder)
383 if (sys->encoderStarted)
384 mediacodec_warn(log, AMediaCodec_stop(sys->encoder),
"AMediaCodec_stop");
385 mediacodec_warn(log, AMediaCodec_delete(sys->encoder),
"AMediaCodec_delete");
386 sys->encoder =
nullptr;
387 sys->encoderStarted = FALSE;
390 free(sys->spsPpsData);
391 sys->spsPpsData =
nullptr;
397static BOOL mediacodec_encoder_probe(wLog* log)
399 AMediaCodec* probe = AMediaCodec_createEncoderByType(CODEC_NAME);
402 WLog_Print(log, WLOG_WARN,
"No MediaCodec H.264 encoder on this device");
407 AMediaFormat* format = AMediaFormat_new();
410 AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, CODEC_NAME);
411 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_WIDTH, MEDIACODEC_MINIMUM_WIDTH);
412 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, MEDIACODEC_MINIMUM_HEIGHT);
413 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, COLOR_FormatYUV420SemiPlanar);
414 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_BIT_RATE, MEDIACODEC_DEFAULT_BITRATE);
415 if (__builtin_available(android 28, *))
416 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_BITRATE_MODE,
417 MEDIACODEC_BITRATE_MODE_VBR);
418 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_FRAME_RATE, MEDIACODEC_DEFAULT_FRAMERATE);
419 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_I_FRAME_INTERVAL,
420 MEDIACODEC_IFRAME_INTERVAL);
422 if (AMediaCodec_configure(probe, format,
nullptr,
nullptr,
423 AMEDIACODEC_CONFIGURE_FLAG_ENCODE) == AMEDIA_OK &&
424 AMediaCodec_start(probe) == AMEDIA_OK)
427 mediacodec_warn(log, AMediaCodec_stop(probe),
"AMediaCodec_stop");
429 mediacodec_warn(log, AMediaFormat_delete(format),
"AMediaFormat_delete");
432 mediacodec_warn(log, AMediaCodec_delete(probe),
"AMediaCodec_delete");
434 WLog_Print(log, WLOG_WARN,
435 "MediaCodec H.264 encoder cannot be configured (color format unsupported?)");
440static int32_t mediacodec_rate(UINT32 value, int32_t fallback)
444 return (value > INT32_MAX) ? INT32_MAX : WINPR_ASSERTING_INT_CAST(int32_t, value);
449static BOOL mediacodec_encoder_configure(H264_CONTEXT* h264)
452 H264_CONTEXT_MEDIACODEC* sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
455 mediacodec_encoder_release(h264->log, sys);
457 if (h264->width < MEDIACODEC_MINIMUM_WIDTH || h264->height < MEDIACODEC_MINIMUM_HEIGHT ||
458 h264->width > (UINT32)MEDIACODEC_MAXIMUM_DIMENSION ||
459 h264->height > (UINT32)MEDIACODEC_MAXIMUM_DIMENSION)
461 WLog_Print(h264->log, WLOG_ERROR,
"MediaCodec encoder dimensions out of range [%u,%u]",
462 h264->width, h264->height);
466 const int32_t width = WINPR_ASSERTING_INT_CAST(int32_t, h264->width);
467 const int32_t height = WINPR_ASSERTING_INT_CAST(int32_t, h264->height);
468 const int32_t framerate = mediacodec_rate(h264->FrameRate, MEDIACODEC_DEFAULT_FRAMERATE);
469 const int32_t bitrate = mediacodec_rate(h264->BitRate, MEDIACODEC_DEFAULT_BITRATE);
471 sys->encoder = AMediaCodec_createEncoderByType(CODEC_NAME);
474 WLog_Print(h264->log, WLOG_ERROR,
"AMediaCodec_createEncoderByType failed");
478 AMediaFormat* format = AMediaFormat_new();
481 WLog_Print(h264->log, WLOG_ERROR,
"AMediaFormat_new failed");
482 mediacodec_encoder_release(h264->log, sys);
486 AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, CODEC_NAME);
487 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_WIDTH, width);
488 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, height);
489 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, COLOR_FormatYUV420SemiPlanar);
490 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_BIT_RATE, bitrate);
492 if (__builtin_available(android 28, *))
493 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_BITRATE_MODE, MEDIACODEC_BITRATE_MODE_VBR);
494 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_FRAME_RATE, framerate);
495 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_I_FRAME_INTERVAL, MEDIACODEC_IFRAME_INTERVAL);
496 AMediaFormat_setInt32(format, MEDIACODEC_KEY_PREPEND_HEADER, 1);
498 media_status_t status = AMediaCodec_configure(sys->encoder, format,
nullptr,
nullptr,
499 AMEDIACODEC_CONFIGURE_FLAG_ENCODE);
500 mediacodec_warn(h264->log, AMediaFormat_delete(format),
"AMediaFormat_delete");
501 if (status != AMEDIA_OK)
503 WLog_Print(h264->log, WLOG_ERROR,
"AMediaCodec_configure (encoder) failed: %d", status);
504 mediacodec_encoder_release(h264->log, sys);
508 status = AMediaCodec_start(sys->encoder);
509 if (status != AMEDIA_OK)
511 WLog_Print(h264->log, WLOG_ERROR,
"AMediaCodec_start (encoder) failed: %d", status);
512 mediacodec_encoder_release(h264->log, sys);
516 sys->encoderStarted = TRUE;
517 sys->encoderWidth = width;
518 sys->encoderHeight = height;
519 sys->encoderBitrate = bitrate;
520 sys->syncRequested = FALSE;
524 sys->encoderStride = sys->encoderWidth + (sys->encoderWidth & 1);
525 sys->encoderSliceHeight = sys->encoderHeight;
526 if (__builtin_available(android 28, *))
528 AMediaFormat* inputFormat = AMediaCodec_getInputFormat(sys->encoder);
532 int32_t sliceHeight = 0;
533 if (AMediaFormat_getInt32(inputFormat, AMEDIAFORMAT_KEY_STRIDE, &stride) &&
534 (stride >= sys->encoderWidth))
535 sys->encoderStride = stride;
536 if (AMediaFormat_getInt32(inputFormat, AMEDIAFORMAT_KEY_SLICE_HEIGHT, &sliceHeight) &&
537 (sliceHeight >= sys->encoderHeight))
538 sys->encoderSliceHeight = sliceHeight;
539 mediacodec_warn(h264->log, AMediaFormat_delete(inputFormat),
"AMediaFormat_delete");
543 if ((sys->encoderStride > MEDIACODEC_MAXIMUM_DIMENSION) ||
544 (sys->encoderSliceHeight > MEDIACODEC_MAXIMUM_DIMENSION))
546 WLog_Print(h264->log, WLOG_ERROR,
547 "MediaCodec encoder input layout out of range: stride %d slice-height %d",
548 sys->encoderStride, sys->encoderSliceHeight);
549 mediacodec_encoder_release(h264->log, sys);
554 h264->log, WLOG_DEBUG,
555 "MediaCodec H.264 encoder started: %ux%u @ %d fps, %d bps, stride %d, slice-height %d",
556 h264->width, h264->height, framerate, bitrate, sys->encoderStride, sys->encoderSliceHeight);
563static BOOL mediacodec_copy_i420_to_nv12(uint8_t* dst,
size_t dstCapacity,
const BYTE** pSrcYuv,
564 const UINT32* pStride, int32_t width, int32_t height,
565 int32_t dstStride, int32_t dstSliceHeight)
568 WINPR_ASSERT(pSrcYuv);
569 WINPR_ASSERT(pStride);
571 const int32_t cw = (width + 1) / 2;
572 const int32_t ch = (height + 1) / 2;
573 const size_t ySize = (size_t)dstStride * dstSliceHeight;
575 if (dstCapacity < ySize + (
size_t)dstStride * ((dstSliceHeight + 1) / 2))
578 for (int32_t row = 0; row < height; row++)
579 memcpy(dst + (
size_t)row * dstStride, pSrcYuv[0] + (size_t)row * pStride[0], (
size_t)width);
582 const int32_t pairs = (dstStride / 2 < cw) ? dstStride / 2 : cw;
583 uint8_t* uv = dst + ySize;
584 for (int32_t row = 0; row < ch; row++)
586 const BYTE* uRow = pSrcYuv[1] + (size_t)row * pStride[1];
587 const BYTE* vRow = pSrcYuv[2] + (size_t)row * pStride[2];
588 uint8_t* uvRow = uv + (size_t)row * dstStride;
589 for (int32_t col = 0; col < pairs; col++)
591 uvRow[(size_t)col * 2] = uRow[col];
592 uvRow[(size_t)col * 2 + 1] = vRow[col];
601static BOOL mediacodec_has_sps(
const uint8_t* data,
size_t size)
605 for (
size_t i = 0; i + 3 < size; i++)
607 if ((data[i] != 0) || (data[i + 1] != 0))
611 if (data[i + 2] == 1)
613 else if ((data[i + 2] == 0) && (data[i + 3] == 1))
621 const uint8_t type = data[nal] & 0x1F;
624 if ((type == 1) || (type == 5))
633static void mediacodec_copy_nv12_to_i420(BYTE* dstYuv[3],
const UINT32 dstStride[3],
634 const uint8_t* src, int32_t width, int32_t height,
635 int32_t srcStride, int32_t srcSliceHeight,
636 int32_t cropLeft, int32_t cropTop)
638 WINPR_ASSERT(dstYuv);
639 WINPR_ASSERT(dstStride);
642 const int32_t cw = (width + 1) / 2;
643 const int32_t ch = (height + 1) / 2;
644 const size_t srcLumaOffset = (size_t)cropTop * (
size_t)srcStride + (size_t)cropLeft;
645 const size_t srcChromaOffset = (size_t)srcStride * (
size_t)srcSliceHeight +
646 (size_t)(cropTop / 2) * (size_t)srcStride +
647 ((
size_t)cropLeft & ~1ULL);
649 const uint8_t* srcY = src + srcLumaOffset;
650 const uint8_t* srcUV = src + srcChromaOffset;
653 const int32_t chromaLeft = cropLeft & ~1;
654 const int32_t avail = (srcStride > chromaLeft) ? (srcStride - chromaLeft) / 2 : 0;
655 const int32_t pairs = (avail < cw) ? avail : cw;
657 for (int32_t row = 0; row < height; row++)
659 memcpy(dstYuv[0] + (
size_t)row * (
size_t)dstStride[0],
660 srcY + (
size_t)row * (
size_t)srcStride, (
size_t)width);
663 for (int32_t row = 0; row < ch; row++)
665 const uint8_t* uvRow = srcUV + (size_t)row * (
size_t)srcStride;
666 BYTE* uRow = dstYuv[1] + (size_t)row * (
size_t)dstStride[1];
667 BYTE* vRow = dstYuv[2] + (size_t)row * (
size_t)dstStride[2];
668 for (int32_t col = 0; col < pairs; col++)
670 uRow[col] = uvRow[(size_t)col * 2];
671 vRow[col] = uvRow[(size_t)col * 2 + 1];
675 for (int32_t col = pairs; col < cw; col++)
684static int mediacodec_compress(H264_CONTEXT* h264,
const BYTE** pSrcYuv,
const UINT32* pStride,
685 BYTE** ppDstData, UINT32* pDstSize)
688 WINPR_ASSERT(pSrcYuv);
689 WINPR_ASSERT(pStride);
690 WINPR_ASSERT(ppDstData);
691 WINPR_ASSERT(pDstSize);
693 H264_CONTEXT_MEDIACODEC* sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
696 if (!pSrcYuv[0] || !pSrcYuv[1] || !pSrcYuv[2])
699 if ((h264->width > (UINT32)MEDIACODEC_MAXIMUM_DIMENSION) ||
700 (h264->height > (UINT32)MEDIACODEC_MAXIMUM_DIMENSION))
703 const int32_t width = WINPR_ASSERTING_INT_CAST(int32_t, h264->width);
704 const int32_t height = WINPR_ASSERTING_INT_CAST(int32_t, h264->height);
706 if (!sys->encoderStarted || sys->encoderWidth != width || sys->encoderHeight != height)
708 if ((width == sys->encoderFailedWidth) && (height == sys->encoderFailedHeight))
711 if (!mediacodec_encoder_configure(h264))
713 sys->encoderFailedWidth = width;
714 sys->encoderFailedHeight = height;
717 sys->encoderFailedWidth = 0;
718 sys->encoderFailedHeight = 0;
723 const int32_t bitrate = mediacodec_rate(h264->BitRate, sys->encoderBitrate);
724 if (bitrate != sys->encoderBitrate)
726 if (__builtin_available(android 26, *))
728 AMediaFormat* params = AMediaFormat_new();
731 AMediaFormat_setInt32(params, MEDIACODEC_PARAM_VIDEO_BITRATE, bitrate);
732 if (AMediaCodec_setParameters(sys->encoder, params) == AMEDIA_OK)
733 sys->encoderBitrate = bitrate;
734 mediacodec_warn(h264->log, AMediaFormat_delete(params),
"AMediaFormat_delete");
741 if (!h264->firstLumaFrameDone && (sys->frameIndex > 0) && !sys->syncRequested)
743 if (__builtin_available(android 26, *))
745 AMediaFormat* syncParams = AMediaFormat_new();
748 AMediaFormat_setInt32(syncParams,
"request-sync", 0);
749 if (AMediaCodec_setParameters(sys->encoder, syncParams) == AMEDIA_OK)
750 sys->syncRequested = TRUE;
751 mediacodec_warn(h264->log, AMediaFormat_delete(syncParams),
"AMediaFormat_delete");
756 const int32_t dstStride = sys->encoderStride;
757 const int32_t dstSliceHeight = sys->encoderSliceHeight;
758 const size_t frameSize =
759 (size_t)dstStride * dstSliceHeight + (
size_t)dstStride * ((dstSliceHeight + 1) / 2);
762 const ssize_t inIdx =
763 AMediaCodec_dequeueInputBuffer(sys->encoder, MEDIACODEC_INPUT_DEQUEUE_TIMEOUT);
767 AMediaCodecBufferInfo stale = { 0 };
768 const ssize_t staleIdx = AMediaCodec_dequeueOutputBuffer(sys->encoder, &stale, 0);
770 mediacodec_warn(h264->log,
771 AMediaCodec_releaseOutputBuffer(sys->encoder, (
size_t)staleIdx, FALSE),
772 "AMediaCodec_releaseOutputBuffer");
774 WLog_Print(h264->log, WLOG_WARN,
"MediaCodec encoder has no input buffer, dropping frame");
778 size_t inCapacity = 0;
779 uint8_t* inBuffer = AMediaCodec_getInputBuffer(sys->encoder, (
size_t)inIdx, &inCapacity);
780 if (!inBuffer || inCapacity < frameSize)
782 WLog_Print(h264->log, WLOG_ERROR,
"MediaCodec encoder input buffer too small: %zu < %zu",
783 inCapacity, frameSize);
784 mediacodec_warn(h264->log,
785 AMediaCodec_queueInputBuffer(sys->encoder, (
size_t)inIdx, 0, 0, 0, 0),
786 "AMediaCodec_queueInputBuffer");
790 if (!mediacodec_copy_i420_to_nv12(inBuffer, inCapacity, pSrcYuv, pStride, width, height,
791 dstStride, dstSliceHeight))
793 WLog_Print(h264->log, WLOG_ERROR,
"MediaCodec encoder input buffer too small for layout");
794 mediacodec_warn(h264->log,
795 AMediaCodec_queueInputBuffer(sys->encoder, (
size_t)inIdx, 0, 0, 0, 0),
796 "AMediaCodec_queueInputBuffer");
800 const int32_t framerate = mediacodec_rate(h264->FrameRate, MEDIACODEC_DEFAULT_FRAMERATE);
801 const uint64_t pts = (uint64_t)sys->frameIndex++ * 1000000ULL / (uint64_t)framerate;
802 media_status_t status =
803 AMediaCodec_queueInputBuffer(sys->encoder, (
size_t)inIdx, 0, frameSize, pts, 0);
804 if (status != AMEDIA_OK)
806 WLog_Print(h264->log, WLOG_ERROR,
"AMediaCodec_queueInputBuffer (encoder) failed: %d",
814 AMediaCodecBufferInfo info = { 0 };
815 const ssize_t outIdx =
816 AMediaCodec_dequeueOutputBuffer(sys->encoder, &info, MEDIACODEC_OUTPUT_DEQUEUE_TIMEOUT);
819 size_t outCapacity = 0;
821 AMediaCodec_getOutputBuffer(sys->encoder, (
size_t)outIdx, &outCapacity);
822 if (!outBuffer || (info.size < 0) || (info.offset < 0) ||
823 ((
size_t)info.offset > outCapacity) ||
824 ((
size_t)info.size > outCapacity - (
size_t)info.offset))
826 WLog_Print(h264->log, WLOG_ERROR,
827 "MediaCodec encoder output out of range: offset %d size %d cap %zu",
828 info.offset, info.size, outCapacity);
830 h264->log, AMediaCodec_releaseOutputBuffer(sys->encoder, (
size_t)outIdx, FALSE),
831 "AMediaCodec_releaseOutputBuffer");
835 if (((info.flags & AMEDIACODEC_BUFFER_FLAG_CODEC_CONFIG) != 0) && (info.size > 0))
837 BYTE* newCfg = (BYTE*)realloc(sys->spsPpsData, (
size_t)info.size);
840 sys->spsPpsData = newCfg;
841 sys->spsPpsSize = (size_t)info.size;
842 memcpy(sys->spsPpsData, outBuffer + info.offset, (
size_t)info.size);
846 WLog_Print(h264->log, WLOG_WARN,
"MediaCodec encoder failed to store SPS/PPS");
850 if (((info.flags & AMEDIACODEC_BUFFER_FLAG_CODEC_CONFIG) != 0) || (info.size == 0))
852 const media_status_t rc =
853 AMediaCodec_releaseOutputBuffer(sys->encoder, (
size_t)outIdx, FALSE);
856 WLog_Print(h264->log, WLOG_ERROR,
857 "AMediaCodec_releaseOutputBuffer (encoder) failed: %d", rc);
863 const BOOL isKeyFrame = (info.flags & AMEDIACODEC_BUFFER_FLAG_KEY_FRAME) != 0;
864 const BOOL prependSps = isKeyFrame && sys->spsPpsData && sys->spsPpsSize > 0 &&
865 !mediacodec_has_sps(outBuffer + info.offset, (
size_t)info.size);
866 const size_t totalSize = (size_t)info.size + (prependSps ? sys->spsPpsSize : 0);
868 if (sys->outputCapacity < totalSize)
870 BYTE* resized = (BYTE*)realloc(sys->outputData, totalSize);
875 AMediaCodec_releaseOutputBuffer(sys->encoder, (
size_t)outIdx, FALSE),
876 "AMediaCodec_releaseOutputBuffer");
879 sys->outputData = resized;
880 sys->outputCapacity = totalSize;
885 memcpy(sys->outputData, sys->spsPpsData, sys->spsPpsSize);
886 memcpy(sys->outputData + sys->spsPpsSize, outBuffer + info.offset,
891 memcpy(sys->outputData, outBuffer + info.offset, (
size_t)info.size);
894 *ppDstData = sys->outputData;
895 *pDstSize = WINPR_ASSERTING_INT_CAST(UINT32, totalSize);
896 const media_status_t rc =
897 AMediaCodec_releaseOutputBuffer(sys->encoder, (
size_t)outIdx, FALSE);
899 WLog_Print(h264->log, WLOG_ERROR,
900 "AMediaCodec_releaseOutputBuffer (encoder) failed: %d", rc);
903 else if (outIdx == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED ||
904 outIdx == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED)
908 else if (outIdx == AMEDIACODEC_INFO_TRY_AGAIN_LATER)
911 WLog_Print(h264->log, WLOG_DEBUG,
912 "MediaCodec encoder output not ready, dropping frame");
917 WLog_Print(h264->log, WLOG_ERROR,
918 "AMediaCodec_dequeueOutputBuffer (encoder) returned %zd", outIdx);
925static int mediacodec_decompress(H264_CONTEXT* h264,
const BYTE* pSrcData, UINT32 SrcSize)
928 WINPR_ASSERT(pSrcData);
930 H264_CONTEXT_MEDIACODEC* sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
933 BYTE** pYUVData = h264->pYUVData;
934 WINPR_ASSERT(pYUVData);
936 UINT32* iStride = h264->iStride;
937 WINPR_ASSERT(iStride);
939 release_current_outputbuffer(h264);
941 if ((h264->width > (UINT32)MEDIACODEC_MAXIMUM_DIMENSION) ||
942 (h264->height > (UINT32)MEDIACODEC_MAXIMUM_DIMENSION))
944 WLog_Print(h264->log, WLOG_ERROR,
945 "MediaCodec size [%" PRIu32
",%" PRIu32
"] exceeds the maximum", h264->width,
950 if ((sys->width != WINPR_ASSERTING_INT_CAST(int32_t, h264->width)) ||
951 (sys->height != WINPR_ASSERTING_INT_CAST(int32_t, h264->height)))
953 sys->width = WINPR_ASSERTING_INT_CAST(int32_t, h264->width);
954 sys->height = WINPR_ASSERTING_INT_CAST(int32_t, h264->height);
956 if (sys->width < MEDIACODEC_MINIMUM_WIDTH || sys->height < MEDIACODEC_MINIMUM_HEIGHT)
958 WLog_Print(h264->log, WLOG_ERROR,
959 "MediaCodec got width or height smaller than minimum [%d,%d]", sys->width,
964 WLog_Print(h264->log, WLOG_DEBUG,
"MediaCodec setting new input width and height [%d,%d]",
965 sys->width, sys->height);
967 AMediaFormat_setInt32(sys->inputFormat, AMEDIAFORMAT_KEY_WIDTH, sys->width);
968 AMediaFormat_setInt32(sys->inputFormat, AMEDIAFORMAT_KEY_HEIGHT, sys->height);
969 if (__builtin_available(android 26, *))
971 const media_status_t status = AMediaCodec_setParameters(sys->decoder, sys->inputFormat);
972 if (status != AMEDIA_OK)
975 WLog_Print(h264->log, WLOG_DEBUG,
976 "AMediaCodec_setParameters returned %d, relying on "
977 "in-band SPS/PPS for the size change",
983 WLog_Print(h264->log, WLOG_DEBUG,
984 "AMediaCodec_setParameters unavailable below API 26, relying on "
985 "in-band SPS/PPS for the size change");
989 if (update_mediacodec_outputformat(h264) < 0)
991 WLog_Print(h264->log, WLOG_ERROR,
"MediaCodec failed updating input format");
998 UINT32 inputBufferCurrnetOffset = 0;
999 int inputRetries = 0;
1000 while (inputBufferCurrnetOffset < SrcSize)
1002 UINT32 numberOfBytesToCopy = SrcSize - inputBufferCurrnetOffset;
1003 const ssize_t inputBufferId =
1004 AMediaCodec_dequeueInputBuffer(sys->decoder, MEDIACODEC_INPUT_DEQUEUE_TIMEOUT);
1005 if (inputBufferId < 0)
1007 if (++inputRetries > MEDIACODEC_DECODE_MAX_RETRIES)
1009 WLog_Print(h264->log, WLOG_ERROR,
1010 "AMediaCodec_dequeueInputBuffer gave up [%zd]", inputBufferId);
1017 size_t inputBufferSize = 0;
1018 uint8_t* inputBuffer =
1019 AMediaCodec_getInputBuffer(sys->decoder, inputBufferId, &inputBufferSize);
1020 if (inputBuffer ==
nullptr)
1022 WLog_Print(h264->log, WLOG_ERROR,
"AMediaCodec_getInputBuffer failed");
1026 if (numberOfBytesToCopy > inputBufferSize)
1028 WLog_Print(h264->log, WLOG_WARN,
1029 "MediaCodec inputBufferSize: got [%zu] but wanted [%u]", inputBufferSize,
1030 numberOfBytesToCopy);
1031 numberOfBytesToCopy = WINPR_ASSERTING_INT_CAST(UINT32, inputBufferSize);
1034 memcpy(inputBuffer, &pSrcData[inputBufferCurrnetOffset], numberOfBytesToCopy);
1035 inputBufferCurrnetOffset += numberOfBytesToCopy;
1037 const media_status_t status = AMediaCodec_queueInputBuffer(
1038 sys->decoder, inputBufferId, 0, numberOfBytesToCopy, 0, 0);
1039 if (status != AMEDIA_OK)
1041 WLog_Print(h264->log, WLOG_ERROR,
"Error AMediaCodec_queueInputBuffer %d", status);
1046 int outputRetries = 0;
1049 AMediaCodecBufferInfo bufferInfo = WINPR_C_ARRAY_INIT;
1050 ssize_t outputBufferId = AMediaCodec_dequeueOutputBuffer(
1051 sys->decoder, &bufferInfo, MEDIACODEC_OUTPUT_DEQUEUE_TIMEOUT);
1052 if (outputBufferId >= 0)
1054 sys->currentOutputBufferIndex = outputBufferId;
1056 size_t outputBufferSize = 0;
1057 uint8_t* outputBuffer =
nullptr;
1059 AMediaCodec_getOutputBuffer(sys->decoder, outputBufferId, &outputBufferSize);
1060 sys->currentOutputBufferIndex = outputBufferId;
1063 if (bufferInfo.size <= 0)
1065 release_current_outputbuffer(h264);
1066 if (++outputRetries > MEDIACODEC_DECODE_MAX_RETRIES)
1072 if (!outputBuffer || (bufferInfo.offset < 0) ||
1073 ((
size_t)bufferInfo.offset > outputBufferSize))
1075 WLog_Print(h264->log, WLOG_ERROR,
1076 "MediaCodec output offset %d out of range %zu", bufferInfo.offset,
1080 outputBuffer += bufferInfo.offset;
1081 outputBufferSize -= (size_t)bufferInfo.offset;
1083 const size_t lumaSize = (size_t)sys->outputStride * (
size_t)sys->outputSliceHeight;
1084 const int32_t chromaStride = sys->outputStride / 2;
1085 const int32_t chromaHeight = (sys->outputSliceHeight + 1) / 2;
1086 const size_t chromaSize = (size_t)chromaStride * (
size_t)chromaHeight;
1087 const size_t minRequiredSize =
1088 lumaSize + (size_t)sys->outputStride * (
size_t)chromaHeight;
1090 if (outputBufferSize < minRequiredSize)
1092 WLog_Print(h264->log, WLOG_ERROR,
1093 "Error MediaCodec output buffer too small %zu < %zu",
1094 outputBufferSize, minRequiredSize);
1098 if (sys->colorFormat == COLOR_FormatYUV420Planar)
1100 iStride[0] = (UINT32)sys->outputStride;
1101 iStride[1] = (UINT32)chromaStride;
1102 iStride[2] = (UINT32)chromaStride;
1103 pYUVData[0] = outputBuffer +
1104 ((size_t)sys->cropTop * (
size_t)sys->outputStride) +
1105 (
size_t)sys->cropLeft;
1106 pYUVData[1] = outputBuffer + lumaSize +
1107 ((size_t)(sys->cropTop / 2) * (size_t)chromaStride) +
1108 ((size_t)sys->cropLeft / 2);
1109 pYUVData[2] = outputBuffer + lumaSize + chromaSize +
1110 ((size_t)(sys->cropTop / 2) * (size_t)chromaStride) +
1111 ((size_t)sys->cropLeft / 2);
1116 const UINT32 dstYStride = (UINT32)sys->outputWidth;
1117 const UINT32 dstUVStride = (UINT32)((sys->outputWidth + 1) / 2);
1118 const size_t yBytes = (size_t)dstYStride * (
size_t)sys->outputHeight;
1119 const size_t uvBytes =
1120 (size_t)dstUVStride * (
size_t)((sys->outputHeight + 1) / 2);
1121 const size_t neededCapacity = yBytes + uvBytes * 2;
1123 if (sys->decodeI420Capacity < neededCapacity)
1125 BYTE* newBuf = (BYTE*)realloc(sys->decodeI420Buffer, neededCapacity);
1128 WLog_Print(h264->log, WLOG_ERROR,
1129 "Failed to allocate I420 deinterleave buffer");
1132 sys->decodeI420Buffer = newBuf;
1133 sys->decodeI420Capacity = neededCapacity;
1136 iStride[0] = dstYStride;
1137 iStride[1] = dstUVStride;
1138 iStride[2] = dstUVStride;
1139 pYUVData[0] = sys->decodeI420Buffer;
1140 pYUVData[1] = sys->decodeI420Buffer + yBytes;
1141 pYUVData[2] = sys->decodeI420Buffer + yBytes + uvBytes;
1143 mediacodec_copy_nv12_to_i420(
1144 pYUVData, iStride, outputBuffer, sys->outputWidth, sys->outputHeight,
1145 sys->outputStride, sys->outputSliceHeight, sys->cropLeft, sys->cropTop);
1146 release_current_outputbuffer(h264);
1149 h264->YUVWidth = (UINT32)sys->outputWidth;
1150 h264->YUVHeight = (UINT32)sys->outputHeight;
1153 else if (outputBufferId == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED)
1155 if (update_mediacodec_outputformat(h264) < 0)
1157 WLog_Print(h264->log, WLOG_ERROR,
1158 "MediaCodec failed updating output format in decompress");
1162 else if (outputBufferId == AMEDIACODEC_INFO_TRY_AGAIN_LATER)
1164 if (++outputRetries > MEDIACODEC_DECODE_MAX_RETRIES)
1166 WLog_Print(h264->log, WLOG_DEBUG,
1167 "AMediaCodec_dequeueOutputBuffer no buffer ready yet");
1171 else if (outputBufferId == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED)
1173 WLog_Print(h264->log, WLOG_WARN,
1174 "AMediaCodec_dequeueOutputBuffer returned deprecated value "
1175 "AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED, ignoring");
1179 WLog_Print(h264->log, WLOG_ERROR,
1180 "AMediaCodec_dequeueOutputBuffer returned unknown value [%zd]",
1192static void mediacodec_uninit(H264_CONTEXT* h264)
1194 media_status_t status = AMEDIA_OK;
1195 H264_CONTEXT_MEDIACODEC* sys =
nullptr;
1199 sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
1201 WLog_Print(h264->log, WLOG_DEBUG,
"Uninitializing MediaCodec");
1206 if (sys->decoder !=
nullptr)
1208 release_current_outputbuffer(h264);
1210 if (sys->decoderStarted)
1212 status = AMediaCodec_stop(sys->decoder);
1213 if (status != AMEDIA_OK)
1215 WLog_Print(h264->log, WLOG_ERROR,
"Error AMediaCodec_stop %d", status);
1217 sys->decoderStarted = FALSE;
1220 status = AMediaCodec_delete(sys->decoder);
1221 if (status != AMEDIA_OK)
1223 WLog_Print(h264->log, WLOG_ERROR,
"Error AMediaCodec_delete %d", status);
1226 sys->decoder =
nullptr;
1229 mediacodec_encoder_release(h264->log, sys);
1230 free(sys->outputData);
1231 sys->outputData =
nullptr;
1232 free(sys->decodeI420Buffer);
1233 sys->decodeI420Buffer =
nullptr;
1234 free(sys->spsPpsData);
1235 sys->spsPpsData =
nullptr;
1237 set_mediacodec_format(h264, &sys->inputFormat,
nullptr);
1238 set_mediacodec_format(h264, &sys->outputFormat,
nullptr);
1241 h264->pSystemData =
nullptr;
1245static BOOL mediacodec_init(H264_CONTEXT* h264)
1247 H264_CONTEXT_MEDIACODEC* sys =
nullptr;
1248 media_status_t status = AMEDIA_OK;
1252 WLog_Print(h264->log, WLOG_DEBUG,
"Initializing MediaCodec");
1254 sys = (H264_CONTEXT_MEDIACODEC*)calloc(1,
sizeof(H264_CONTEXT_MEDIACODEC));
1261 h264->pSystemData = (
void*)sys;
1263 sys->currentOutputBufferIndex = -1;
1266 sys->width = sys->outputWidth = MEDIACODEC_MINIMUM_WIDTH;
1267 sys->height = sys->outputHeight = MEDIACODEC_MINIMUM_HEIGHT;
1270 if (h264->Compressor)
1272 if (!mediacodec_encoder_probe(h264->log))
1274 WLog_Print(h264->log, WLOG_DEBUG,
"MediaCodec H.264 encoder available");
1278 sys->decoder = AMediaCodec_createDecoderByType(CODEC_NAME);
1279 if (sys->decoder ==
nullptr)
1281 WLog_Print(h264->log, WLOG_ERROR,
"AMediaCodec_createCodecByName failed");
1286 if (__builtin_available(android 28, *))
1288 char* codec_name =
nullptr;
1289 status = AMediaCodec_getName(sys->decoder, &codec_name);
1290 if (status != AMEDIA_OK)
1292 WLog_Print(h264->log, WLOG_ERROR,
"AMediaCodec_getName failed: %d", status);
1296 WLog_Print(h264->log, WLOG_DEBUG,
"MediaCodec using %s codec [%s]", CODEC_NAME, codec_name);
1297 AMediaCodec_releaseName(sys->decoder, codec_name);
1300 WLog_Print(h264->log, WLOG_DEBUG,
"MediaCodec using %s codec", CODEC_NAME);
1302 set_mediacodec_format(h264, &sys->inputFormat,
1303 mediacodec_format_new(h264->log, sys->width, sys->height));
1305 status = AMediaCodec_configure(sys->decoder, sys->inputFormat,
nullptr,
nullptr, 0);
1306 if (status != AMEDIA_OK)
1308 WLog_Print(h264->log, WLOG_ERROR,
"AMediaCodec_configure failed: %d", status);
1312 if (update_mediacodec_inputformat(h264) < 0)
1314 WLog_Print(h264->log, WLOG_ERROR,
"MediaCodec failed updating input format");
1318 if (update_mediacodec_outputformat(h264) < 0)
1320 WLog_Print(h264->log, WLOG_ERROR,
"MediaCodec failed updating output format");
1324 WLog_Print(h264->log, WLOG_DEBUG,
"Starting MediaCodec");
1325 status = AMediaCodec_start(sys->decoder);
1326 if (status != AMEDIA_OK)
1328 WLog_Print(h264->log, WLOG_ERROR,
"AMediaCodec_start failed %d", status);
1332 sys->decoderStarted = TRUE;
1335 mediacodec_uninit(h264);
1339const H264_CONTEXT_SUBSYSTEM g_Subsystem_mediacodec = {
"MediaCodec", mediacodec_init,
1340 mediacodec_uninit, mediacodec_decompress,
1341 mediacodec_compress };