FreeRDP
Loading...
Searching...
No Matches
h264_mediacodec.c
1
21/* Must precede any NDK header: without it, APIs gated by __INTRODUCED_IN are annotated
22 * "strict", making any use a hard compile error that no __builtin_available guard can
23 * suppress. This switches them to weak symbols, which __builtin_available guards against. */
24#define __ANDROID_UNAVAILABLE_SYMBOLS_ARE_WEAK__
25
26#include <freerdp/config.h>
27
28#include <winpr/wlog.h>
29#include <winpr/assert.h>
30#include <winpr/cast.h>
31#include <winpr/library.h>
32
33#include <freerdp/log.h>
34#include <freerdp/codec/h264.h>
35
36#include <media/NdkMediaCodec.h>
37#include <media/NdkMediaFormat.h>
38
39#include "h264.h"
40
41static const char* CODEC_NAME = "video/avc";
42
43static const int COLOR_FormatYUV420Planar = 19;
44static const int COLOR_FormatYUV420SemiPlanar = 21; /* NV12, widest encoder support */
45static const int COLOR_FormatYUV420Flexible = 0x7f420888;
46
47/* MediaFormat.KEY_PREPEND_HEADER_TO_SYNC_FRAMES has no NDK constant; emits SPS/PPS before every IDR
48 */
49static const char* MEDIACODEC_KEY_PREPEND_HEADER = "prepend-sps-pps-to-idr-frames";
50
51/* AMEDIAFORMAT_KEY_LOW_LATENCY has no NDK constant below API 30; the string works at runtime */
52static const char* MEDIACODEC_KEY_LOW_LATENCY = "low-latency";
53
54/* Output crop rectangle; inclusive bounds, no NDK constants exist for these */
55static const char* MEDIACODEC_KEY_CROP_LEFT = "crop-left";
56static const char* MEDIACODEC_KEY_CROP_RIGHT = "crop-right";
57static const char* MEDIACODEC_KEY_CROP_TOP = "crop-top";
58static const char* MEDIACODEC_KEY_CROP_BOTTOM = "crop-bottom";
59
60/* Dequeue timeouts in microseconds; output may lag a frame while the pipeline primes */
61static const int64_t MEDIACODEC_INPUT_DEQUEUE_TIMEOUT = 10000;
62static const int64_t MEDIACODEC_OUTPUT_DEQUEUE_TIMEOUT = 5000;
63
64/* Bound the decode dequeue retries so a stalled codec cannot spin forever. */
65static const int MEDIACODEC_DECODE_MAX_RETRIES = 100;
66
67/* https://developer.android.com/reference/android/media/MediaCodec#qualityFloor */
68static const int MEDIACODEC_MINIMUM_WIDTH = 320;
69static const int MEDIACODEC_MINIMUM_HEIGHT = 240;
70static const int32_t MEDIACODEC_MAXIMUM_DIMENSION = 16384;
71
72/* Keyframe interval in seconds; short so a late-joining server recovers quickly */
73static const int MEDIACODEC_IFRAME_INTERVAL = 1;
74/* Fallbacks if the context did not specify a rate. */
75static const int MEDIACODEC_DEFAULT_FRAMERATE = 30;
76static const int MEDIACODEC_DEFAULT_BITRATE = 4000000;
77
78/* MediaCodecInfo.EncoderCapabilities bitrate modes */
79static const int MEDIACODEC_BITRATE_MODE_VBR = 1;
80
81/* setParameters keys; no NDK constant below API 31, the strings work at runtime */
82static const char* MEDIACODEC_PARAM_VIDEO_BITRATE = "video-bitrate";
83
84typedef struct
85{
86 AMediaCodec* decoder;
87 AMediaCodec* encoder;
88 AMediaFormat* inputFormat;
89 AMediaFormat* outputFormat;
90 int32_t width;
91 int32_t height;
92 int32_t outputWidth;
93 int32_t outputHeight;
94 int32_t outputStride;
95 int32_t outputSliceHeight;
96 int32_t cropLeft;
97 int32_t cropTop;
98 int32_t colorFormat;
99 ssize_t currentOutputBufferIndex;
100
101 BOOL decoderStarted;
102
103 /* decoder deinterleave buffer for NV12 / semiplanar output */
104 BYTE* decodeI420Buffer;
105 size_t decodeI420Capacity;
106
107 /* encoder state */
108 BOOL encoderStarted;
109 int32_t encoderWidth; /* dimensions the encoder is currently configured for */
110 int32_t encoderHeight;
111 int32_t encoderStride; /* input buffer layout reported by the encoder */
112 int32_t encoderSliceHeight;
113 int32_t encoderBitrate;
114 BOOL syncRequested; /* an IDR was already asked for since the last configure */
115 int64_t frameIndex; /* monotonic source for presentation timestamps */
116 BYTE* outputData; /* staging copy of the most recent encoded frame */
117 size_t outputCapacity; /* allocated size of outputData */
118 int32_t encoderFailedWidth; /* size the encoder refused */
119 int32_t encoderFailedHeight;
120
121 /* encoder cached SPS/PPS header */
122 BYTE* spsPpsData;
123 size_t spsPpsSize;
124} H264_CONTEXT_MEDIACODEC;
125
126#define mediacodec_warn(log, status, what) \
127 mediacodec_warn_((log), (status), (what), __FILE__, __func__, __LINE__)
128
129static void mediacodec_warn_(wLog* log, media_status_t status, const char* what, const char* file,
130 const char* fkt, size_t line)
131{
132 if ((status != AMEDIA_OK) && WLog_IsLevelActive(log, WLOG_WARN))
133 WLog_PrintTextMessage(log, WLOG_WARN, line, file, fkt, "%s failed: %d", what, status);
134}
135
136WINPR_ATTR_NODISCARD
137static AMediaFormat* mediacodec_format_new(wLog* log, int width, int height)
138{
139 const char* media_format = nullptr;
140 AMediaFormat* format = AMediaFormat_new();
141 if (format == nullptr)
142 {
143 WLog_Print(log, WLOG_ERROR, "AMediaFormat_new failed");
144 return nullptr;
145 }
146
147 AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, CODEC_NAME);
148 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_WIDTH, width);
149 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, height);
150 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, COLOR_FormatYUV420Planar);
151 /* Low-latency decode for remote desktop: drop frame reordering, ask for realtime scheduling. */
152 AMediaFormat_setInt32(format, MEDIACODEC_KEY_LOW_LATENCY, 1);
153 if (__builtin_available(android 28, *))
154 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_PRIORITY, 0);
155
156 media_format = AMediaFormat_toString(format);
157 if (media_format == nullptr)
158 {
159 WLog_Print(log, WLOG_ERROR, "AMediaFormat_toString failed");
160 AMediaFormat_delete(format);
161 return nullptr;
162 }
163
164 WLog_Print(log, WLOG_DEBUG, "MediaCodec configuring with desired output format [%s]",
165 media_format);
166
167 return format;
168}
169
170static void set_mediacodec_format(H264_CONTEXT* h264, AMediaFormat** formatVariable,
171 AMediaFormat* newFormat)
172{
173 media_status_t status = AMEDIA_OK;
174 H264_CONTEXT_MEDIACODEC* sys = nullptr;
175
176 WINPR_ASSERT(h264);
177 WINPR_ASSERT(formatVariable);
178
179 sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
180 WINPR_ASSERT(sys);
181
182 if (*formatVariable == newFormat)
183 return;
184
185 if (*formatVariable != nullptr)
186 {
187 status = AMediaFormat_delete(*formatVariable);
188 if (status != AMEDIA_OK)
189 {
190 WLog_Print(h264->log, WLOG_ERROR, "Error AMediaFormat_delete %d", status);
191 }
192 }
193
194 *formatVariable = newFormat;
195}
196
197WINPR_ATTR_NODISCARD
198static int update_mediacodec_inputformat(H264_CONTEXT* h264)
199{
200 H264_CONTEXT_MEDIACODEC* sys = nullptr;
201 AMediaFormat* inputFormat = nullptr;
202 const char* mediaFormatName = nullptr;
203
204 WINPR_ASSERT(h264);
205
206 sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
207 WINPR_ASSERT(sys);
208
209 if (__builtin_available(android 28, *))
210 {
211 inputFormat = AMediaCodec_getInputFormat(sys->decoder);
212 if (inputFormat == nullptr)
213 {
214 WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_getInputFormat failed");
215 return -1;
216 }
217 }
218 else
219 /* Reuse the format we configured the decoder with. */
220 inputFormat = sys->inputFormat;
221 set_mediacodec_format(h264, &sys->inputFormat, inputFormat);
222
223 mediaFormatName = AMediaFormat_toString(sys->inputFormat);
224 if (mediaFormatName == nullptr)
225 {
226 WLog_Print(h264->log, WLOG_ERROR, "AMediaFormat_toString failed");
227 return -1;
228 }
229 WLog_Print(h264->log, WLOG_DEBUG, "Using MediaCodec with input MediaFormat [%s]",
230 mediaFormatName);
231
232 return 1;
233}
234
235WINPR_ATTR_NODISCARD
236static int update_mediacodec_outputformat(H264_CONTEXT* h264)
237{
238 H264_CONTEXT_MEDIACODEC* sys = nullptr;
239 AMediaFormat* outputFormat = nullptr;
240 const char* mediaFormatName = nullptr;
241 int32_t outputWidth = 0;
242 int32_t outputHeight = 0;
243
244 WINPR_ASSERT(h264);
245
246 sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
247 WINPR_ASSERT(sys);
248
249 outputFormat = AMediaCodec_getOutputFormat(sys->decoder);
250 if (outputFormat == nullptr)
251 {
252 WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_getOutputFormat failed");
253 return -1;
254 }
255 set_mediacodec_format(h264, &sys->outputFormat, outputFormat);
256
257 mediaFormatName = AMediaFormat_toString(sys->outputFormat);
258 if (mediaFormatName == nullptr)
259 {
260 WLog_Print(h264->log, WLOG_ERROR, "AMediaFormat_toString failed");
261 return -1;
262 }
263 WLog_Print(h264->log, WLOG_DEBUG, "Using MediaCodec with output MediaFormat [%s]",
264 mediaFormatName);
265
266 if (!AMediaFormat_getInt32(sys->outputFormat, AMEDIAFORMAT_KEY_WIDTH, &outputWidth))
267 {
268 WLog_Print(h264->log, WLOG_ERROR, "fnAMediaFormat_getInt32 failed getting width");
269 return -1;
270 }
271
272 if (!AMediaFormat_getInt32(sys->outputFormat, AMEDIAFORMAT_KEY_HEIGHT, &outputHeight))
273 {
274 WLog_Print(h264->log, WLOG_ERROR, "fnAMediaFormat_getInt32 failed getting height");
275 return -1;
276 }
277
278 /* Honor codec stride/slice-height padding; fall back to tight packing. */
279 int32_t outputStride = 0;
280 int32_t outputSliceHeight = 0;
281 if (!AMediaFormat_getInt32(sys->outputFormat, AMEDIAFORMAT_KEY_STRIDE, &outputStride) ||
282 outputStride < outputWidth)
283 outputStride = outputWidth;
284 outputSliceHeight = outputHeight; /* tight-pack below that */
285 if (__builtin_available(android 28, *))
286 {
287 int32_t reportedSliceHeight = 0;
288 if (AMediaFormat_getInt32(sys->outputFormat, AMEDIAFORMAT_KEY_SLICE_HEIGHT,
289 &reportedSliceHeight) &&
290 reportedSliceHeight >= outputHeight)
291 outputSliceHeight = reportedSliceHeight;
292 }
293 if ((outputWidth <= 0) || (outputHeight <= 0) ||
294 (outputStride > MEDIACODEC_MAXIMUM_DIMENSION) ||
295 (outputSliceHeight > MEDIACODEC_MAXIMUM_DIMENSION))
296 {
297 WLog_Print(h264->log, WLOG_ERROR,
298 "MediaCodec output layout out of range: %dx%d stride %d slice-height %d",
299 outputWidth, outputHeight, outputStride, outputSliceHeight);
300 return -1;
301 }
302 sys->outputStride = outputStride;
303 sys->outputSliceHeight = outputSliceHeight;
304
305 /* Width and height above describe the padded allocation; the picture inside it is the crop
306 * rectangle, with inclusive bounds. */
307 sys->outputWidth = outputWidth;
308 sys->outputHeight = outputHeight;
309 sys->cropLeft = 0;
310 sys->cropTop = 0;
311
312 int32_t cropLeft = 0;
313 int32_t cropRight = 0;
314 int32_t cropTop = 0;
315 int32_t cropBottom = 0;
316 /* Only the far edges are always published; a missing near edge means zero. */
317 if (!AMediaFormat_getInt32(sys->outputFormat, MEDIACODEC_KEY_CROP_LEFT, &cropLeft))
318 cropLeft = 0;
319 if (!AMediaFormat_getInt32(sys->outputFormat, MEDIACODEC_KEY_CROP_TOP, &cropTop))
320 cropTop = 0;
321
322 if (AMediaFormat_getInt32(sys->outputFormat, MEDIACODEC_KEY_CROP_RIGHT, &cropRight) &&
323 AMediaFormat_getInt32(sys->outputFormat, MEDIACODEC_KEY_CROP_BOTTOM, &cropBottom) &&
324 (cropLeft >= 0) && (cropTop >= 0) && (cropRight >= cropLeft) && (cropBottom >= cropTop) &&
325 (cropRight < outputWidth) && (cropBottom < outputHeight))
326 {
327 sys->outputWidth = cropRight - cropLeft + 1;
328 sys->outputHeight = cropBottom - cropTop + 1;
329 sys->cropLeft = cropLeft;
330 sys->cropTop = cropTop;
331 }
332
333 /* The requested planar format is a hint; vendors may output NV12 or flexible YUV. */
334 int32_t colorFormat = 0;
335 if (!AMediaFormat_getInt32(sys->outputFormat, AMEDIAFORMAT_KEY_COLOR_FORMAT, &colorFormat))
336 colorFormat = COLOR_FormatYUV420Planar;
337 sys->colorFormat = colorFormat;
338
339 if (colorFormat == COLOR_FormatYUV420Flexible)
340 WLog_Print(h264->log, WLOG_DEBUG, "MediaCodec output is flexible YUV, assuming NV12");
341
342 /* Publish visible dimensions for region rectangle validation. */
343 h264->YUVWidth = (UINT32)sys->outputWidth;
344 h264->YUVHeight = (UINT32)sys->outputHeight;
345
346 WLog_Print(h264->log, WLOG_DEBUG,
347 "MediaCodec output %dx%d stride %d slice-height %d crop [%d,%d] color 0x%x",
348 sys->outputWidth, sys->outputHeight, sys->outputStride, sys->outputSliceHeight,
349 sys->cropLeft, sys->cropTop, colorFormat);
350
351 return 1;
352}
353
354static void release_current_outputbuffer(H264_CONTEXT* h264)
355{
356 media_status_t status = AMEDIA_OK;
357 H264_CONTEXT_MEDIACODEC* sys = nullptr;
358
359 WINPR_ASSERT(h264);
360 sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
361 WINPR_ASSERT(sys);
362
363 if (sys->currentOutputBufferIndex < 0)
364 {
365 return;
366 }
367
368 status = AMediaCodec_releaseOutputBuffer(sys->decoder, sys->currentOutputBufferIndex, FALSE);
369 if (status != AMEDIA_OK)
370 {
371 WLog_Print(h264->log, WLOG_ERROR, "Error AMediaCodec_releaseOutputBuffer %d", status);
372 }
373
374 sys->currentOutputBufferIndex = -1;
375}
376
377/* Tear down the encoder (used on reconfigure and uninit). */
378static void mediacodec_encoder_release(wLog* log, H264_CONTEXT_MEDIACODEC* sys)
379{
380 if (!sys || !sys->encoder)
381 return;
382
383 if (sys->encoderStarted)
384 mediacodec_warn(log, AMediaCodec_stop(sys->encoder), "AMediaCodec_stop");
385 mediacodec_warn(log, AMediaCodec_delete(sys->encoder), "AMediaCodec_delete");
386 sys->encoder = nullptr;
387 sys->encoderStarted = FALSE;
388
389 /* Invalidate cached SPS/PPS on release. */
390 free(sys->spsPpsData);
391 sys->spsPpsData = nullptr;
392 sys->spsPpsSize = 0;
393}
394
395/* Probe H.264 encoder support to allow fallback if unsupported. */
396WINPR_ATTR_NODISCARD
397static BOOL mediacodec_encoder_probe(wLog* log)
398{
399 AMediaCodec* probe = AMediaCodec_createEncoderByType(CODEC_NAME);
400 if (!probe)
401 {
402 WLog_Print(log, WLOG_WARN, "No MediaCodec H.264 encoder on this device");
403 return FALSE;
404 }
405
406 BOOL ok = FALSE;
407 AMediaFormat* format = AMediaFormat_new();
408 if (format)
409 {
410 AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, CODEC_NAME);
411 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_WIDTH, MEDIACODEC_MINIMUM_WIDTH);
412 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, MEDIACODEC_MINIMUM_HEIGHT);
413 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, COLOR_FormatYUV420SemiPlanar);
414 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_BIT_RATE, MEDIACODEC_DEFAULT_BITRATE);
415 if (__builtin_available(android 28, *))
416 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_BITRATE_MODE,
417 MEDIACODEC_BITRATE_MODE_VBR);
418 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_FRAME_RATE, MEDIACODEC_DEFAULT_FRAMERATE);
419 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_I_FRAME_INTERVAL,
420 MEDIACODEC_IFRAME_INTERVAL);
421
422 if (AMediaCodec_configure(probe, format, nullptr, nullptr,
423 AMEDIACODEC_CONFIGURE_FLAG_ENCODE) == AMEDIA_OK &&
424 AMediaCodec_start(probe) == AMEDIA_OK)
425 {
426 ok = TRUE;
427 mediacodec_warn(log, AMediaCodec_stop(probe), "AMediaCodec_stop");
428 }
429 mediacodec_warn(log, AMediaFormat_delete(format), "AMediaFormat_delete");
430 }
431
432 mediacodec_warn(log, AMediaCodec_delete(probe), "AMediaCodec_delete");
433 if (!ok)
434 WLog_Print(log, WLOG_WARN,
435 "MediaCodec H.264 encoder cannot be configured (color format unsupported?)");
436 return ok;
437}
438
439WINPR_ATTR_NODISCARD
440static int32_t mediacodec_rate(UINT32 value, int32_t fallback)
441{
442 if (value == 0)
443 return fallback;
444 return (value > INT32_MAX) ? INT32_MAX : WINPR_ASSERTING_INT_CAST(int32_t, value);
445}
446
447/* (Re)configure the encoder for current dimensions. */
448WINPR_ATTR_NODISCARD
449static BOOL mediacodec_encoder_configure(H264_CONTEXT* h264)
450{
451 WINPR_ASSERT(h264);
452 H264_CONTEXT_MEDIACODEC* sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
453 WINPR_ASSERT(sys);
454
455 mediacodec_encoder_release(h264->log, sys);
456
457 if (h264->width < MEDIACODEC_MINIMUM_WIDTH || h264->height < MEDIACODEC_MINIMUM_HEIGHT ||
458 h264->width > (UINT32)MEDIACODEC_MAXIMUM_DIMENSION ||
459 h264->height > (UINT32)MEDIACODEC_MAXIMUM_DIMENSION)
460 {
461 WLog_Print(h264->log, WLOG_ERROR, "MediaCodec encoder dimensions out of range [%u,%u]",
462 h264->width, h264->height);
463 return FALSE;
464 }
465
466 const int32_t width = WINPR_ASSERTING_INT_CAST(int32_t, h264->width);
467 const int32_t height = WINPR_ASSERTING_INT_CAST(int32_t, h264->height);
468 const int32_t framerate = mediacodec_rate(h264->FrameRate, MEDIACODEC_DEFAULT_FRAMERATE);
469 const int32_t bitrate = mediacodec_rate(h264->BitRate, MEDIACODEC_DEFAULT_BITRATE);
470
471 sys->encoder = AMediaCodec_createEncoderByType(CODEC_NAME);
472 if (!sys->encoder)
473 {
474 WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_createEncoderByType failed");
475 return FALSE;
476 }
477
478 AMediaFormat* format = AMediaFormat_new();
479 if (!format)
480 {
481 WLog_Print(h264->log, WLOG_ERROR, "AMediaFormat_new failed");
482 mediacodec_encoder_release(h264->log, sys);
483 return FALSE;
484 }
485
486 AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, CODEC_NAME);
487 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_WIDTH, width);
488 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, height);
489 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_COLOR_FORMAT, COLOR_FormatYUV420SemiPlanar);
490 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_BIT_RATE, bitrate);
491 /* Hardware AVC encoders commonly reject CQ, so CQP requests are served as VBR. */
492 if (__builtin_available(android 28, *))
493 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_BITRATE_MODE, MEDIACODEC_BITRATE_MODE_VBR);
494 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_FRAME_RATE, framerate);
495 AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_I_FRAME_INTERVAL, MEDIACODEC_IFRAME_INTERVAL);
496 AMediaFormat_setInt32(format, MEDIACODEC_KEY_PREPEND_HEADER, 1);
497
498 media_status_t status = AMediaCodec_configure(sys->encoder, format, nullptr, nullptr,
499 AMEDIACODEC_CONFIGURE_FLAG_ENCODE);
500 mediacodec_warn(h264->log, AMediaFormat_delete(format), "AMediaFormat_delete");
501 if (status != AMEDIA_OK)
502 {
503 WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_configure (encoder) failed: %d", status);
504 mediacodec_encoder_release(h264->log, sys);
505 return FALSE;
506 }
507
508 status = AMediaCodec_start(sys->encoder);
509 if (status != AMEDIA_OK)
510 {
511 WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_start (encoder) failed: %d", status);
512 mediacodec_encoder_release(h264->log, sys);
513 return FALSE;
514 }
515
516 sys->encoderStarted = TRUE;
517 sys->encoderWidth = width;
518 sys->encoderHeight = height;
519 sys->encoderBitrate = bitrate;
520 sys->syncRequested = FALSE;
521 sys->frameIndex = 0;
522
523 /* Encoder stride/slice-height may be padded. Align stride to even for NV12. */
524 sys->encoderStride = sys->encoderWidth + (sys->encoderWidth & 1);
525 sys->encoderSliceHeight = sys->encoderHeight;
526 if (__builtin_available(android 28, *))
527 {
528 AMediaFormat* inputFormat = AMediaCodec_getInputFormat(sys->encoder);
529 if (inputFormat)
530 {
531 int32_t stride = 0;
532 int32_t sliceHeight = 0;
533 if (AMediaFormat_getInt32(inputFormat, AMEDIAFORMAT_KEY_STRIDE, &stride) &&
534 (stride >= sys->encoderWidth))
535 sys->encoderStride = stride;
536 if (AMediaFormat_getInt32(inputFormat, AMEDIAFORMAT_KEY_SLICE_HEIGHT, &sliceHeight) &&
537 (sliceHeight >= sys->encoderHeight))
538 sys->encoderSliceHeight = sliceHeight;
539 mediacodec_warn(h264->log, AMediaFormat_delete(inputFormat), "AMediaFormat_delete");
540 }
541 }
542
543 if ((sys->encoderStride > MEDIACODEC_MAXIMUM_DIMENSION) ||
544 (sys->encoderSliceHeight > MEDIACODEC_MAXIMUM_DIMENSION))
545 {
546 WLog_Print(h264->log, WLOG_ERROR,
547 "MediaCodec encoder input layout out of range: stride %d slice-height %d",
548 sys->encoderStride, sys->encoderSliceHeight);
549 mediacodec_encoder_release(h264->log, sys);
550 return FALSE;
551 }
552
553 WLog_Print(
554 h264->log, WLOG_DEBUG,
555 "MediaCodec H.264 encoder started: %ux%u @ %d fps, %d bps, stride %d, slice-height %d",
556 h264->width, h264->height, framerate, bitrate, sys->encoderStride, sys->encoderSliceHeight);
557 return TRUE;
558}
559
560/* Copy a planar I420 source frame into the encoder's NV12 (COLOR_FormatYUV420SemiPlanar) input,
561 * honoring the destination stride and slice height. */
562WINPR_ATTR_NODISCARD
563static BOOL mediacodec_copy_i420_to_nv12(uint8_t* dst, size_t dstCapacity, const BYTE** pSrcYuv,
564 const UINT32* pStride, int32_t width, int32_t height,
565 int32_t dstStride, int32_t dstSliceHeight)
566{
567 WINPR_ASSERT(dst);
568 WINPR_ASSERT(pSrcYuv);
569 WINPR_ASSERT(pStride);
570
571 const int32_t cw = (width + 1) / 2;
572 const int32_t ch = (height + 1) / 2;
573 const size_t ySize = (size_t)dstStride * dstSliceHeight;
574
575 if (dstCapacity < ySize + (size_t)dstStride * ((dstSliceHeight + 1) / 2))
576 return FALSE;
577
578 for (int32_t row = 0; row < height; row++)
579 memcpy(dst + (size_t)row * dstStride, pSrcYuv[0] + (size_t)row * pStride[0], (size_t)width);
580
581 /* Clamp pairs to prevent overflow on odd width. */
582 const int32_t pairs = (dstStride / 2 < cw) ? dstStride / 2 : cw;
583 uint8_t* uv = dst + ySize;
584 for (int32_t row = 0; row < ch; row++)
585 {
586 const BYTE* uRow = pSrcYuv[1] + (size_t)row * pStride[1];
587 const BYTE* vRow = pSrcYuv[2] + (size_t)row * pStride[2];
588 uint8_t* uvRow = uv + (size_t)row * dstStride;
589 for (int32_t col = 0; col < pairs; col++)
590 {
591 uvRow[(size_t)col * 2] = uRow[col];
592 uvRow[(size_t)col * 2 + 1] = vRow[col];
593 }
594 }
595
596 return TRUE;
597}
598
599/* Check if an SPS NAL appears before the first coded slice. */
600WINPR_ATTR_NODISCARD
601static BOOL mediacodec_has_sps(const uint8_t* data, size_t size)
602{
603 WINPR_ASSERT(data);
604
605 for (size_t i = 0; i + 3 < size; i++)
606 {
607 if ((data[i] != 0) || (data[i + 1] != 0))
608 continue;
609
610 size_t nal = 0;
611 if (data[i + 2] == 1)
612 nal = i + 3;
613 else if ((data[i + 2] == 0) && (data[i + 3] == 1))
614 nal = i + 4;
615 else
616 continue;
617
618 if (nal >= size)
619 break;
620
621 const uint8_t type = data[nal] & 0x1F;
622 if (type == 7)
623 return TRUE;
624 if ((type == 1) || (type == 5))
625 break; /* coded slice reached, no parameter set in the header */
626
627 i = nal;
628 }
629
630 return FALSE;
631}
632
633static void mediacodec_copy_nv12_to_i420(BYTE* dstYuv[3], const UINT32 dstStride[3],
634 const uint8_t* src, int32_t width, int32_t height,
635 int32_t srcStride, int32_t srcSliceHeight,
636 int32_t cropLeft, int32_t cropTop)
637{
638 WINPR_ASSERT(dstYuv);
639 WINPR_ASSERT(dstStride);
640 WINPR_ASSERT(src);
641
642 const int32_t cw = (width + 1) / 2;
643 const int32_t ch = (height + 1) / 2;
644 const size_t srcLumaOffset = (size_t)cropTop * (size_t)srcStride + (size_t)cropLeft;
645 const size_t srcChromaOffset = (size_t)srcStride * (size_t)srcSliceHeight +
646 (size_t)(cropTop / 2) * (size_t)srcStride +
647 ((size_t)cropLeft & ~1ULL);
648
649 const uint8_t* srcY = src + srcLumaOffset;
650 const uint8_t* srcUV = src + srcChromaOffset;
651
652 /* Clamp chroma pair count to row bounds. */
653 const int32_t chromaLeft = cropLeft & ~1;
654 const int32_t avail = (srcStride > chromaLeft) ? (srcStride - chromaLeft) / 2 : 0;
655 const int32_t pairs = (avail < cw) ? avail : cw;
656
657 for (int32_t row = 0; row < height; row++)
658 {
659 memcpy(dstYuv[0] + (size_t)row * (size_t)dstStride[0],
660 srcY + (size_t)row * (size_t)srcStride, (size_t)width);
661 }
662
663 for (int32_t row = 0; row < ch; row++)
664 {
665 const uint8_t* uvRow = srcUV + (size_t)row * (size_t)srcStride;
666 BYTE* uRow = dstYuv[1] + (size_t)row * (size_t)dstStride[1];
667 BYTE* vRow = dstYuv[2] + (size_t)row * (size_t)dstStride[2];
668 for (int32_t col = 0; col < pairs; col++)
669 {
670 uRow[col] = uvRow[(size_t)col * 2];
671 vRow[col] = uvRow[(size_t)col * 2 + 1];
672 }
673
674 /* Fill unused chroma with neutral gray (0x80). */
675 for (int32_t col = pairs; col < cw; col++)
676 {
677 uRow[col] = 0x80;
678 vRow[col] = 0x80;
679 }
680 }
681}
682
683WINPR_ATTR_NODISCARD
684static int mediacodec_compress(H264_CONTEXT* h264, const BYTE** pSrcYuv, const UINT32* pStride,
685 BYTE** ppDstData, UINT32* pDstSize)
686{
687 WINPR_ASSERT(h264);
688 WINPR_ASSERT(pSrcYuv);
689 WINPR_ASSERT(pStride);
690 WINPR_ASSERT(ppDstData);
691 WINPR_ASSERT(pDstSize);
692
693 H264_CONTEXT_MEDIACODEC* sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
694 WINPR_ASSERT(sys);
695
696 if (!pSrcYuv[0] || !pSrcYuv[1] || !pSrcYuv[2])
697 return -1;
698
699 if ((h264->width > (UINT32)MEDIACODEC_MAXIMUM_DIMENSION) ||
700 (h264->height > (UINT32)MEDIACODEC_MAXIMUM_DIMENSION))
701 return -1;
702
703 const int32_t width = WINPR_ASSERTING_INT_CAST(int32_t, h264->width);
704 const int32_t height = WINPR_ASSERTING_INT_CAST(int32_t, h264->height);
705
706 if (!sys->encoderStarted || sys->encoderWidth != width || sys->encoderHeight != height)
707 {
708 if ((width == sys->encoderFailedWidth) && (height == sys->encoderFailedHeight))
709 return -1;
710
711 if (!mediacodec_encoder_configure(h264))
712 {
713 sys->encoderFailedWidth = width;
714 sys->encoderFailedHeight = height;
715 return -1;
716 }
717 sys->encoderFailedWidth = 0;
718 sys->encoderFailedHeight = 0;
719 }
720
721 /* Adapt the encoder bitrate on the fly when the caller changes it; below API 26 it only
722 * takes effect on the next full reconfigure. */
723 const int32_t bitrate = mediacodec_rate(h264->BitRate, sys->encoderBitrate);
724 if (bitrate != sys->encoderBitrate)
725 {
726 if (__builtin_available(android 26, *))
727 {
728 AMediaFormat* params = AMediaFormat_new();
729 if (params)
730 {
731 AMediaFormat_setInt32(params, MEDIACODEC_PARAM_VIDEO_BITRATE, bitrate);
732 if (AMediaCodec_setParameters(sys->encoder, params) == AMEDIA_OK)
733 sys->encoderBitrate = bitrate;
734 mediacodec_warn(h264->log, AMediaFormat_delete(params), "AMediaFormat_delete");
735 }
736 }
737 }
738
739 /* Below API 26 there is no way to force an early IDR; firstLumaFrameDone catches up once
740 * the next natural keyframe arrives. */
741 if (!h264->firstLumaFrameDone && (sys->frameIndex > 0) && !sys->syncRequested)
742 {
743 if (__builtin_available(android 26, *))
744 {
745 AMediaFormat* syncParams = AMediaFormat_new();
746 if (syncParams)
747 {
748 AMediaFormat_setInt32(syncParams, "request-sync", 0);
749 if (AMediaCodec_setParameters(sys->encoder, syncParams) == AMEDIA_OK)
750 sys->syncRequested = TRUE;
751 mediacodec_warn(h264->log, AMediaFormat_delete(syncParams), "AMediaFormat_delete");
752 }
753 }
754 }
755
756 const int32_t dstStride = sys->encoderStride;
757 const int32_t dstSliceHeight = sys->encoderSliceHeight;
758 const size_t frameSize =
759 (size_t)dstStride * dstSliceHeight + (size_t)dstStride * ((dstSliceHeight + 1) / 2);
760
761 /* 1. Feed one raw frame to the encoder. */
762 const ssize_t inIdx =
763 AMediaCodec_dequeueInputBuffer(sys->encoder, MEDIACODEC_INPUT_DEQUEUE_TIMEOUT);
764 if (inIdx < 0)
765 {
766 /* Drain pending output buffer to unblock input queue. */
767 AMediaCodecBufferInfo stale = { 0 };
768 const ssize_t staleIdx = AMediaCodec_dequeueOutputBuffer(sys->encoder, &stale, 0);
769 if (staleIdx >= 0)
770 mediacodec_warn(h264->log,
771 AMediaCodec_releaseOutputBuffer(sys->encoder, (size_t)staleIdx, FALSE),
772 "AMediaCodec_releaseOutputBuffer");
773
774 WLog_Print(h264->log, WLOG_WARN, "MediaCodec encoder has no input buffer, dropping frame");
775 return -1;
776 }
777
778 size_t inCapacity = 0;
779 uint8_t* inBuffer = AMediaCodec_getInputBuffer(sys->encoder, (size_t)inIdx, &inCapacity);
780 if (!inBuffer || inCapacity < frameSize)
781 {
782 WLog_Print(h264->log, WLOG_ERROR, "MediaCodec encoder input buffer too small: %zu < %zu",
783 inCapacity, frameSize);
784 mediacodec_warn(h264->log,
785 AMediaCodec_queueInputBuffer(sys->encoder, (size_t)inIdx, 0, 0, 0, 0),
786 "AMediaCodec_queueInputBuffer");
787 return -1;
788 }
789
790 if (!mediacodec_copy_i420_to_nv12(inBuffer, inCapacity, pSrcYuv, pStride, width, height,
791 dstStride, dstSliceHeight))
792 {
793 WLog_Print(h264->log, WLOG_ERROR, "MediaCodec encoder input buffer too small for layout");
794 mediacodec_warn(h264->log,
795 AMediaCodec_queueInputBuffer(sys->encoder, (size_t)inIdx, 0, 0, 0, 0),
796 "AMediaCodec_queueInputBuffer");
797 return -1;
798 }
799
800 const int32_t framerate = mediacodec_rate(h264->FrameRate, MEDIACODEC_DEFAULT_FRAMERATE);
801 const uint64_t pts = (uint64_t)sys->frameIndex++ * 1000000ULL / (uint64_t)framerate;
802 media_status_t status =
803 AMediaCodec_queueInputBuffer(sys->encoder, (size_t)inIdx, 0, frameSize, pts, 0);
804 if (status != AMEDIA_OK)
805 {
806 WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_queueInputBuffer (encoder) failed: %d",
807 status);
808 return -1;
809 }
810
811 /* 2. Collect one encoded frame; skip standalone codec-config buffer. */
812 for (;;)
813 {
814 AMediaCodecBufferInfo info = { 0 };
815 const ssize_t outIdx =
816 AMediaCodec_dequeueOutputBuffer(sys->encoder, &info, MEDIACODEC_OUTPUT_DEQUEUE_TIMEOUT);
817 if (outIdx >= 0)
818 {
819 size_t outCapacity = 0;
820 uint8_t* outBuffer =
821 AMediaCodec_getOutputBuffer(sys->encoder, (size_t)outIdx, &outCapacity);
822 if (!outBuffer || (info.size < 0) || (info.offset < 0) ||
823 ((size_t)info.offset > outCapacity) ||
824 ((size_t)info.size > outCapacity - (size_t)info.offset))
825 {
826 WLog_Print(h264->log, WLOG_ERROR,
827 "MediaCodec encoder output out of range: offset %d size %d cap %zu",
828 info.offset, info.size, outCapacity);
829 mediacodec_warn(
830 h264->log, AMediaCodec_releaseOutputBuffer(sys->encoder, (size_t)outIdx, FALSE),
831 "AMediaCodec_releaseOutputBuffer");
832 return -1;
833 }
834
835 if (((info.flags & AMEDIACODEC_BUFFER_FLAG_CODEC_CONFIG) != 0) && (info.size > 0))
836 {
837 BYTE* newCfg = (BYTE*)realloc(sys->spsPpsData, (size_t)info.size);
838 if (newCfg)
839 {
840 sys->spsPpsData = newCfg;
841 sys->spsPpsSize = (size_t)info.size;
842 memcpy(sys->spsPpsData, outBuffer + info.offset, (size_t)info.size);
843 }
844 else
845 {
846 WLog_Print(h264->log, WLOG_WARN, "MediaCodec encoder failed to store SPS/PPS");
847 }
848 }
849
850 if (((info.flags & AMEDIACODEC_BUFFER_FLAG_CODEC_CONFIG) != 0) || (info.size == 0))
851 {
852 const media_status_t rc =
853 AMediaCodec_releaseOutputBuffer(sys->encoder, (size_t)outIdx, FALSE);
854 if (rc != AMEDIA_OK)
855 {
856 WLog_Print(h264->log, WLOG_ERROR,
857 "AMediaCodec_releaseOutputBuffer (encoder) failed: %d", rc);
858 return -1;
859 }
860 continue;
861 }
862
863 const BOOL isKeyFrame = (info.flags & AMEDIACODEC_BUFFER_FLAG_KEY_FRAME) != 0;
864 const BOOL prependSps = isKeyFrame && sys->spsPpsData && sys->spsPpsSize > 0 &&
865 !mediacodec_has_sps(outBuffer + info.offset, (size_t)info.size);
866 const size_t totalSize = (size_t)info.size + (prependSps ? sys->spsPpsSize : 0);
867
868 if (sys->outputCapacity < totalSize)
869 {
870 BYTE* resized = (BYTE*)realloc(sys->outputData, totalSize);
871 if (!resized)
872 {
873 mediacodec_warn(
874 h264->log,
875 AMediaCodec_releaseOutputBuffer(sys->encoder, (size_t)outIdx, FALSE),
876 "AMediaCodec_releaseOutputBuffer");
877 return -1;
878 }
879 sys->outputData = resized;
880 sys->outputCapacity = totalSize;
881 }
882
883 if (prependSps)
884 {
885 memcpy(sys->outputData, sys->spsPpsData, sys->spsPpsSize);
886 memcpy(sys->outputData + sys->spsPpsSize, outBuffer + info.offset,
887 (size_t)info.size);
888 }
889 else
890 {
891 memcpy(sys->outputData, outBuffer + info.offset, (size_t)info.size);
892 }
893
894 *ppDstData = sys->outputData;
895 *pDstSize = WINPR_ASSERTING_INT_CAST(UINT32, totalSize);
896 const media_status_t rc =
897 AMediaCodec_releaseOutputBuffer(sys->encoder, (size_t)outIdx, FALSE);
898 if (rc != AMEDIA_OK)
899 WLog_Print(h264->log, WLOG_ERROR,
900 "AMediaCodec_releaseOutputBuffer (encoder) failed: %d", rc);
901 return 1;
902 }
903 else if (outIdx == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED ||
904 outIdx == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED)
905 {
906 continue;
907 }
908 else if (outIdx == AMEDIACODEC_INFO_TRY_AGAIN_LATER)
909 {
910 /* Output not ready yet (pipeline priming). */
911 WLog_Print(h264->log, WLOG_DEBUG,
912 "MediaCodec encoder output not ready, dropping frame");
913 return -1;
914 }
915 else
916 {
917 WLog_Print(h264->log, WLOG_ERROR,
918 "AMediaCodec_dequeueOutputBuffer (encoder) returned %zd", outIdx);
919 return -1;
920 }
921 }
922}
923
924WINPR_ATTR_NODISCARD
925static int mediacodec_decompress(H264_CONTEXT* h264, const BYTE* pSrcData, UINT32 SrcSize)
926{
927 WINPR_ASSERT(h264);
928 WINPR_ASSERT(pSrcData);
929
930 H264_CONTEXT_MEDIACODEC* sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
931 WINPR_ASSERT(sys);
932
933 BYTE** pYUVData = h264->pYUVData;
934 WINPR_ASSERT(pYUVData);
935
936 UINT32* iStride = h264->iStride;
937 WINPR_ASSERT(iStride);
938
939 release_current_outputbuffer(h264);
940
941 if ((h264->width > (UINT32)MEDIACODEC_MAXIMUM_DIMENSION) ||
942 (h264->height > (UINT32)MEDIACODEC_MAXIMUM_DIMENSION))
943 {
944 WLog_Print(h264->log, WLOG_ERROR,
945 "MediaCodec size [%" PRIu32 ",%" PRIu32 "] exceeds the maximum", h264->width,
946 h264->height);
947 return -1;
948 }
949
950 if ((sys->width != WINPR_ASSERTING_INT_CAST(int32_t, h264->width)) ||
951 (sys->height != WINPR_ASSERTING_INT_CAST(int32_t, h264->height)))
952 {
953 sys->width = WINPR_ASSERTING_INT_CAST(int32_t, h264->width);
954 sys->height = WINPR_ASSERTING_INT_CAST(int32_t, h264->height);
955
956 if (sys->width < MEDIACODEC_MINIMUM_WIDTH || sys->height < MEDIACODEC_MINIMUM_HEIGHT)
957 {
958 WLog_Print(h264->log, WLOG_ERROR,
959 "MediaCodec got width or height smaller than minimum [%d,%d]", sys->width,
960 sys->height);
961 return -1;
962 }
963
964 WLog_Print(h264->log, WLOG_DEBUG, "MediaCodec setting new input width and height [%d,%d]",
965 sys->width, sys->height);
966
967 AMediaFormat_setInt32(sys->inputFormat, AMEDIAFORMAT_KEY_WIDTH, sys->width);
968 AMediaFormat_setInt32(sys->inputFormat, AMEDIAFORMAT_KEY_HEIGHT, sys->height);
969 if (__builtin_available(android 26, *))
970 {
971 const media_status_t status = AMediaCodec_setParameters(sys->decoder, sys->inputFormat);
972 if (status != AMEDIA_OK)
973 {
974 /* Decoder adapts via in-band SPS/PPS; ignoring failure here. */
975 WLog_Print(h264->log, WLOG_DEBUG,
976 "AMediaCodec_setParameters returned %d, relying on "
977 "in-band SPS/PPS for the size change",
978 status);
979 }
980 }
981 else
982 {
983 WLog_Print(h264->log, WLOG_DEBUG,
984 "AMediaCodec_setParameters unavailable below API 26, relying on "
985 "in-band SPS/PPS for the size change");
986 }
987
988 /* The codec can change output width and height */
989 if (update_mediacodec_outputformat(h264) < 0)
990 {
991 WLog_Print(h264->log, WLOG_ERROR, "MediaCodec failed updating input format");
992 return -1;
993 }
994 }
995
996 while (true)
997 {
998 UINT32 inputBufferCurrnetOffset = 0;
999 int inputRetries = 0;
1000 while (inputBufferCurrnetOffset < SrcSize)
1001 {
1002 UINT32 numberOfBytesToCopy = SrcSize - inputBufferCurrnetOffset;
1003 const ssize_t inputBufferId =
1004 AMediaCodec_dequeueInputBuffer(sys->decoder, MEDIACODEC_INPUT_DEQUEUE_TIMEOUT);
1005 if (inputBufferId < 0)
1006 {
1007 if (++inputRetries > MEDIACODEC_DECODE_MAX_RETRIES)
1008 {
1009 WLog_Print(h264->log, WLOG_ERROR,
1010 "AMediaCodec_dequeueInputBuffer gave up [%zd]", inputBufferId);
1011 return -1;
1012 }
1013 continue;
1014 }
1015 inputRetries = 0;
1016
1017 size_t inputBufferSize = 0;
1018 uint8_t* inputBuffer =
1019 AMediaCodec_getInputBuffer(sys->decoder, inputBufferId, &inputBufferSize);
1020 if (inputBuffer == nullptr)
1021 {
1022 WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_getInputBuffer failed");
1023 return -1;
1024 }
1025
1026 if (numberOfBytesToCopy > inputBufferSize)
1027 {
1028 WLog_Print(h264->log, WLOG_WARN,
1029 "MediaCodec inputBufferSize: got [%zu] but wanted [%u]", inputBufferSize,
1030 numberOfBytesToCopy);
1031 numberOfBytesToCopy = WINPR_ASSERTING_INT_CAST(UINT32, inputBufferSize);
1032 }
1033
1034 memcpy(inputBuffer, &pSrcData[inputBufferCurrnetOffset], numberOfBytesToCopy);
1035 inputBufferCurrnetOffset += numberOfBytesToCopy;
1036
1037 const media_status_t status = AMediaCodec_queueInputBuffer(
1038 sys->decoder, inputBufferId, 0, numberOfBytesToCopy, 0, 0);
1039 if (status != AMEDIA_OK)
1040 {
1041 WLog_Print(h264->log, WLOG_ERROR, "Error AMediaCodec_queueInputBuffer %d", status);
1042 return -1;
1043 }
1044 }
1045
1046 int outputRetries = 0;
1047 while (true)
1048 {
1049 AMediaCodecBufferInfo bufferInfo = WINPR_C_ARRAY_INIT;
1050 ssize_t outputBufferId = AMediaCodec_dequeueOutputBuffer(
1051 sys->decoder, &bufferInfo, MEDIACODEC_OUTPUT_DEQUEUE_TIMEOUT);
1052 if (outputBufferId >= 0)
1053 {
1054 sys->currentOutputBufferIndex = outputBufferId;
1055
1056 size_t outputBufferSize = 0;
1057 uint8_t* outputBuffer = nullptr;
1058 outputBuffer =
1059 AMediaCodec_getOutputBuffer(sys->decoder, outputBufferId, &outputBufferSize);
1060 sys->currentOutputBufferIndex = outputBufferId;
1061
1062 /* End of stream and metadata buffers carry no picture. */
1063 if (bufferInfo.size <= 0)
1064 {
1065 release_current_outputbuffer(h264);
1066 if (++outputRetries > MEDIACODEC_DECODE_MAX_RETRIES)
1067 return 0;
1068 continue;
1069 }
1070
1071 /* The picture starts at the reported offset, not at the buffer base. */
1072 if (!outputBuffer || (bufferInfo.offset < 0) ||
1073 ((size_t)bufferInfo.offset > outputBufferSize))
1074 {
1075 WLog_Print(h264->log, WLOG_ERROR,
1076 "MediaCodec output offset %d out of range %zu", bufferInfo.offset,
1077 outputBufferSize);
1078 return -1;
1079 }
1080 outputBuffer += bufferInfo.offset;
1081 outputBufferSize -= (size_t)bufferInfo.offset;
1082
1083 const size_t lumaSize = (size_t)sys->outputStride * (size_t)sys->outputSliceHeight;
1084 const int32_t chromaStride = sys->outputStride / 2;
1085 const int32_t chromaHeight = (sys->outputSliceHeight + 1) / 2;
1086 const size_t chromaSize = (size_t)chromaStride * (size_t)chromaHeight;
1087 const size_t minRequiredSize =
1088 lumaSize + (size_t)sys->outputStride * (size_t)chromaHeight;
1089
1090 if (outputBufferSize < minRequiredSize)
1091 {
1092 WLog_Print(h264->log, WLOG_ERROR,
1093 "Error MediaCodec output buffer too small %zu < %zu",
1094 outputBufferSize, minRequiredSize);
1095 return -1;
1096 }
1097
1098 if (sys->colorFormat == COLOR_FormatYUV420Planar)
1099 {
1100 iStride[0] = (UINT32)sys->outputStride;
1101 iStride[1] = (UINT32)chromaStride;
1102 iStride[2] = (UINT32)chromaStride;
1103 pYUVData[0] = outputBuffer +
1104 ((size_t)sys->cropTop * (size_t)sys->outputStride) +
1105 (size_t)sys->cropLeft;
1106 pYUVData[1] = outputBuffer + lumaSize +
1107 ((size_t)(sys->cropTop / 2) * (size_t)chromaStride) +
1108 ((size_t)sys->cropLeft / 2);
1109 pYUVData[2] = outputBuffer + lumaSize + chromaSize +
1110 ((size_t)(sys->cropTop / 2) * (size_t)chromaStride) +
1111 ((size_t)sys->cropLeft / 2);
1112 }
1113 else
1114 {
1115 /* SemiPlanar (NV12) or flexible YUV: deinterleave into staging I420 buffer */
1116 const UINT32 dstYStride = (UINT32)sys->outputWidth;
1117 const UINT32 dstUVStride = (UINT32)((sys->outputWidth + 1) / 2);
1118 const size_t yBytes = (size_t)dstYStride * (size_t)sys->outputHeight;
1119 const size_t uvBytes =
1120 (size_t)dstUVStride * (size_t)((sys->outputHeight + 1) / 2);
1121 const size_t neededCapacity = yBytes + uvBytes * 2;
1122
1123 if (sys->decodeI420Capacity < neededCapacity)
1124 {
1125 BYTE* newBuf = (BYTE*)realloc(sys->decodeI420Buffer, neededCapacity);
1126 if (!newBuf)
1127 {
1128 WLog_Print(h264->log, WLOG_ERROR,
1129 "Failed to allocate I420 deinterleave buffer");
1130 return -1;
1131 }
1132 sys->decodeI420Buffer = newBuf;
1133 sys->decodeI420Capacity = neededCapacity;
1134 }
1135
1136 iStride[0] = dstYStride;
1137 iStride[1] = dstUVStride;
1138 iStride[2] = dstUVStride;
1139 pYUVData[0] = sys->decodeI420Buffer;
1140 pYUVData[1] = sys->decodeI420Buffer + yBytes;
1141 pYUVData[2] = sys->decodeI420Buffer + yBytes + uvBytes;
1142
1143 mediacodec_copy_nv12_to_i420(
1144 pYUVData, iStride, outputBuffer, sys->outputWidth, sys->outputHeight,
1145 sys->outputStride, sys->outputSliceHeight, sys->cropLeft, sys->cropTop);
1146 release_current_outputbuffer(h264);
1147 }
1148
1149 h264->YUVWidth = (UINT32)sys->outputWidth;
1150 h264->YUVHeight = (UINT32)sys->outputHeight;
1151 break;
1152 }
1153 else if (outputBufferId == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED)
1154 {
1155 if (update_mediacodec_outputformat(h264) < 0)
1156 {
1157 WLog_Print(h264->log, WLOG_ERROR,
1158 "MediaCodec failed updating output format in decompress");
1159 return -1;
1160 }
1161 }
1162 else if (outputBufferId == AMEDIACODEC_INFO_TRY_AGAIN_LATER)
1163 {
1164 if (++outputRetries > MEDIACODEC_DECODE_MAX_RETRIES)
1165 {
1166 WLog_Print(h264->log, WLOG_DEBUG,
1167 "AMediaCodec_dequeueOutputBuffer no buffer ready yet");
1168 return 0;
1169 }
1170 }
1171 else if (outputBufferId == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED)
1172 {
1173 WLog_Print(h264->log, WLOG_WARN,
1174 "AMediaCodec_dequeueOutputBuffer returned deprecated value "
1175 "AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED, ignoring");
1176 }
1177 else
1178 {
1179 WLog_Print(h264->log, WLOG_ERROR,
1180 "AMediaCodec_dequeueOutputBuffer returned unknown value [%zd]",
1181 outputBufferId);
1182 return -1;
1183 }
1184 }
1185
1186 break;
1187 }
1188
1189 return 1;
1190}
1191
1192static void mediacodec_uninit(H264_CONTEXT* h264)
1193{
1194 media_status_t status = AMEDIA_OK;
1195 H264_CONTEXT_MEDIACODEC* sys = nullptr;
1196
1197 WINPR_ASSERT(h264);
1198
1199 sys = (H264_CONTEXT_MEDIACODEC*)h264->pSystemData;
1200
1201 WLog_Print(h264->log, WLOG_DEBUG, "Uninitializing MediaCodec");
1202
1203 if (!sys)
1204 return;
1205
1206 if (sys->decoder != nullptr)
1207 {
1208 release_current_outputbuffer(h264);
1209
1210 if (sys->decoderStarted)
1211 {
1212 status = AMediaCodec_stop(sys->decoder);
1213 if (status != AMEDIA_OK)
1214 {
1215 WLog_Print(h264->log, WLOG_ERROR, "Error AMediaCodec_stop %d", status);
1216 }
1217 sys->decoderStarted = FALSE;
1218 }
1219
1220 status = AMediaCodec_delete(sys->decoder);
1221 if (status != AMEDIA_OK)
1222 {
1223 WLog_Print(h264->log, WLOG_ERROR, "Error AMediaCodec_delete %d", status);
1224 }
1225
1226 sys->decoder = nullptr;
1227 }
1228
1229 mediacodec_encoder_release(h264->log, sys);
1230 free(sys->outputData);
1231 sys->outputData = nullptr;
1232 free(sys->decodeI420Buffer);
1233 sys->decodeI420Buffer = nullptr;
1234 free(sys->spsPpsData);
1235 sys->spsPpsData = nullptr;
1236
1237 set_mediacodec_format(h264, &sys->inputFormat, nullptr);
1238 set_mediacodec_format(h264, &sys->outputFormat, nullptr);
1239
1240 free(sys);
1241 h264->pSystemData = nullptr;
1242}
1243
1244WINPR_ATTR_NODISCARD
1245static BOOL mediacodec_init(H264_CONTEXT* h264)
1246{
1247 H264_CONTEXT_MEDIACODEC* sys = nullptr;
1248 media_status_t status = AMEDIA_OK;
1249
1250 WINPR_ASSERT(h264);
1251
1252 WLog_Print(h264->log, WLOG_DEBUG, "Initializing MediaCodec");
1253
1254 sys = (H264_CONTEXT_MEDIACODEC*)calloc(1, sizeof(H264_CONTEXT_MEDIACODEC));
1255
1256 if (!sys)
1257 {
1258 goto EXCEPTION;
1259 }
1260
1261 h264->pSystemData = (void*)sys;
1262
1263 sys->currentOutputBufferIndex = -1;
1264
1265 /* Updated when given height and width for the first time */
1266 sys->width = sys->outputWidth = MEDIACODEC_MINIMUM_WIDTH;
1267 sys->height = sys->outputHeight = MEDIACODEC_MINIMUM_HEIGHT;
1268
1269 /* Probe encoder support to allow fallback during initialization. */
1270 if (h264->Compressor)
1271 {
1272 if (!mediacodec_encoder_probe(h264->log))
1273 goto EXCEPTION;
1274 WLog_Print(h264->log, WLOG_DEBUG, "MediaCodec H.264 encoder available");
1275 return TRUE;
1276 }
1277
1278 sys->decoder = AMediaCodec_createDecoderByType(CODEC_NAME);
1279 if (sys->decoder == nullptr)
1280 {
1281 WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_createCodecByName failed");
1282 goto EXCEPTION;
1283 }
1284
1285 /* Codec name is purely informational; skip logging it below that instead of failing init. */
1286 if (__builtin_available(android 28, *))
1287 {
1288 char* codec_name = nullptr;
1289 status = AMediaCodec_getName(sys->decoder, &codec_name);
1290 if (status != AMEDIA_OK)
1291 {
1292 WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_getName failed: %d", status);
1293 goto EXCEPTION;
1294 }
1295
1296 WLog_Print(h264->log, WLOG_DEBUG, "MediaCodec using %s codec [%s]", CODEC_NAME, codec_name);
1297 AMediaCodec_releaseName(sys->decoder, codec_name);
1298 }
1299 else
1300 WLog_Print(h264->log, WLOG_DEBUG, "MediaCodec using %s codec", CODEC_NAME);
1301
1302 set_mediacodec_format(h264, &sys->inputFormat,
1303 mediacodec_format_new(h264->log, sys->width, sys->height));
1304
1305 status = AMediaCodec_configure(sys->decoder, sys->inputFormat, nullptr, nullptr, 0);
1306 if (status != AMEDIA_OK)
1307 {
1308 WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_configure failed: %d", status);
1309 goto EXCEPTION;
1310 }
1311
1312 if (update_mediacodec_inputformat(h264) < 0)
1313 {
1314 WLog_Print(h264->log, WLOG_ERROR, "MediaCodec failed updating input format");
1315 goto EXCEPTION;
1316 }
1317
1318 if (update_mediacodec_outputformat(h264) < 0)
1319 {
1320 WLog_Print(h264->log, WLOG_ERROR, "MediaCodec failed updating output format");
1321 goto EXCEPTION;
1322 }
1323
1324 WLog_Print(h264->log, WLOG_DEBUG, "Starting MediaCodec");
1325 status = AMediaCodec_start(sys->decoder);
1326 if (status != AMEDIA_OK)
1327 {
1328 WLog_Print(h264->log, WLOG_ERROR, "AMediaCodec_start failed %d", status);
1329 goto EXCEPTION;
1330 }
1331
1332 sys->decoderStarted = TRUE;
1333 return TRUE;
1334EXCEPTION:
1335 mediacodec_uninit(h264);
1336 return FALSE;
1337}
1338
1339const H264_CONTEXT_SUBSYSTEM g_Subsystem_mediacodec = { "MediaCodec", mediacodec_init,
1340 mediacodec_uninit, mediacodec_decompress,
1341 mediacodec_compress };