21#include <freerdp/config.h>
24#include <winpr/print.h>
25#include <winpr/library.h>
26#include <winpr/bitstream.h>
27#include <winpr/synch.h>
29#include <freerdp/primitives.h>
30#include <freerdp/codec/h264.h>
31#include <freerdp/codec/yuv.h>
32#include <freerdp/log.h>
36#define TAG FREERDP_TAG("codec")
38static BOOL avc444_ensure_buffer(H264_CONTEXT* h264, DWORD nDstHeight);
40static BOOL yuv_ensure_buffer(H264_CONTEXT* h264, UINT32 stride, UINT32 width, UINT32 height)
43 UINT32 pheight = height;
52 stride += 16 - stride % 16;
54 if (pheight % 16 != 0)
55 pheight += 16 - pheight % 16;
57 const size_t nPlanes = h264->hwAccel ? 2 : 3;
59 for (
size_t x = 0; x < nPlanes; x++)
61 if (!h264->pYUVData[x] || !h264->pOldYUVData[x])
70 if (isNull || (width != h264->width) || (height != h264->height) ||
71 (stride != h264->iStride[0]))
75 h264->iStride[0] = stride;
76 h264->iStride[1] = stride;
81 h264->iStride[0] = stride;
82 h264->iStride[1] = (stride + 1) / 2;
83 h264->iStride[2] = (stride + 1) / 2;
87 h264->height = height;
89 for (
size_t x = 0; x < nPlanes; x++)
91 BYTE* tmp1 = winpr_aligned_recalloc(h264->pYUVData[x], h264->iStride[x], pheight, 16);
93 winpr_aligned_recalloc(h264->pOldYUVData[x], h264->iStride[x], pheight, 16);
95 h264->pYUVData[x] = tmp1;
97 h264->pOldYUVData[x] = tmp2;
106BOOL avc420_ensure_buffer(H264_CONTEXT* h264, UINT32 stride, UINT32 width, UINT32 height)
108 return yuv_ensure_buffer(h264, stride, width, height);
111INT32 avc420_decompress(H264_CONTEXT* h264,
const BYTE* pSrcData, UINT32 SrcSize, BYTE* pDstData,
112 DWORD DstFormat, UINT32 nDstStep, WINPR_ATTR_UNUSED UINT32 nDstWidth,
113 WINPR_ATTR_UNUSED UINT32 nDstHeight,
const RECTANGLE_16* regionRects,
114 UINT32 numRegionRects)
117 const BYTE* pYUVData[3];
119 if (!h264 || h264->Compressor)
122 status = h264->subsystem->Decompress(h264, pSrcData, SrcSize);
130 pYUVData[0] = h264->pYUVData[0];
131 pYUVData[1] = h264->pYUVData[1];
132 pYUVData[2] = h264->pYUVData[2];
133 if (!yuv420_context_decode(h264->yuv, pYUVData, h264->iStride, h264->height, DstFormat,
134 pDstData, nDstStep, regionRects, numRegionRects))
140static BOOL allocate_h264_metablock(UINT32 QP,
RECTANGLE_16* rectangles,
144 if (!meta || (QP > UINT8_MAX))
150 meta->regionRects = rectangles;
154 if (count > UINT32_MAX)
159 if (!meta->quantQualityVals || !meta->regionRects)
161 meta->numRegionRects = (UINT32)count;
162 for (
size_t x = 0; x < count; x++)
169 cur->qualityVal = 100 - (QP & 0x3F);
174static inline BOOL diff_tile(
const RECTANGLE_16* regionRect, BYTE* pYUVData[3],
175 BYTE* pOldYUVData[3], UINT32
const iStride[3])
179 if (!regionRect || !pYUVData || !pOldYUVData || !iStride)
181 size = regionRect->right - regionRect->left;
182 if (regionRect->right > iStride[0])
184 if (regionRect->right / 2u > iStride[1])
186 if (regionRect->right / 2u > iStride[2])
189 for (
size_t y = regionRect->top; y < regionRect->bottom; y++)
191 const BYTE* cur0 = &pYUVData[0][y * iStride[0]];
192 const BYTE* cur1 = &pYUVData[1][y * iStride[1]];
193 const BYTE* cur2 = &pYUVData[2][y * iStride[2]];
194 const BYTE* old0 = &pOldYUVData[0][y * iStride[0]];
195 const BYTE* old1 = &pOldYUVData[1][y * iStride[1]];
196 const BYTE* old2 = &pOldYUVData[2][y * iStride[2]];
198 if (memcmp(&cur0[regionRect->left], &old0[regionRect->left], size) != 0)
200 if (memcmp(&cur1[regionRect->left / 2], &old1[regionRect->left / 2], size / 2) != 0)
202 if (memcmp(&cur2[regionRect->left / 2], &old2[regionRect->left / 2], size / 2) != 0)
208static BOOL detect_changes(BOOL firstFrameDone,
const UINT32 QP,
const RECTANGLE_16* regionRect,
209 BYTE* pYUVData[3], BYTE* pOldYUVData[3], UINT32
const iStride[3],
217 if (!regionRect || !pYUVData || !pOldYUVData || !iStride || !meta)
220 wc = (regionRect->right - regionRect->left) / 64 + 1;
221 hc = (regionRect->bottom - regionRect->top) / 64 + 1;
227 rectangles[0] = *regionRect;
232 for (
size_t y = regionRect->top; y < regionRect->bottom; y += 64)
234 for (
size_t x = regionRect->left; x < regionRect->right; x += 64)
237 rect.left = (UINT16)MIN(UINT16_MAX, regionRect->left + x);
238 rect.top = (UINT16)MIN(UINT16_MAX, regionRect->top + y);
240 (UINT16)MIN(UINT16_MAX, MIN(regionRect->left + x + 64, regionRect->right));
242 (UINT16)MIN(UINT16_MAX, MIN(regionRect->top + y + 64, regionRect->bottom));
243 if (diff_tile(&rect, pYUVData, pOldYUVData, iStride))
244 rectangles[count++] = rect;
248 if (!allocate_h264_metablock(QP, rectangles, meta, count))
253INT32 h264_get_yuv_buffer(H264_CONTEXT* h264, UINT32 nSrcStride, UINT32 nSrcWidth,
254 UINT32 nSrcHeight, BYTE* YUVData[3], UINT32 stride[3])
256 if (!h264 || !h264->Compressor || !h264->subsystem || !h264->subsystem->Compress)
259 if (!yuv_ensure_buffer(h264, nSrcStride, nSrcWidth, nSrcHeight))
262 for (
size_t x = 0; x < 3; x++)
264 YUVData[x] = h264->pYUVData[x];
265 stride[x] = h264->iStride[x];
271INT32 h264_compress(H264_CONTEXT* h264, BYTE** ppDstData, UINT32* pDstSize)
273 if (!h264 || !h264->Compressor || !h264->subsystem || !h264->subsystem->Compress)
276 const BYTE* pcYUVData[3] = { h264->pYUVData[0], h264->pYUVData[1], h264->pYUVData[2] };
278 return h264->subsystem->Compress(h264, pcYUVData, h264->iStride, ppDstData, pDstSize);
281INT32 avc420_compress(H264_CONTEXT* h264,
const BYTE* pSrcData, DWORD SrcFormat, UINT32 nSrcStep,
282 UINT32 nSrcWidth, UINT32 nSrcHeight,
const RECTANGLE_16* regionRect,
286 BYTE* pYUVData[3] = WINPR_C_ARRAY_INIT;
287 const BYTE* pcYUVData[3] = WINPR_C_ARRAY_INIT;
288 BYTE* pOldYUVData[3] = WINPR_C_ARRAY_INIT;
290 if (!h264 || !regionRect || !meta || !h264->Compressor)
293 if (!h264->subsystem->Compress)
296 if (!avc420_ensure_buffer(h264, nSrcStep, nSrcWidth, nSrcHeight))
299 if (h264->encodingBuffer)
301 for (
size_t x = 0; x < 3; x++)
303 pYUVData[x] = h264->pYUVData[x];
304 pOldYUVData[x] = h264->pOldYUVData[x];
309 for (
size_t x = 0; x < 3; x++)
311 pYUVData[x] = h264->pOldYUVData[x];
312 pOldYUVData[x] = h264->pYUVData[x];
315 h264->encodingBuffer = !h264->encodingBuffer;
317 if (!yuv420_context_encode(h264->yuv, pSrcData, nSrcStep, SrcFormat, h264->iStride, pYUVData,
321 if (!detect_changes(h264->firstLumaFrameDone, h264->QP, regionRect, pYUVData, pOldYUVData,
322 h264->iStride, meta))
325 if (meta->numRegionRects == 0)
331 for (
size_t x = 0; x < 3; x++)
332 pcYUVData[x] = pYUVData[x];
334 rc = h264->subsystem->Compress(h264, pcYUVData, h264->iStride, ppDstData, pDstSize);
336 h264->firstLumaFrameDone = TRUE;
340 free_h264_metablock(meta);
344INT32 avc444_compress(H264_CONTEXT* h264,
const BYTE* pSrcData, DWORD SrcFormat, UINT32 nSrcStep,
345 UINT32 nSrcWidth, UINT32 nSrcHeight, BYTE version,
const RECTANGLE_16* region,
346 BYTE* op, BYTE** ppDstData, UINT32* pDstSize, BYTE** ppAuxDstData,
351 BYTE* coded =
nullptr;
352 UINT32 codedSize = 0;
353 BYTE** pYUV444Data =
nullptr;
354 BYTE** pOldYUV444Data =
nullptr;
355 BYTE** pYUVData =
nullptr;
356 BYTE** pOldYUVData =
nullptr;
358 if (!h264 || !h264->Compressor)
361 if (!h264->subsystem->Compress)
364 if (!avc420_ensure_buffer(h264, nSrcStep, nSrcWidth, nSrcHeight))
367 if (!avc444_ensure_buffer(h264, nSrcHeight))
370 if (h264->encodingBuffer)
372 pYUV444Data = h264->pOldYUV444Data;
373 pOldYUV444Data = h264->pYUV444Data;
374 pYUVData = h264->pOldYUVData;
375 pOldYUVData = h264->pYUVData;
379 pYUV444Data = h264->pYUV444Data;
380 pOldYUV444Data = h264->pOldYUV444Data;
381 pYUVData = h264->pYUVData;
382 pOldYUVData = h264->pOldYUVData;
384 h264->encodingBuffer = !h264->encodingBuffer;
386 if (!yuv444_context_encode(h264->yuv, version, pSrcData, nSrcStep, SrcFormat, h264->iStride,
387 pYUV444Data, pYUVData, region, 1))
390 if (!detect_changes(h264->firstLumaFrameDone, h264->QP, region, pYUV444Data, pOldYUV444Data,
391 h264->iStride, meta))
393 if (!detect_changes(h264->firstChromaFrameDone, h264->QP, region, pYUVData, pOldYUVData,
394 h264->iStride, auxMeta))
403 if ((meta->numRegionRects > 0) && (auxMeta->numRegionRects > 0))
405 else if (meta->numRegionRects > 0)
407 else if (auxMeta->numRegionRects > 0)
411 WLog_INFO(TAG,
"no changes detected for luma or chroma frame");
416 if ((*op == 0) || (*op == 1))
418 const BYTE* pcYUV444Data[3] = { pYUV444Data[0], pYUV444Data[1], pYUV444Data[2] };
420 if (h264->subsystem->Compress(h264, pcYUV444Data, h264->iStride, &coded, &codedSize) < 0)
422 h264->firstLumaFrameDone = TRUE;
423 memcpy(h264->lumaData, coded, codedSize);
424 *ppDstData = h264->lumaData;
425 *pDstSize = codedSize;
428 if ((*op == 0) || (*op == 2))
430 const BYTE* pcYUVData[3] = { pYUVData[0], pYUVData[1], pYUVData[2] };
432 if (h264->subsystem->Compress(h264, pcYUVData, h264->iStride, &coded, &codedSize) < 0)
434 h264->firstChromaFrameDone = TRUE;
435 *ppAuxDstData = coded;
436 *pAuxDstSize = codedSize;
443 free_h264_metablock(meta);
444 free_h264_metablock(auxMeta);
449static BOOL avc444_ensure_buffer(H264_CONTEXT* h264, DWORD nDstHeight)
453 const UINT32* piMainStride = h264->iStride;
454 UINT32* piDstSize = h264->iYUV444Size;
455 UINT32* piDstStride = h264->iYUV444Stride;
456 BYTE** ppYUVDstData = h264->pYUV444Data;
457 BYTE** ppOldYUVDstData = h264->pOldYUV444Data;
459 nDstHeight = MAX(h264->height, nDstHeight);
460 const UINT32 pad = nDstHeight % 16;
461 UINT32 padDstHeight = nDstHeight;
464 padDstHeight += 16 - pad;
466 if ((piMainStride[0] != piDstStride[0]) ||
467 (piDstSize[0] != 1ull * piMainStride[0] * padDstHeight))
469 for (UINT32 x = 0; x < 3; x++)
471 piDstStride[x] = piMainStride[0];
472 piDstSize[x] = piDstStride[x] * padDstHeight;
474 if (piDstSize[x] == 0)
477 BYTE* tmp1 = winpr_aligned_recalloc(ppYUVDstData[x], piDstSize[x], 1, 16);
480 ppYUVDstData[x] = tmp1;
481 BYTE* tmp2 = winpr_aligned_recalloc(ppOldYUVDstData[x], piDstSize[x], 1, 16);
484 ppOldYUVDstData[x] = tmp2;
488 BYTE* tmp = winpr_aligned_recalloc(h264->lumaData, piDstSize[0], 4, 16);
491 h264->lumaData = tmp;
495 for (UINT32 x = 0; x < 3; x++)
497 if (!ppOldYUVDstData[x] || !ppYUVDstData[x] || (piDstSize[x] == 0) || (piDstStride[x] == 0))
499 WLog_Print(h264->log, WLOG_ERROR,
500 "YUV buffer not initialized! check your decoder settings");
513static BOOL avc444_process_rects(H264_CONTEXT* h264,
const BYTE* pSrcData, UINT32 SrcSize,
514 BYTE* pDstData, UINT32 DstFormat, UINT32 nDstStep,
515 WINPR_ATTR_UNUSED UINT32 nDstWidth, UINT32 nDstHeight,
516 const RECTANGLE_16* rects, UINT32 nrRects, avc444_frame_type type)
518 const BYTE* pYUVData[3];
519 BYTE* pYUVDstData[3];
520 UINT32* piDstStride = h264->iYUV444Stride;
521 BYTE** ppYUVDstData = h264->pYUV444Data;
522 const UINT32* piStride = h264->iStride;
524 if (h264->subsystem->Decompress(h264, pSrcData, SrcSize) < 0)
527 pYUVData[0] = h264->pYUVData[0];
528 pYUVData[1] = h264->pYUVData[1];
529 pYUVData[2] = h264->pYUVData[2];
530 if (!avc444_ensure_buffer(h264, nDstHeight))
533 pYUVDstData[0] = ppYUVDstData[0];
534 pYUVDstData[1] = ppYUVDstData[1];
535 pYUVDstData[2] = ppYUVDstData[2];
536 return (yuv444_context_decode(h264->yuv, (BYTE)type, pYUVData, piStride, h264->height,
537 pYUVDstData, piDstStride, DstFormat, pDstData, nDstStep, rects,
541#if defined(AVC444_FRAME_STAT)
542static UINT64 op1 = 0;
543static double op1sum = 0;
544static UINT64 op2 = 0;
545static double op2sum = 0;
546static UINT64 op3 = 0;
547static double op3sum = 0;
548static double avg(UINT64* count,
double old,
double size)
550 double tmp = size + *count * old;
557INT32 avc444_decompress(H264_CONTEXT* h264, BYTE op,
const RECTANGLE_16* regionRects,
558 UINT32 numRegionRects,
const BYTE* pSrcData, UINT32 SrcSize,
559 const RECTANGLE_16* auxRegionRects, UINT32 numAuxRegionRect,
560 const BYTE* pAuxSrcData, UINT32 AuxSrcSize, BYTE* pDstData, DWORD DstFormat,
561 UINT32 nDstStep, UINT32 nDstWidth, UINT32 nDstHeight, UINT32 codecId)
564 avc444_frame_type chroma =
565 (codecId == RDPGFX_CODECID_AVC444) ? AVC444_CHROMAv1 : AVC444_CHROMAv2;
567 if (!h264 || !regionRects || !pSrcData || !pDstData || h264->Compressor)
574 if (!avc444_process_rects(h264, pSrcData, SrcSize, pDstData, DstFormat, nDstStep,
575 nDstWidth, nDstHeight, regionRects, numRegionRects,
578 else if (!avc444_process_rects(h264, pAuxSrcData, AuxSrcSize, pDstData, DstFormat,
579 nDstStep, nDstWidth, nDstHeight, auxRegionRects,
580 numAuxRegionRect, chroma))
588 if (!avc444_process_rects(h264, pSrcData, SrcSize, pDstData, DstFormat, nDstStep,
589 nDstWidth, nDstHeight, regionRects, numRegionRects, chroma))
597 if (!avc444_process_rects(h264, pSrcData, SrcSize, pDstData, DstFormat, nDstStep,
598 nDstWidth, nDstHeight, regionRects, numRegionRects,
610#if defined(AVC444_FRAME_STAT)
615 op1sum = avg(&op1, op1sum, SrcSize + AuxSrcSize);
619 op2sum = avg(&op2, op2sum, SrcSize);
623 op3sum = avg(&op3, op3sum, SrcSize);
630 WLog_Print(h264->log, WLOG_INFO,
631 "luma=%" PRIu64
" [avg=%lf] chroma=%" PRIu64
" [avg=%lf] combined=%" PRIu64
633 op1, op1sum, op2, op2sum, op3, op3sum);
638#define MAX_SUBSYSTEMS 10
639static INIT_ONCE subsystems_once = INIT_ONCE_STATIC_INIT;
640static const H264_CONTEXT_SUBSYSTEM* subSystems[MAX_SUBSYSTEMS] = WINPR_C_ARRAY_INIT;
642static BOOL CALLBACK h264_register_subsystems(WINPR_ATTR_UNUSED
PINIT_ONCE once,
643 WINPR_ATTR_UNUSED PVOID param,
644 WINPR_ATTR_UNUSED PVOID* context)
648#ifdef WITH_MEDIACODEC
650 subSystems[i] = &g_Subsystem_mediacodec;
654#if defined(_WIN32) && defined(WITH_MEDIA_FOUNDATION)
656 subSystems[i] = &g_Subsystem_MF;
662 subSystems[i] = &g_Subsystem_OpenH264;
666#ifdef WITH_VIDEO_FFMPEG
668 subSystems[i] = &g_Subsystem_libavcodec;
675static BOOL h264_context_init(H264_CONTEXT* h264)
680 h264->subsystem =
nullptr;
681 if (!InitOnceExecuteOnce(&subsystems_once, h264_register_subsystems,
nullptr,
nullptr))
684 for (
size_t i = 0; i < MAX_SUBSYSTEMS; i++)
686 const H264_CONTEXT_SUBSYSTEM* subsystem = subSystems[i];
688 if (!subsystem || !subsystem->Init)
691 if (subsystem->Init(h264))
693 h264->subsystem = subsystem;
701BOOL h264_context_reset(H264_CONTEXT* h264, UINT32 width, UINT32 height)
707 h264->height = height;
709 if (h264->subsystem && h264->subsystem->Uninit)
710 h264->subsystem->Uninit(h264);
711 if (!h264_context_init(h264))
714 return yuv_context_reset(h264->yuv, width, height);
717H264_CONTEXT* h264_context_new(BOOL Compressor)
719 H264_CONTEXT* h264 = (H264_CONTEXT*)calloc(1,
sizeof(H264_CONTEXT));
723 h264->log = WLog_Get(TAG);
728 h264->Compressor = Compressor;
732 h264->BitRate = 1000000;
733 h264->FrameRate = 30;
736 if (!h264_context_init(h264))
739 h264->yuv = yuv_context_new(Compressor, 0);
746 WINPR_PRAGMA_DIAG_PUSH
747 WINPR_PRAGMA_DIAG_IGNORED_MISMATCHED_DEALLOC
748 h264_context_free(h264);
749 WINPR_PRAGMA_DIAG_POP
753void h264_context_free(H264_CONTEXT* h264)
759 WINPR_ASSERT(h264->subsystem->Uninit);
760 h264->subsystem->Uninit(h264);
763 for (
size_t x = 0; x < 3; x++)
765 if (h264->Compressor)
767 winpr_aligned_free(h264->pYUVData[x]);
768 winpr_aligned_free(h264->pOldYUVData[x]);
770 winpr_aligned_free(h264->pYUV444Data[x]);
771 winpr_aligned_free(h264->pOldYUV444Data[x]);
773 winpr_aligned_free(h264->lumaData);
775 yuv_context_free(h264->yuv);
785 free(meta->quantQualityVals);
786 free(meta->regionRects);
790BOOL h264_context_set_option(H264_CONTEXT* h264, H264_CONTEXT_OPTION option, UINT32 value)
795 case H264_CONTEXT_OPTION_BITRATE:
796 h264->BitRate = value;
798 case H264_CONTEXT_OPTION_FRAMERATE:
799 h264->FrameRate = value;
801 case H264_CONTEXT_OPTION_RATECONTROL:
805 case H264_RATECONTROL_VBR:
806 h264->RateControlMode = H264_RATECONTROL_VBR;
808 case H264_RATECONTROL_CQP:
809 h264->RateControlMode = H264_RATECONTROL_CQP;
812 WLog_Print(h264->log, WLOG_WARN,
813 "Unknown H264_CONTEXT_OPTION_RATECONTROL value [0x%08" PRIx32
"]",
819 case H264_CONTEXT_OPTION_QP:
822 case H264_CONTEXT_OPTION_USAGETYPE:
823 h264->UsageType = value;
825 case H264_CONTEXT_OPTION_HW_ACCEL:
826 h264->hwAccel = (value);
829 WLog_Print(h264->log, WLOG_WARN,
"Unknown H264_CONTEXT_OPTION[0x%08" PRIx32
"]",
835UINT32 h264_context_get_option(H264_CONTEXT* h264, H264_CONTEXT_OPTION option)
840 case H264_CONTEXT_OPTION_BITRATE:
841 return h264->BitRate;
842 case H264_CONTEXT_OPTION_FRAMERATE:
843 return h264->FrameRate;
844 case H264_CONTEXT_OPTION_RATECONTROL:
845 return h264->RateControlMode;
846 case H264_CONTEXT_OPTION_QP:
848 case H264_CONTEXT_OPTION_USAGETYPE:
849 return h264->UsageType;
850 case H264_CONTEXT_OPTION_HW_ACCEL:
851 return h264->hwAccel;
853 WLog_Print(h264->log, WLOG_WARN,
"Unknown H264_CONTEXT_OPTION[0x%08" PRIx32
"]",