FreeRDP
Loading...
Searching...
No Matches
planar.c
1
22#include <freerdp/config.h>
23
24#include <winpr/crt.h>
25#include <winpr/wtypes.h>
26#include <winpr/assert.h>
27#include <winpr/cast.h>
28#include <winpr/print.h>
29
30#include <freerdp/primitives.h>
31#include <freerdp/log.h>
32#include <freerdp/codec/bitmap.h>
33#include <freerdp/codec/planar.h>
34
35#define TAG FREERDP_TAG("codec")
36
37#define PLANAR_ALIGN(val, align) \
38 ((val) % (align) == 0) ? (val) : ((val) + (align) - (val) % (align))
39
40typedef struct
41{
47 BYTE controlByte;
48 BYTE* rawValues;
49} RDP6_RLE_SEGMENT;
50
51typedef struct
52{
53 UINT32 cSegments;
54 RDP6_RLE_SEGMENT* segments;
55} RDP6_RLE_SEGMENTS;
56
57typedef struct
58{
67 BYTE formatHeader;
68} RDP6_BITMAP_STREAM;
69
70struct S_BITMAP_PLANAR_CONTEXT
71{
72 UINT32 maxWidth;
73 UINT32 maxHeight;
74 UINT32 maxPlaneSize;
75
76 BOOL AllowSkipAlpha;
77 BOOL AllowRunLengthEncoding;
78 BOOL AllowColorSubsampling;
79 BOOL AllowDynamicColorFidelity;
80
81 UINT32 ColorLossLevel;
82
83 BYTE* planes[4];
84 BYTE* planesBuffer;
85
86 BYTE* deltaPlanes[4];
87 BYTE* deltaPlanesBuffer;
88
89 BYTE* rlePlanes[4];
90 BYTE* rlePlanesBuffer;
91
92 BYTE* pTempData;
93 UINT32 nTempStep;
94
95 BOOL bgr;
96 BOOL topdown;
97};
98
99static inline BYTE PLANAR_CONTROL_BYTE(UINT32 nRunLength, UINT32 cRawBytes)
100{
101 return WINPR_ASSERTING_INT_CAST(UINT8, ((nRunLength & 0x0F) | ((cRawBytes & 0x0F) << 4)));
102}
103
104static inline BYTE PLANAR_CONTROL_BYTE_RUN_LENGTH(UINT32 controlByte)
105{
106 return (controlByte & 0x0F);
107}
108static inline BYTE PLANAR_CONTROL_BYTE_RAW_BYTES(UINT32 controlByte)
109{
110 return ((controlByte >> 4) & 0x0F);
111}
112
113static inline UINT32 planar_invert_format(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL alpha,
114 UINT32 DstFormat)
115{
116 WINPR_ASSERT(planar);
117 if (planar->bgr && alpha)
118 {
119 switch (DstFormat)
120 {
121 case PIXEL_FORMAT_ARGB32:
122 DstFormat = PIXEL_FORMAT_ABGR32;
123 break;
124 case PIXEL_FORMAT_XRGB32:
125 DstFormat = PIXEL_FORMAT_XBGR32;
126 break;
127 case PIXEL_FORMAT_ABGR32:
128 DstFormat = PIXEL_FORMAT_ARGB32;
129 break;
130 case PIXEL_FORMAT_XBGR32:
131 DstFormat = PIXEL_FORMAT_XRGB32;
132 break;
133 case PIXEL_FORMAT_BGRA32:
134 DstFormat = PIXEL_FORMAT_RGBA32;
135 break;
136 case PIXEL_FORMAT_BGRX32:
137 DstFormat = PIXEL_FORMAT_RGBX32;
138 break;
139 case PIXEL_FORMAT_RGBA32:
140 DstFormat = PIXEL_FORMAT_BGRA32;
141 break;
142 case PIXEL_FORMAT_RGBX32:
143 DstFormat = PIXEL_FORMAT_BGRX32;
144 break;
145 case PIXEL_FORMAT_RGB24:
146 DstFormat = PIXEL_FORMAT_BGR24;
147 break;
148 case PIXEL_FORMAT_BGR24:
149 DstFormat = PIXEL_FORMAT_RGB24;
150 break;
151 case PIXEL_FORMAT_RGB16:
152 DstFormat = PIXEL_FORMAT_BGR16;
153 break;
154 case PIXEL_FORMAT_BGR16:
155 DstFormat = PIXEL_FORMAT_RGB16;
156 break;
157 case PIXEL_FORMAT_ARGB15:
158 DstFormat = PIXEL_FORMAT_ABGR15;
159 break;
160 case PIXEL_FORMAT_RGB15:
161 DstFormat = PIXEL_FORMAT_BGR15;
162 break;
163 case PIXEL_FORMAT_ABGR15:
164 DstFormat = PIXEL_FORMAT_ARGB15;
165 break;
166 case PIXEL_FORMAT_BGR15:
167 DstFormat = PIXEL_FORMAT_RGB15;
168 break;
169 default:
170 break;
171 }
172 }
173 return DstFormat;
174}
175
176static inline BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* WINPR_RESTRICT inPlane,
177 UINT32 width, UINT32 height,
178 BYTE* WINPR_RESTRICT outPlane,
179 UINT32* WINPR_RESTRICT dstSize);
180static inline BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* WINPR_RESTRICT inPlane,
181 UINT32 width, UINT32 height,
182 BYTE* WINPR_RESTRICT outPlane);
183
184static inline INT32 planar_skip_plane_rle(const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize,
185 UINT32 nWidth, UINT32 nHeight)
186{
187 UINT32 used = 0;
188
189 WINPR_ASSERT(pSrcData);
190
191 for (UINT32 y = 0; y < nHeight; y++)
192 {
193 for (UINT32 x = 0; x < nWidth;)
194 {
195 if (used >= SrcSize)
196 {
197 WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRIu32, used,
198 SrcSize);
199 return -1;
200 }
201
202 const uint8_t controlByte = pSrcData[used++];
203 uint32_t nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
204 uint32_t cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
205
206 if (nRunLength == 1)
207 {
208 nRunLength = cRawBytes + 16;
209 cRawBytes = 0;
210 }
211 else if (nRunLength == 2)
212 {
213 nRunLength = cRawBytes + 32;
214 cRawBytes = 0;
215 }
216
217 used += cRawBytes;
218 x += cRawBytes;
219 x += nRunLength;
220
221 if (x > nWidth)
222 {
223 WLog_ERR(TAG, "planar plane x %" PRIu32 " exceeds width %" PRIu32, x, nWidth);
224 return -1;
225 }
226
227 if (used > SrcSize)
228 {
229 WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRId32, used,
230 INT32_MAX);
231 return -1;
232 }
233 }
234 }
235
236 if (used > INT32_MAX)
237 {
238 WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRIu32, used, SrcSize);
239 return -1;
240 }
241 return (INT32)used;
242}
243
244static inline UINT8 clamp(INT16 val)
245{
246 return (UINT8)val;
247}
248
249static inline bool check_source_available(const BYTE* buffer, size_t length, const BYTE* cur,
250 size_t required)
251{
252 WINPR_ASSERT(cur >= buffer);
253 const size_t len = WINPR_ASSERTING_INT_CAST(size_t, (cur - buffer));
254
255 if ((SIZE_MAX - len < required) || (len + required > length))
256 {
257 WLog_ERR(TAG, "error reading input buffer");
258 return false;
259 }
260 return true;
261}
262
263static inline INT32 planar_decompress_plane_rle_only(const BYTE* WINPR_RESTRICT pSrcData,
264 UINT32 SrcSize, BYTE* WINPR_RESTRICT pDstData,
265 UINT32 nWidth, UINT32 nHeight)
266{
267 BYTE* previousScanline = nullptr;
268 const BYTE* srcp = pSrcData;
269
270 WINPR_ASSERT(nHeight <= INT32_MAX);
271 WINPR_ASSERT(nWidth <= INT32_MAX);
272
273 for (UINT32 y = 0; y < nHeight; y++)
274 {
275 BYTE* dstp = &pDstData[1ULL * (y)*nWidth];
276 INT16 pixel = 0;
277 BYTE* currentScanline = dstp;
278
279 for (UINT32 x = 0; x < nWidth;)
280 {
281 if (!check_source_available(pSrcData, SrcSize, srcp, 1))
282 return -1;
283
284 const BYTE controlByte = *srcp++;
285
286 UINT32 nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
287 UINT32 cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
288
289 if (nRunLength == 1)
290 {
291 nRunLength = cRawBytes + 16;
292 cRawBytes = 0;
293 }
294 else if (nRunLength == 2)
295 {
296 nRunLength = cRawBytes + 32;
297 cRawBytes = 0;
298 }
299
300 if (((dstp + (cRawBytes + nRunLength)) - currentScanline) > nWidth * 1ll)
301 {
302 WLog_ERR(TAG, "too many pixels in scanline");
303 return -1;
304 }
305
306 if (!previousScanline)
307 {
308 if (!check_source_available(pSrcData, SrcSize, srcp, cRawBytes))
309 return -1;
310 /* first scanline, absolute values */
311 while (cRawBytes > 0)
312 {
313 pixel = *srcp++;
314 *dstp = clamp(pixel);
315 dstp++;
316 x++;
317 cRawBytes--;
318 }
319
320 while (nRunLength > 0)
321 {
322 *dstp = clamp(pixel);
323 dstp++;
324 x++;
325 nRunLength--;
326 }
327 }
328 else
329 {
330 if (!check_source_available(pSrcData, SrcSize, srcp, cRawBytes))
331 return -1;
332
333 /* delta values relative to previous scanline */
334 while (cRawBytes > 0)
335 {
336 UINT8 deltaValue = *srcp++;
337
338 if (deltaValue & 1)
339 {
340 deltaValue = deltaValue >> 1;
341 deltaValue = deltaValue + 1;
342 pixel = WINPR_ASSERTING_INT_CAST(int16_t, -1 * (int16_t)deltaValue);
343 }
344 else
345 {
346 deltaValue = deltaValue >> 1;
347 pixel = WINPR_ASSERTING_INT_CAST(INT16, deltaValue);
348 }
349
350 const INT16 delta =
351 WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[x] + pixel);
352 *dstp = clamp(delta);
353 dstp++;
354 x++;
355 cRawBytes--;
356 }
357
358 while (nRunLength > 0)
359 {
360 const INT16 deltaValue =
361 WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[x] + pixel);
362 *dstp = clamp(deltaValue);
363 dstp++;
364 x++;
365 nRunLength--;
366 }
367 }
368 }
369
370 previousScanline = currentScanline;
371 }
372
373 return (INT32)(srcp - pSrcData);
374}
375
376static inline INT32 planar_decompress_plane_rle(const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize,
377 BYTE* WINPR_RESTRICT pDstData, UINT32 nDstStep,
378 UINT32 nXDst, UINT32 nYDst, UINT32 nWidth,
379 UINT32 nHeight, UINT32 nChannel, BOOL vFlip)
380{
381 INT32 beg = 0;
382 INT32 end = 0;
383 INT32 inc = 0;
384 BYTE* previousScanline = nullptr;
385 const BYTE* srcp = pSrcData;
386
387 WINPR_ASSERT(nHeight <= INT32_MAX);
388 WINPR_ASSERT(nWidth <= INT32_MAX);
389 WINPR_ASSERT(nDstStep <= INT32_MAX);
390
391 if (vFlip)
392 {
393 beg = (INT32)nHeight - 1;
394 end = -1;
395 inc = -1;
396 }
397 else
398 {
399 beg = 0;
400 end = (INT32)nHeight;
401 inc = 1;
402 }
403
404 for (INT32 y = beg; y != end; y += inc)
405 {
406 const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (4LL * nXDst) + nChannel * 1LL;
407 BYTE* dstp = &pDstData[off];
408 BYTE* currentScanline = dstp;
409 INT16 pixel = 0;
410
411 for (INT32 x = 0; x < (INT32)nWidth;)
412 {
413 if (!check_source_available(pSrcData, SrcSize, srcp, 1))
414 return -1;
415
416 const BYTE controlByte = *srcp++;
417
418 UINT32 nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
419 UINT32 cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
420
421 if (nRunLength == 1)
422 {
423 nRunLength = cRawBytes + 16;
424 cRawBytes = 0;
425 }
426 else if (nRunLength == 2)
427 {
428 nRunLength = cRawBytes + 32;
429 cRawBytes = 0;
430 }
431
432 if (((dstp + (cRawBytes + nRunLength)) - currentScanline) > nWidth * 4ll)
433 {
434 WLog_ERR(TAG, "too many pixels in scanline");
435 return -1;
436 }
437
438 if (!previousScanline)
439 {
440 if (!check_source_available(pSrcData, SrcSize, srcp, cRawBytes))
441 return -1;
442
443 /* first scanline, absolute values */
444 while (cRawBytes > 0)
445 {
446 pixel = *srcp++;
447
448 *dstp = WINPR_ASSERTING_INT_CAST(BYTE, pixel);
449 dstp += 4;
450 x++;
451 cRawBytes--;
452 }
453
454 while (nRunLength > 0)
455 {
456 *dstp = WINPR_ASSERTING_INT_CAST(BYTE, pixel);
457 dstp += 4;
458 x++;
459 nRunLength--;
460 }
461 }
462 else
463 {
464 if (!check_source_available(pSrcData, SrcSize, srcp, cRawBytes))
465 return -1;
466
467 /* delta values relative to previous scanline */
468 while (cRawBytes > 0)
469 {
470 BYTE deltaValue = *srcp++;
471
472 if (deltaValue & 1)
473 {
474 deltaValue = deltaValue >> 1;
475 deltaValue = deltaValue + 1;
476 pixel = WINPR_ASSERTING_INT_CAST(int16_t, -deltaValue);
477 }
478 else
479 {
480 deltaValue = deltaValue >> 1;
481 pixel = deltaValue;
482 }
483
484 const INT16 delta =
485 WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[4LL * x] + pixel);
486 *dstp = clamp(delta);
487 dstp += 4;
488 x++;
489 cRawBytes--;
490 }
491
492 while (nRunLength > 0)
493 {
494 const INT16 deltaValue =
495 WINPR_ASSERTING_INT_CAST(int16_t, pixel + previousScanline[4LL * x]);
496 *dstp = clamp(deltaValue);
497 dstp += 4;
498 x++;
499 nRunLength--;
500 }
501 }
502 }
503
504 previousScanline = currentScanline;
505 }
506
507 return (INT32)(srcp - pSrcData);
508}
509
510static inline INT32 planar_set_plane(BYTE bValue, BYTE* pDstData, UINT32 nDstStep, UINT32 nXDst,
511 UINT32 nYDst, UINT32 nWidth, UINT32 nHeight, UINT32 nChannel,
512 BOOL vFlip)
513{
514 INT32 beg = 0;
515 INT32 end = (INT32)nHeight;
516 INT32 inc = 1;
517
518 WINPR_ASSERT(nHeight <= INT32_MAX);
519 WINPR_ASSERT(nWidth <= INT32_MAX);
520 WINPR_ASSERT(nDstStep <= INT32_MAX);
521
522 if (vFlip)
523 {
524 beg = (INT32)nHeight - 1;
525 end = -1;
526 inc = -1;
527 }
528
529 for (INT32 y = beg; y != end; y += inc)
530 {
531 const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (4LL * nXDst) + nChannel * 1LL;
532 BYTE* dstp = &pDstData[off];
533
534 for (INT32 x = 0; x < (INT32)nWidth; ++x)
535 {
536 *dstp = bValue;
537 dstp += 4;
538 }
539 }
540
541 return 0;
542}
543
544static inline BOOL writeLine(BYTE** WINPR_RESTRICT ppRgba, UINT32 DstFormat, UINT32 width,
545 const BYTE** WINPR_RESTRICT ppR, const BYTE** WINPR_RESTRICT ppG,
546 const BYTE** WINPR_RESTRICT ppB, const BYTE** WINPR_RESTRICT ppA)
547{
548 WINPR_ASSERT(ppRgba);
549 WINPR_ASSERT(ppR);
550 WINPR_ASSERT(ppG);
551 WINPR_ASSERT(ppB);
552
553 switch (DstFormat)
554 {
555 case PIXEL_FORMAT_BGRA32:
556 for (UINT32 x = 0; x < width; x++)
557 {
558 *(*ppRgba)++ = *(*ppB)++;
559 *(*ppRgba)++ = *(*ppG)++;
560 *(*ppRgba)++ = *(*ppR)++;
561 *(*ppRgba)++ = *(*ppA)++;
562 }
563
564 return TRUE;
565
566 case PIXEL_FORMAT_BGRX32:
567 for (UINT32 x = 0; x < width; x++)
568 {
569 *(*ppRgba)++ = *(*ppB)++;
570 *(*ppRgba)++ = *(*ppG)++;
571 *(*ppRgba)++ = *(*ppR)++;
572 *(*ppRgba)++ = 0xFF;
573 }
574
575 return TRUE;
576
577 default:
578 if (ppA)
579 {
580 for (UINT32 x = 0; x < width; x++)
581 {
582 BYTE alpha = *(*ppA)++;
583 UINT32 color =
584 FreeRDPGetColor(DstFormat, *(*ppR)++, *(*ppG)++, *(*ppB)++, alpha);
585 FreeRDPWriteColor(*ppRgba, DstFormat, color);
586 *ppRgba += FreeRDPGetBytesPerPixel(DstFormat);
587 }
588 }
589 else
590 {
591 const BYTE alpha = 0xFF;
592
593 for (UINT32 x = 0; x < width; x++)
594 {
595 UINT32 color =
596 FreeRDPGetColor(DstFormat, *(*ppR)++, *(*ppG)++, *(*ppB)++, alpha);
597 FreeRDPWriteColor(*ppRgba, DstFormat, color);
598 *ppRgba += FreeRDPGetBytesPerPixel(DstFormat);
599 }
600 }
601
602 return TRUE;
603 }
604}
605
606static inline BOOL planar_decompress_planes_raw(const BYTE* WINPR_RESTRICT pSrcData[4],
607 BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat,
608 UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst,
609 UINT32 nWidth, UINT32 nHeight, BOOL vFlip,
610 UINT32 totalHeight)
611{
612 INT32 beg = 0;
613 INT32 end = 0;
614 INT32 inc = 0;
615 const BYTE* pR = pSrcData[0];
616 const BYTE* pG = pSrcData[1];
617 const BYTE* pB = pSrcData[2];
618 const BYTE* pA = pSrcData[3];
619 const UINT32 bpp = FreeRDPGetBytesPerPixel(DstFormat);
620
621 if (vFlip)
622 {
623 beg = WINPR_ASSERTING_INT_CAST(int32_t, nHeight - 1);
624 end = -1;
625 inc = -1;
626 }
627 else
628 {
629 beg = 0;
630 end = WINPR_ASSERTING_INT_CAST(int32_t, nHeight);
631 inc = 1;
632 }
633
634 if (nYDst + nHeight > totalHeight)
635 {
636 WLog_ERR(TAG,
637 "planar plane destination Y %" PRIu32 " + height %" PRIu32
638 " exceeds totalHeight %" PRIu32,
639 nYDst, nHeight, totalHeight);
640 return FALSE;
641 }
642
643 if ((nXDst + nWidth) * bpp > nDstStep)
644 {
645 WLog_ERR(TAG,
646 "planar plane destination (X %" PRIu32 " + width %" PRIu32 ") * bpp %" PRIu32
647 " exceeds stride %" PRIu32,
648 nXDst, nWidth, bpp, nDstStep);
649 return FALSE;
650 }
651
652 for (INT32 y = beg; y != end; y += inc)
653 {
654 BYTE* pRGB = nullptr;
655
656 if (y > WINPR_ASSERTING_INT_CAST(INT64, nHeight))
657 {
658 WLog_ERR(TAG, "planar plane destination Y %" PRId32 " exceeds height %" PRIu32, y,
659 nHeight);
660 return FALSE;
661 }
662
663 const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (1LL * nXDst * bpp);
664 pRGB = &pDstData[off];
665
666 if (!writeLine(&pRGB, DstFormat, nWidth, &pR, &pG, &pB, &pA))
667 return FALSE;
668 }
669
670 return TRUE;
671}
672
673static BOOL planar_subsample_expand(const BYTE* WINPR_RESTRICT plane, size_t planeLength,
674 UINT32 nWidth, UINT32 nHeight, UINT32 nPlaneWidth,
675 UINT32 nPlaneHeight, BYTE* WINPR_RESTRICT deltaPlane)
676{
677 size_t pos = 0;
678 WINPR_UNUSED(planeLength);
679
680 WINPR_ASSERT(plane);
681 WINPR_ASSERT(deltaPlane);
682
683 if (nWidth > nPlaneWidth * 2)
684 {
685 WLog_ERR(TAG, "planar subsample width %" PRIu32 " > PlaneWidth %" PRIu32 " * 2", nWidth,
686 nPlaneWidth);
687 return FALSE;
688 }
689
690 if (nHeight > nPlaneHeight * 2)
691 {
692 WLog_ERR(TAG, "planar subsample height %" PRIu32 " > PlaneHeight %" PRIu32 " * 2", nHeight,
693 nPlaneHeight);
694 return FALSE;
695 }
696
697 for (size_t y = 0; y < nHeight; y++)
698 {
699 const BYTE* src = plane + y / 2 * nPlaneWidth;
700
701 for (UINT32 x = 0; x < nWidth; x++)
702 {
703 deltaPlane[pos++] = src[x / 2];
704 }
705 }
706
707 return TRUE;
708}
709
710#if !defined(WITHOUT_FREERDP_3x_DEPRECATED)
711BOOL planar_decompress(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar,
712 const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize, UINT32 nSrcWidth,
713 UINT32 nSrcHeight, BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat,
714 UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst, UINT32 nDstWidth,
715 UINT32 nDstHeight, BOOL vFlip)
716{
717 return freerdp_bitmap_decompress_planar(planar, pSrcData, SrcSize, nSrcWidth, nSrcHeight,
718 pDstData, DstFormat, nDstStep, nXDst, nYDst, nDstWidth,
719 nDstHeight, vFlip);
720}
721#endif
722
723BOOL freerdp_bitmap_decompress_planar(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar,
724 const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize,
725 UINT32 nSrcWidth, UINT32 nSrcHeight,
726 BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat,
727 UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst, UINT32 nDstWidth,
728 UINT32 nDstHeight, BOOL vFlip)
729{
730 BOOL useAlpha = FALSE;
731 INT32 status = 0;
732 INT32 rleSizes[4] = { 0, 0, 0, 0 };
733 UINT32 rawSizes[4] = WINPR_C_ARRAY_INIT;
734 UINT32 rawWidths[4] = WINPR_C_ARRAY_INIT;
735 UINT32 rawHeights[4] = WINPR_C_ARRAY_INIT;
736 const BYTE* planes[4] = WINPR_C_ARRAY_INIT;
737 const UINT32 w = MIN(nSrcWidth, nDstWidth);
738 const UINT32 h = MIN(nSrcHeight, nDstHeight);
739 const primitives_t* prims = primitives_get();
740
741 WINPR_ASSERT(planar);
742 WINPR_ASSERT(prims);
743
744 if (planar->maxWidth < nSrcWidth)
745 {
746 WLog_ERR(TAG, "planar->maxWidth %" PRIu32 " < nSrcWidth %" PRIu32, planar->maxWidth,
747 nSrcWidth);
748 return FALSE;
749 }
750 if (planar->maxHeight < nSrcHeight)
751 {
752 WLog_ERR(TAG, "planar->maxHeight %" PRIu32 " < nSrcHeight %" PRIu32, planar->maxHeight,
753 nSrcHeight);
754 return FALSE;
755 }
756
757 const UINT32 bpp = FreeRDPGetBytesPerPixel(DstFormat);
758 if (nDstStep <= 0)
759 nDstStep = nDstWidth * bpp;
760
761 const BYTE* srcp = pSrcData;
762
763 if (!pSrcData || (SrcSize < 1))
764 {
765 WLog_ERR(TAG, "Invalid argument pSrcData=%p [size=%" PRIu32 "]",
766 WINPR_CXX_COMPAT_CAST(const void*, pSrcData), SrcSize);
767 return FALSE;
768 }
769
770 if (!pDstData)
771 {
772 WLog_ERR(TAG, "Invalid argument pDstData=nullptr");
773 return FALSE;
774 }
775
776 const BYTE FormatHeader = *srcp++;
777 const BYTE cll = (FormatHeader & PLANAR_FORMAT_HEADER_CLL_MASK);
778 const BYTE cs = (FormatHeader & PLANAR_FORMAT_HEADER_CS) != 0;
779 const BYTE rle = (FormatHeader & PLANAR_FORMAT_HEADER_RLE) != 0;
780 const BYTE alpha = !(FormatHeader & PLANAR_FORMAT_HEADER_NA);
781
782 DstFormat = planar_invert_format(planar, alpha, DstFormat);
783
784 if (alpha)
785 useAlpha = FreeRDPColorHasAlpha(DstFormat);
786
787 // WLog_INFO(TAG, "CLL: %"PRIu32" CS: %"PRIu8" RLE: %"PRIu8" ALPHA: %"PRIu8"", cll, cs, rle,
788 // alpha);
789
790 if (!cll && cs)
791 {
792 WLog_ERR(TAG, "Chroma subsampling requires YCoCg and does not work with RGB data");
793 return FALSE; /* Chroma subsampling requires YCoCg */
794 }
795
796 const UINT32 subWidth = (nSrcWidth / 2) + (nSrcWidth % 2);
797 const UINT32 subHeight = (nSrcHeight / 2) + (nSrcHeight % 2);
798 const UINT32 planeSize = nSrcWidth * nSrcHeight;
799 const UINT32 subSize = subWidth * subHeight;
800
801 if (!cs)
802 {
803 rawSizes[0] = planeSize; /* LumaOrRedPlane */
804 rawWidths[0] = nSrcWidth;
805 rawHeights[0] = nSrcHeight;
806 rawSizes[1] = planeSize; /* OrangeChromaOrGreenPlane */
807 rawWidths[1] = nSrcWidth;
808 rawHeights[1] = nSrcHeight;
809 rawSizes[2] = planeSize; /* GreenChromaOrBluePlane */
810 rawWidths[2] = nSrcWidth;
811 rawHeights[2] = nSrcHeight;
812 rawSizes[3] = planeSize; /* AlphaPlane */
813 rawWidths[3] = nSrcWidth;
814 rawHeights[3] = nSrcHeight;
815 }
816 else /* Chroma Subsampling */
817 {
818 rawSizes[0] = planeSize; /* LumaOrRedPlane */
819 rawWidths[0] = nSrcWidth;
820 rawHeights[0] = nSrcHeight;
821 rawSizes[1] = subSize; /* OrangeChromaOrGreenPlane */
822 rawWidths[1] = subWidth;
823 rawHeights[1] = subHeight;
824 rawSizes[2] = subSize; /* GreenChromaOrBluePlane */
825 rawWidths[2] = subWidth;
826 rawHeights[2] = subHeight;
827 rawSizes[3] = planeSize; /* AlphaPlane */
828 rawWidths[3] = nSrcWidth;
829 rawHeights[3] = nSrcHeight;
830 }
831
832 const size_t diff = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(srcp - pSrcData));
833 if (SrcSize < diff)
834 {
835 WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff);
836 return FALSE;
837 }
838
839 if (!rle) /* RAW */
840 {
841
842 UINT32 base = planeSize * 3;
843 if (cs)
844 base = planeSize + planeSize / 2;
845
846 if (alpha)
847 {
848 if ((SrcSize - diff) < (planeSize + base))
849 {
850 WLog_ERR(TAG, "Alpha plane size mismatch %" PRIuz " < %" PRIu32, SrcSize - diff,
851 (planeSize + base));
852 return FALSE;
853 }
854
855 planes[3] = srcp; /* AlphaPlane */
856 planes[0] = planes[3] + rawSizes[3]; /* LumaOrRedPlane */
857 planes[1] = planes[0] + rawSizes[0]; /* OrangeChromaOrGreenPlane */
858 planes[2] = planes[1] + rawSizes[1]; /* GreenChromaOrBluePlane */
859
860 if ((planes[2] + rawSizes[2]) > &pSrcData[SrcSize])
861 {
862 WLog_ERR(TAG, "plane size mismatch %p + %" PRIu32 " > %p",
863 WINPR_CXX_COMPAT_CAST(const void*, planes[2]), rawSizes[2],
864 WINPR_CXX_COMPAT_CAST(const void*, &pSrcData[SrcSize]));
865 return FALSE;
866 }
867 }
868 else
869 {
870 if ((SrcSize - diff) < base)
871 {
872 WLog_ERR(TAG, "plane size mismatch %" PRIuz " < %" PRIu32, SrcSize - diff, base);
873 return FALSE;
874 }
875
876 planes[0] = srcp; /* LumaOrRedPlane */
877 planes[1] = planes[0] + rawSizes[0]; /* OrangeChromaOrGreenPlane */
878 planes[2] = planes[1] + rawSizes[1]; /* GreenChromaOrBluePlane */
879
880 if ((planes[2] + rawSizes[2]) > &pSrcData[SrcSize])
881 {
882 WLog_ERR(TAG, "plane size mismatch %p + %" PRIu32 " > %p",
883 WINPR_CXX_COMPAT_CAST(const void*, planes[2]), rawSizes[2],
884 WINPR_CXX_COMPAT_CAST(const void*, &pSrcData[SrcSize]));
885 return FALSE;
886 }
887 }
888 }
889 else /* RLE */
890 {
891 if (alpha)
892 {
893 planes[3] = srcp;
894 rleSizes[3] = planar_skip_plane_rle(planes[3], (UINT32)(SrcSize - diff), rawWidths[3],
895 rawHeights[3]); /* AlphaPlane */
896
897 if (rleSizes[3] < 0)
898 return FALSE;
899
900 planes[0] = planes[3] + rleSizes[3];
901 }
902 else
903 planes[0] = srcp;
904
905 const size_t diff0 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[0] - pSrcData));
906 if (SrcSize < diff0)
907 {
908 WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff0);
909 return FALSE;
910 }
911 rleSizes[0] = planar_skip_plane_rle(planes[0], (UINT32)(SrcSize - diff0), rawWidths[0],
912 rawHeights[0]); /* RedPlane */
913
914 if (rleSizes[0] < 0)
915 return FALSE;
916
917 planes[1] = planes[0] + rleSizes[0];
918
919 const size_t diff1 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[1] - pSrcData));
920 if (SrcSize < diff1)
921 {
922 WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff1);
923 return FALSE;
924 }
925 rleSizes[1] = planar_skip_plane_rle(planes[1], (UINT32)(SrcSize - diff1), rawWidths[1],
926 rawHeights[1]); /* GreenPlane */
927
928 if (rleSizes[1] < 1)
929 return FALSE;
930
931 planes[2] = planes[1] + rleSizes[1];
932 const size_t diff2 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[2] - pSrcData));
933 if (SrcSize < diff2)
934 {
935 WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff);
936 return FALSE;
937 }
938 rleSizes[2] = planar_skip_plane_rle(planes[2], (UINT32)(SrcSize - diff2), rawWidths[2],
939 rawHeights[2]); /* BluePlane */
940
941 if (rleSizes[2] < 1)
942 return FALSE;
943 }
944
945 if (!cll) /* RGB */
946 {
947 UINT32 TempFormat = 0;
948 BYTE* pTempData = pDstData;
949 UINT32 nTempStep = nDstStep;
950 UINT32 nTotalHeight = nYDst + nDstHeight;
951
952 if (useAlpha)
953 TempFormat = PIXEL_FORMAT_BGRA32;
954 else
955 TempFormat = PIXEL_FORMAT_BGRX32;
956
957 TempFormat = planar_invert_format(planar, alpha, TempFormat);
958
959 if ((TempFormat != DstFormat) || (nSrcWidth != nDstWidth) || (nSrcHeight != nDstHeight))
960 {
961 pTempData = planar->pTempData;
962 nTempStep = planar->nTempStep;
963 nTotalHeight = planar->maxHeight;
964 }
965
966 if (!rle) /* RAW */
967 {
968 if (!planar_decompress_planes_raw(planes, pTempData, TempFormat, nTempStep, nXDst,
969 nYDst, nSrcWidth, nSrcHeight, vFlip, nTotalHeight))
970 return FALSE;
971
972 if (alpha)
973 srcp += rawSizes[0] + rawSizes[1] + rawSizes[2] + rawSizes[3];
974 else /* NoAlpha */
975 srcp += rawSizes[0] + rawSizes[1] + rawSizes[2];
976
977 if ((SrcSize - (srcp - pSrcData)) == 1)
978 srcp++; /* pad */
979 }
980 else /* RLE */
981 {
982 if (nYDst + nSrcHeight > nTotalHeight)
983 {
984 WLog_ERR(TAG,
985 "planar plane destination Y %" PRIu32 " + height %" PRIu32
986 " exceeds totalHeight %" PRIu32,
987 nYDst, nSrcHeight, nTotalHeight);
988 return FALSE;
989 }
990
991 if ((nXDst + nSrcWidth) * bpp > nTempStep)
992 {
993 WLog_ERR(TAG,
994 "planar plane destination (X %" PRIu32 " + width %" PRIu32
995 ") * bpp %" PRIu32 " exceeds stride %" PRIu32,
996 nXDst, nSrcWidth, bpp, nTempStep);
997 return FALSE;
998 }
999
1000 status = planar_decompress_plane_rle(
1001 planes[0], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[0]), pTempData, nTempStep,
1002 nXDst, nYDst, nSrcWidth, nSrcHeight, 2, vFlip); /* RedPlane */
1003
1004 if (status < 0)
1005 return FALSE;
1006
1007 status = planar_decompress_plane_rle(
1008 planes[1], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[1]), pTempData, nTempStep,
1009 nXDst, nYDst, nSrcWidth, nSrcHeight, 1, vFlip); /* GreenPlane */
1010
1011 if (status < 0)
1012 return FALSE;
1013
1014 status = planar_decompress_plane_rle(
1015 planes[2], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[2]), pTempData, nTempStep,
1016 nXDst, nYDst, nSrcWidth, nSrcHeight, 0, vFlip); /* BluePlane */
1017
1018 if (status < 0)
1019 return FALSE;
1020
1021 srcp += rleSizes[0] + rleSizes[1] + rleSizes[2];
1022
1023 if (useAlpha)
1024 {
1025 status = planar_decompress_plane_rle(
1026 planes[3], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[3]), pTempData,
1027 nTempStep, nXDst, nYDst, nSrcWidth, nSrcHeight, 3, vFlip); /* AlphaPlane */
1028 }
1029 else
1030 status = planar_set_plane(0xFF, pTempData, nTempStep, nXDst, nYDst, nSrcWidth,
1031 nSrcHeight, 3, vFlip);
1032
1033 if (status < 0)
1034 return FALSE;
1035
1036 if (alpha)
1037 srcp += rleSizes[3];
1038 }
1039
1040 if (pTempData != pDstData)
1041 {
1042 if (!freerdp_image_copy_no_overlap(pDstData, DstFormat, nDstStep, nXDst, nYDst, w, h,
1043 pTempData, TempFormat, nTempStep, nXDst, nYDst,
1044 nullptr, FREERDP_FLIP_NONE))
1045 {
1046 WLog_ERR(TAG, "planar image copy failed");
1047 return FALSE;
1048 }
1049 }
1050 }
1051 else /* YCoCg */
1052 {
1053 UINT32 TempFormat = 0;
1054 BYTE* pTempData = planar->pTempData;
1055 UINT32 nTempStep = planar->nTempStep;
1056 UINT32 nTotalHeight = planar->maxHeight;
1057 BYTE* dst = &pDstData[nXDst * FreeRDPGetBytesPerPixel(DstFormat) + nYDst * nDstStep];
1058
1059 if (useAlpha)
1060 TempFormat = PIXEL_FORMAT_BGRA32;
1061 else
1062 TempFormat = PIXEL_FORMAT_BGRX32;
1063
1064 if (!pTempData)
1065 {
1066 WLog_ERR(TAG, "pTempData == NULL");
1067 return FALSE;
1068 }
1069
1070 if (rle) /* RLE encoded data. Decode and handle it like raw data. */
1071 {
1072 BYTE* rleBuffer[4] = WINPR_C_ARRAY_INIT;
1073
1074 if (!planar->rlePlanesBuffer)
1075 {
1076 WLog_ERR(TAG, "planar->rlePlanesBuffer == NULL");
1077 return FALSE;
1078 }
1079
1080 rleBuffer[3] = planar->rlePlanesBuffer; /* AlphaPlane */
1081 rleBuffer[0] = rleBuffer[3] + planeSize; /* LumaOrRedPlane */
1082 rleBuffer[1] = rleBuffer[0] + planeSize; /* OrangeChromaOrGreenPlane */
1083 rleBuffer[2] = rleBuffer[1] + planeSize; /* GreenChromaOrBluePlane */
1084 if (useAlpha)
1085 {
1086 status = planar_decompress_plane_rle_only(
1087 planes[3], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[3]), rleBuffer[3],
1088 rawWidths[3], rawHeights[3]); /* AlphaPlane */
1089
1090 if (status < 0)
1091 return FALSE;
1092 }
1093
1094 if (alpha)
1095 srcp += rleSizes[3];
1096
1097 status = planar_decompress_plane_rle_only(
1098 planes[0], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[0]), rleBuffer[0],
1099 rawWidths[0], rawHeights[0]); /* LumaPlane */
1100
1101 if (status < 0)
1102 return FALSE;
1103
1104 status = planar_decompress_plane_rle_only(
1105 planes[1], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[1]), rleBuffer[1],
1106 rawWidths[1], rawHeights[1]); /* OrangeChromaPlane */
1107
1108 if (status < 0)
1109 return FALSE;
1110
1111 status = planar_decompress_plane_rle_only(
1112 planes[2], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[2]), rleBuffer[2],
1113 rawWidths[2], rawHeights[2]); /* GreenChromaPlane */
1114
1115 if (status < 0)
1116 return FALSE;
1117
1118 planes[0] = rleBuffer[0];
1119 planes[1] = rleBuffer[1];
1120 planes[2] = rleBuffer[2];
1121 planes[3] = rleBuffer[3];
1122 }
1123
1124 /* RAW */
1125 {
1126 if (cs)
1127 { /* Chroma subsampling for Co and Cg:
1128 * Each pixel contains the value that should be expanded to
1129 * [2x,2y;2x+1,2y;2x+1,2y+1;2x;2y+1] */
1130 if (!planar_subsample_expand(planes[1], rawSizes[1], nSrcWidth, nSrcHeight,
1131 rawWidths[1], rawHeights[1], planar->deltaPlanes[0]))
1132 return FALSE;
1133
1134 planes[1] = planar->deltaPlanes[0];
1135 rawSizes[1] = planeSize; /* OrangeChromaOrGreenPlane */
1136 rawWidths[1] = nSrcWidth;
1137 rawHeights[1] = nSrcHeight;
1138
1139 if (!planar_subsample_expand(planes[2], rawSizes[2], nSrcWidth, nSrcHeight,
1140 rawWidths[2], rawHeights[2], planar->deltaPlanes[1]))
1141 return FALSE;
1142
1143 planes[2] = planar->deltaPlanes[1];
1144 rawSizes[2] = planeSize; /* GreenChromaOrBluePlane */
1145 rawWidths[2] = nSrcWidth;
1146 rawHeights[2] = nSrcHeight;
1147 }
1148
1149 if (!planar_decompress_planes_raw(planes, pTempData, TempFormat, nTempStep, nXDst,
1150 nYDst, nSrcWidth, nSrcHeight, vFlip, nTotalHeight))
1151 return FALSE;
1152
1153 if (alpha)
1154 srcp += rawSizes[0] + rawSizes[1] + rawSizes[2] + rawSizes[3];
1155 else /* NoAlpha */
1156 srcp += rawSizes[0] + rawSizes[1] + rawSizes[2];
1157
1158 if ((SrcSize - (srcp - pSrcData)) == 1)
1159 srcp++; /* pad */
1160 }
1161
1162 WINPR_ASSERT(prims->YCoCgToRGB_8u_AC4R);
1163 int rc = prims->YCoCgToRGB_8u_AC4R(
1164 pTempData, WINPR_ASSERTING_INT_CAST(int32_t, nTempStep), dst, DstFormat,
1165 WINPR_ASSERTING_INT_CAST(int32_t, nDstStep), w, h, cll, useAlpha);
1166 if (rc != PRIMITIVES_SUCCESS)
1167 {
1168 WLog_ERR(TAG, "YCoCgToRGB_8u_AC4R failed with %d", rc);
1169 return FALSE;
1170 }
1171 }
1172
1173 WINPR_UNUSED(srcp);
1174 return TRUE;
1175}
1176
1177static inline BOOL freerdp_split_color_planes(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar,
1178 const BYTE* WINPR_RESTRICT data, UINT32 format,
1179 UINT32 width, UINT32 height, UINT32 scanline,
1180 BYTE* WINPR_RESTRICT planes[4])
1181{
1182 WINPR_ASSERT(planar);
1183
1184 if ((width > INT32_MAX) || (height > INT32_MAX) || (scanline > INT32_MAX))
1185 return FALSE;
1186
1187 if (scanline == 0)
1188 scanline = width * FreeRDPGetBytesPerPixel(format);
1189
1190 if (planar->topdown)
1191 {
1192 UINT32 k = 0;
1193 for (UINT32 i = 0; i < height; i++)
1194 {
1195 const BYTE* pixel = &data[1ULL * scanline * i];
1196
1197 for (UINT32 j = 0; j < width; j++)
1198 {
1199 const UINT32 color = FreeRDPReadColor(pixel, format);
1200 pixel += FreeRDPGetBytesPerPixel(format);
1201 FreeRDPSplitColor(color, format, &planes[1][k], &planes[2][k], &planes[3][k],
1202 &planes[0][k], nullptr);
1203 k++;
1204 }
1205 }
1206 }
1207 else
1208 {
1209 UINT32 k = 0;
1210
1211 for (INT64 i = (INT64)height - 1; i >= 0; i--)
1212 {
1213 const BYTE* pixel = &data[1ULL * scanline * (UINT32)i];
1214
1215 for (UINT32 j = 0; j < width; j++)
1216 {
1217 const UINT32 color = FreeRDPReadColor(pixel, format);
1218 pixel += FreeRDPGetBytesPerPixel(format);
1219 FreeRDPSplitColor(color, format, &planes[1][k], &planes[2][k], &planes[3][k],
1220 &planes[0][k], nullptr);
1221 k++;
1222 }
1223 }
1224 }
1225 return TRUE;
1226}
1227
1228static inline UINT32 freerdp_bitmap_planar_write_rle_bytes(const BYTE* WINPR_RESTRICT pInBuffer,
1229 UINT32 cRawBytes, UINT32 nRunLength,
1230 BYTE* WINPR_RESTRICT pOutBuffer,
1231 UINT32 outBufferSize)
1232{
1233 const BYTE* pInput = pInBuffer;
1234 BYTE* pOutput = pOutBuffer;
1235 BYTE controlByte = 0;
1236 UINT32 nBytesToWrite = 0;
1237
1238 if (!cRawBytes && !nRunLength)
1239 return 0;
1240
1241 if (nRunLength < 3)
1242 {
1243 cRawBytes += nRunLength;
1244 nRunLength = 0;
1245 }
1246
1247 while (cRawBytes)
1248 {
1249 if (cRawBytes < 16)
1250 {
1251 if (nRunLength > 15)
1252 {
1253 if (nRunLength < 18)
1254 {
1255 controlByte = PLANAR_CONTROL_BYTE(13, cRawBytes);
1256 nRunLength -= 13;
1257 cRawBytes = 0;
1258 }
1259 else
1260 {
1261 controlByte = PLANAR_CONTROL_BYTE(15, cRawBytes);
1262 nRunLength -= 15;
1263 cRawBytes = 0;
1264 }
1265 }
1266 else
1267 {
1268 controlByte = PLANAR_CONTROL_BYTE(nRunLength, cRawBytes);
1269 nRunLength = 0;
1270 cRawBytes = 0;
1271 }
1272 }
1273 else
1274 {
1275 controlByte = PLANAR_CONTROL_BYTE(0, 15);
1276 cRawBytes -= 15;
1277 }
1278
1279 if (outBufferSize < 1)
1280 return 0;
1281
1282 outBufferSize--;
1283 *pOutput = controlByte;
1284 pOutput++;
1285 nBytesToWrite = (controlByte >> 4);
1286
1287 if (nBytesToWrite)
1288 {
1289 if (outBufferSize < nBytesToWrite)
1290 return 0;
1291
1292 outBufferSize -= nBytesToWrite;
1293 CopyMemory(pOutput, pInput, nBytesToWrite);
1294 pOutput += nBytesToWrite;
1295 pInput += nBytesToWrite;
1296 }
1297 }
1298
1299 while (nRunLength)
1300 {
1301 if (nRunLength > 47)
1302 {
1303 if (nRunLength < 50)
1304 {
1305 controlByte = PLANAR_CONTROL_BYTE(2, 13);
1306 nRunLength -= 45;
1307 }
1308 else
1309 {
1310 controlByte = PLANAR_CONTROL_BYTE(2, 15);
1311 nRunLength -= 47;
1312 }
1313 }
1314 else if (nRunLength > 31)
1315 {
1316 controlByte = PLANAR_CONTROL_BYTE(2, (nRunLength - 32));
1317 nRunLength = 0;
1318 }
1319 else if (nRunLength > 15)
1320 {
1321 controlByte = PLANAR_CONTROL_BYTE(1, (nRunLength - 16));
1322 nRunLength = 0;
1323 }
1324 else
1325 {
1326 controlByte = PLANAR_CONTROL_BYTE(nRunLength, 0);
1327 nRunLength = 0;
1328 }
1329
1330 if (outBufferSize < 1)
1331 return 0;
1332
1333 --outBufferSize;
1334 *pOutput = controlByte;
1335 pOutput++;
1336 }
1337
1338 const intptr_t diff = (pOutput - pOutBuffer);
1339 if ((diff < 0) || (diff > UINT32_MAX))
1340 return 0;
1341 return (UINT32)diff;
1342}
1343
1344static inline UINT32 freerdp_bitmap_planar_encode_rle_bytes(const BYTE* WINPR_RESTRICT pInBuffer,
1345 UINT32 inBufferSize,
1346 BYTE* WINPR_RESTRICT pOutBuffer,
1347 UINT32 outBufferSize)
1348{
1349 BYTE symbol = 0;
1350 const BYTE* pInput = pInBuffer;
1351 BYTE* pOutput = pOutBuffer;
1352 const BYTE* pBytes = nullptr;
1353 UINT32 cRawBytes = 0;
1354 UINT32 nRunLength = 0;
1355 UINT32 nBytesWritten = 0;
1356 UINT32 nTotalBytesWritten = 0;
1357
1358 if (!outBufferSize)
1359 return 0;
1360
1361 do
1362 {
1363 if (!inBufferSize)
1364 break;
1365
1366 const UINT32 bSymbolMatch = (symbol == *pInput) != 0;
1367 symbol = *pInput;
1368 pInput++;
1369 inBufferSize--;
1370
1371 if (nRunLength && !bSymbolMatch)
1372 {
1373 if (nRunLength < 3)
1374 {
1375 cRawBytes += nRunLength;
1376 nRunLength = 0;
1377 }
1378 else
1379 {
1380 pBytes = pInput - (cRawBytes + nRunLength + 1);
1381 nBytesWritten = freerdp_bitmap_planar_write_rle_bytes(pBytes, cRawBytes, nRunLength,
1382 pOutput, outBufferSize);
1383 nRunLength = 0;
1384
1385 if (!nBytesWritten || (nBytesWritten > outBufferSize))
1386 return nRunLength;
1387
1388 nTotalBytesWritten += nBytesWritten;
1389 outBufferSize -= nBytesWritten;
1390 pOutput += nBytesWritten;
1391 cRawBytes = 0;
1392 }
1393 }
1394
1395 nRunLength += bSymbolMatch;
1396 cRawBytes += (!bSymbolMatch) != 0;
1397 } while (outBufferSize);
1398
1399 if (cRawBytes || nRunLength)
1400 {
1401 pBytes = pInput - (cRawBytes + nRunLength);
1402 nBytesWritten = freerdp_bitmap_planar_write_rle_bytes(pBytes, cRawBytes, nRunLength,
1403 pOutput, outBufferSize);
1404
1405 if (!nBytesWritten)
1406 return 0;
1407
1408 nTotalBytesWritten += nBytesWritten;
1409 }
1410
1411 if (inBufferSize)
1412 return 0;
1413
1414 return nTotalBytesWritten;
1415}
1416
1417BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* WINPR_RESTRICT inPlane, UINT32 width,
1418 UINT32 height, BYTE* WINPR_RESTRICT outPlane,
1419 UINT32* WINPR_RESTRICT dstSize)
1420{
1421 if (!outPlane)
1422 return FALSE;
1423
1424 UINT32 index = 0;
1425 const BYTE* pInput = inPlane;
1426 BYTE* pOutput = outPlane;
1427 UINT32 outBufferSize = *dstSize;
1428 UINT32 nTotalBytesWritten = 0;
1429
1430 while (outBufferSize > 0)
1431 {
1432 const UINT32 nBytesWritten =
1433 freerdp_bitmap_planar_encode_rle_bytes(pInput, width, pOutput, outBufferSize);
1434
1435 if ((!nBytesWritten) || (nBytesWritten > outBufferSize))
1436 return FALSE;
1437
1438 outBufferSize -= nBytesWritten;
1439 nTotalBytesWritten += nBytesWritten;
1440 pOutput += nBytesWritten;
1441 pInput += width;
1442 index++;
1443
1444 if (index >= height)
1445 break;
1446 }
1447
1448 *dstSize = nTotalBytesWritten;
1449 return TRUE;
1450}
1451
1452static inline BOOL freerdp_bitmap_planar_compress_planes_rle(BYTE* WINPR_RESTRICT inPlanes[4],
1453 UINT32 width, UINT32 height,
1454 BYTE* WINPR_RESTRICT outPlanes,
1455 UINT32* WINPR_RESTRICT dstSizes,
1456 BOOL skipAlpha)
1457{
1458 UINT32 outPlanesSize = width * height * 4;
1459
1460 /* AlphaPlane */
1461 if (skipAlpha)
1462 {
1463 dstSizes[0] = 0;
1464 }
1465 else
1466 {
1467 dstSizes[0] = outPlanesSize;
1468
1469 if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[0], width, height, outPlanes,
1470 &dstSizes[0]))
1471 return FALSE;
1472
1473 outPlanes += dstSizes[0];
1474 outPlanesSize -= dstSizes[0];
1475 }
1476
1477 /* LumaOrRedPlane */
1478 dstSizes[1] = outPlanesSize;
1479
1480 if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[1], width, height, outPlanes,
1481 &dstSizes[1]))
1482 return FALSE;
1483
1484 outPlanes += dstSizes[1];
1485 outPlanesSize -= dstSizes[1];
1486 /* OrangeChromaOrGreenPlane */
1487 dstSizes[2] = outPlanesSize;
1488
1489 if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[2], width, height, outPlanes,
1490 &dstSizes[2]))
1491 return FALSE;
1492
1493 outPlanes += dstSizes[2];
1494 outPlanesSize -= dstSizes[2];
1495 /* GreenChromeOrBluePlane */
1496 dstSizes[3] = outPlanesSize;
1497
1498 return (freerdp_bitmap_planar_compress_plane_rle(inPlanes[3], width, height, outPlanes,
1499 &dstSizes[3]));
1500}
1501
1502BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* WINPR_RESTRICT inPlane, UINT32 width,
1503 UINT32 height, BYTE* WINPR_RESTRICT outPlane)
1504{
1505 if (!outPlane)
1506 {
1507 if (width * height == 0)
1508 return nullptr;
1509
1510 outPlane = (BYTE*)calloc(height, width);
1511 if (!outPlane)
1512 return nullptr;
1513 }
1514
1515 // first line is copied as is
1516 CopyMemory(outPlane, inPlane, width);
1517
1518 for (UINT32 y = 1; y < height; y++)
1519 {
1520 const size_t off = 1ull * width * y;
1521 BYTE* outPtr = &outPlane[off];
1522 const BYTE* srcPtr = &inPlane[off];
1523 const BYTE* prevLinePtr = &inPlane[off - width];
1524 for (UINT32 x = 0; x < width; x++)
1525 {
1526 const int delta = (int)srcPtr[x] - (int)prevLinePtr[x];
1527 const int s2c1i = (delta >= 0) ? delta : (INT_MAX + delta) + 1;
1528 const int8_t s2c1 = WINPR_CXX_COMPAT_CAST(int8_t, s2c1i);
1529 const uint32_t s2c =
1530 (s2c1 >= 0) ? ((UINT32)s2c1 << 1) : (((UINT32)(~(s2c1) + 1) << 1) - 1);
1531 outPtr[x] = (BYTE)s2c;
1532 }
1533 }
1534
1535 return outPlane;
1536}
1537
1538static inline BOOL freerdp_bitmap_planar_delta_encode_planes(BYTE* WINPR_RESTRICT inPlanes[4],
1539 UINT32 width, UINT32 height,
1540 BYTE* WINPR_RESTRICT outPlanes[4])
1541{
1542 for (UINT32 i = 0; i < 4; i++)
1543 {
1544 outPlanes[i] =
1545 freerdp_bitmap_planar_delta_encode_plane(inPlanes[i], width, height, outPlanes[i]);
1546
1547 if (!outPlanes[i])
1548 return FALSE;
1549 }
1550
1551 return TRUE;
1552}
1553
1554BYTE* freerdp_bitmap_compress_planar(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT context,
1555 const BYTE* WINPR_RESTRICT data, UINT32 format, UINT32 width,
1556 UINT32 height, UINT32 scanline, BYTE* WINPR_RESTRICT dstData,
1557 UINT32* WINPR_RESTRICT pDstSize)
1558{
1559 UINT32 size = 0;
1560 BYTE* dstp = nullptr;
1561 UINT32 dstSizes[4] = WINPR_C_ARRAY_INIT;
1562 BYTE FormatHeader = 0;
1563
1564 if (!context || !context->rlePlanesBuffer)
1565 return nullptr;
1566
1567 if (context->AllowSkipAlpha)
1568 FormatHeader |= PLANAR_FORMAT_HEADER_NA;
1569
1570 const UINT32 planeSize = width * height;
1571
1572 if (!context->AllowSkipAlpha)
1573 format = planar_invert_format(context, TRUE, format);
1574
1575 if (!freerdp_split_color_planes(context, data, format, width, height, scanline,
1576 context->planes))
1577 return nullptr;
1578
1579 if (context->AllowRunLengthEncoding)
1580 {
1581 if (!freerdp_bitmap_planar_delta_encode_planes(context->planes, width, height,
1582 context->deltaPlanes))
1583 return nullptr;
1584
1585 if (!freerdp_bitmap_planar_compress_planes_rle(context->deltaPlanes, width, height,
1586 context->rlePlanesBuffer, dstSizes,
1587 context->AllowSkipAlpha))
1588 return nullptr;
1589
1590 {
1591 uint32_t offset = 0;
1592 FormatHeader |= PLANAR_FORMAT_HEADER_RLE;
1593 context->rlePlanes[0] = &context->rlePlanesBuffer[offset];
1594 offset += dstSizes[0];
1595 context->rlePlanes[1] = &context->rlePlanesBuffer[offset];
1596 offset += dstSizes[1];
1597 context->rlePlanes[2] = &context->rlePlanesBuffer[offset];
1598 offset += dstSizes[2];
1599 context->rlePlanes[3] = &context->rlePlanesBuffer[offset];
1600
1601#if defined(WITH_DEBUG_CODECS)
1602 WLog_DBG(TAG,
1603 "R: [%" PRIu32 "/%" PRIu32 "] G: [%" PRIu32 "/%" PRIu32 "] B: [%" PRIu32
1604 " / %" PRIu32 "] ",
1605 dstSizes[1], planeSize, dstSizes[2], planeSize, dstSizes[3], planeSize);
1606#endif
1607 }
1608 }
1609
1610 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1611 {
1612 if (!context->AllowRunLengthEncoding)
1613 return nullptr;
1614
1615 if (context->rlePlanes[0] == nullptr)
1616 return nullptr;
1617
1618 if (context->rlePlanes[1] == nullptr)
1619 return nullptr;
1620
1621 if (context->rlePlanes[2] == nullptr)
1622 return nullptr;
1623
1624 if (context->rlePlanes[3] == nullptr)
1625 return nullptr;
1626 }
1627
1628 if (!dstData)
1629 {
1630 size = 1;
1631
1632 if (!(FormatHeader & PLANAR_FORMAT_HEADER_NA))
1633 {
1634 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1635 size += dstSizes[0];
1636 else
1637 size += planeSize;
1638 }
1639
1640 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1641 size += (dstSizes[1] + dstSizes[2] + dstSizes[3]);
1642 else
1643 size += (planeSize * 3);
1644
1645 if (!(FormatHeader & PLANAR_FORMAT_HEADER_RLE))
1646 size++;
1647
1648 dstData = malloc(size);
1649
1650 if (!dstData)
1651 return nullptr;
1652
1653 *pDstSize = size;
1654 }
1655
1656 dstp = dstData;
1657 *dstp = FormatHeader; /* FormatHeader */
1658 dstp++;
1659
1660 /* AlphaPlane */
1661
1662 if (!(FormatHeader & PLANAR_FORMAT_HEADER_NA))
1663 {
1664 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1665 {
1666 CopyMemory(dstp, context->rlePlanes[0], dstSizes[0]); /* Alpha */
1667 dstp += dstSizes[0];
1668 }
1669 else
1670 {
1671 CopyMemory(dstp, context->planes[0], planeSize); /* Alpha */
1672 dstp += planeSize;
1673 }
1674 }
1675
1676 /* LumaOrRedPlane */
1677
1678 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1679 {
1680 CopyMemory(dstp, context->rlePlanes[1], dstSizes[1]); /* Red */
1681 dstp += dstSizes[1];
1682 }
1683 else
1684 {
1685 CopyMemory(dstp, context->planes[1], planeSize); /* Red */
1686 dstp += planeSize;
1687 }
1688
1689 /* OrangeChromaOrGreenPlane */
1690
1691 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1692 {
1693 CopyMemory(dstp, context->rlePlanes[2], dstSizes[2]); /* Green */
1694 dstp += dstSizes[2];
1695 }
1696 else
1697 {
1698 CopyMemory(dstp, context->planes[2], planeSize); /* Green */
1699 dstp += planeSize;
1700 }
1701
1702 /* GreenChromeOrBluePlane */
1703
1704 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1705 {
1706 CopyMemory(dstp, context->rlePlanes[3], dstSizes[3]); /* Blue */
1707 dstp += dstSizes[3];
1708 }
1709 else
1710 {
1711 CopyMemory(dstp, context->planes[3], planeSize); /* Blue */
1712 dstp += planeSize;
1713 }
1714
1715 /* Pad1 (1 byte) */
1716
1717 if (!(FormatHeader & PLANAR_FORMAT_HEADER_RLE))
1718 {
1719 *dstp = 0;
1720 dstp++;
1721 }
1722
1723 const intptr_t diff = (dstp - dstData);
1724 if ((diff < 0) || (diff > UINT32_MAX))
1725 {
1726 free(dstData);
1727 return nullptr;
1728 }
1729 size = (UINT32)diff;
1730 *pDstSize = size;
1731 return dstData;
1732}
1733
1734BOOL freerdp_bitmap_planar_context_reset(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT context,
1735 UINT32 width, UINT32 height)
1736{
1737 if (!context)
1738 return FALSE;
1739
1740 context->bgr = FALSE;
1741 context->maxWidth = PLANAR_ALIGN(width, 4);
1742 context->maxHeight = PLANAR_ALIGN(height, 4);
1743 {
1744 const UINT64 tmp = (UINT64)context->maxWidth * context->maxHeight;
1745 if (tmp > UINT32_MAX)
1746 return FALSE;
1747 context->maxPlaneSize = (UINT32)tmp;
1748 }
1749
1750 if (context->maxWidth > UINT32_MAX / 4)
1751 return FALSE;
1752 context->nTempStep = context->maxWidth * 4;
1753
1754 memset((void*)context->planes, 0, sizeof(context->planes));
1755 memset((void*)context->rlePlanes, 0, sizeof(context->rlePlanes));
1756 memset((void*)context->deltaPlanes, 0, sizeof(context->deltaPlanes));
1757
1758 if (context->maxPlaneSize > 0)
1759 {
1760 void* tmp = winpr_aligned_recalloc(context->planesBuffer, context->maxPlaneSize, 4, 32);
1761 if (!tmp)
1762 return FALSE;
1763 context->planesBuffer = tmp;
1764
1765 tmp = winpr_aligned_recalloc(context->pTempData, context->maxPlaneSize, 6, 32);
1766 if (!tmp)
1767 return FALSE;
1768 context->pTempData = tmp;
1769
1770 tmp = winpr_aligned_recalloc(context->deltaPlanesBuffer, context->maxPlaneSize, 4, 32);
1771 if (!tmp)
1772 return FALSE;
1773 context->deltaPlanesBuffer = tmp;
1774
1775 tmp = winpr_aligned_recalloc(context->rlePlanesBuffer, context->maxPlaneSize, 4, 32);
1776 if (!tmp)
1777 return FALSE;
1778 context->rlePlanesBuffer = tmp;
1779
1780 context->planes[0] = &context->planesBuffer[0ULL * context->maxPlaneSize];
1781 context->planes[1] = &context->planesBuffer[1ULL * context->maxPlaneSize];
1782 context->planes[2] = &context->planesBuffer[2ULL * context->maxPlaneSize];
1783 context->planes[3] = &context->planesBuffer[3ULL * context->maxPlaneSize];
1784 context->deltaPlanes[0] = &context->deltaPlanesBuffer[0ULL * context->maxPlaneSize];
1785 context->deltaPlanes[1] = &context->deltaPlanesBuffer[1ULL * context->maxPlaneSize];
1786 context->deltaPlanes[2] = &context->deltaPlanesBuffer[2ULL * context->maxPlaneSize];
1787 context->deltaPlanes[3] = &context->deltaPlanesBuffer[3ULL * context->maxPlaneSize];
1788 }
1789 return TRUE;
1790}
1791
1792BITMAP_PLANAR_CONTEXT* freerdp_bitmap_planar_context_new(DWORD flags, UINT32 maxWidth,
1793 UINT32 maxHeight)
1794{
1795 BITMAP_PLANAR_CONTEXT* context =
1796 (BITMAP_PLANAR_CONTEXT*)winpr_aligned_calloc(1, sizeof(BITMAP_PLANAR_CONTEXT), 32);
1797
1798 if (!context)
1799 return nullptr;
1800
1801 if (flags & PLANAR_FORMAT_HEADER_NA)
1802 context->AllowSkipAlpha = TRUE;
1803
1804 if (flags & PLANAR_FORMAT_HEADER_RLE)
1805 context->AllowRunLengthEncoding = TRUE;
1806
1807 if (flags & PLANAR_FORMAT_HEADER_CS)
1808 context->AllowColorSubsampling = TRUE;
1809
1810 context->ColorLossLevel = flags & PLANAR_FORMAT_HEADER_CLL_MASK;
1811
1812 if (context->ColorLossLevel)
1813 context->AllowDynamicColorFidelity = TRUE;
1814
1815 if (!freerdp_bitmap_planar_context_reset(context, maxWidth, maxHeight))
1816 {
1817 WINPR_PRAGMA_DIAG_PUSH
1818 WINPR_PRAGMA_DIAG_IGNORED_MISMATCHED_DEALLOC
1819 freerdp_bitmap_planar_context_free(context);
1820 WINPR_PRAGMA_DIAG_POP
1821 return nullptr;
1822 }
1823
1824 return context;
1825}
1826
1827void freerdp_bitmap_planar_context_free(BITMAP_PLANAR_CONTEXT* context)
1828{
1829 if (!context)
1830 return;
1831
1832 winpr_aligned_free(context->pTempData);
1833 winpr_aligned_free(context->planesBuffer);
1834 winpr_aligned_free(context->deltaPlanesBuffer);
1835 winpr_aligned_free(context->rlePlanesBuffer);
1836 winpr_aligned_free(context);
1837}
1838
1839void freerdp_planar_switch_bgr(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL bgr)
1840{
1841 WINPR_ASSERT(planar);
1842 planar->bgr = bgr;
1843}
1844
1845void freerdp_planar_topdown_image(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL topdown)
1846{
1847 WINPR_ASSERT(planar);
1848 planar->topdown = topdown;
1849}
primitives_t
Definition primitives.h:255