/** * FreeRDP: A Remote Desktop Protocol Implementation * RDP6 Planar Codec * * Copyright 2013 Marc-Andre Moreau * Copyright 2016 Armin Novak * Copyright 2016 Thincast Technologies GmbH * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include #include #define TAG FREERDP_TAG("codec") #define PLANAR_ALIGN(val, align) \ ((val) % (align) == 0) ? (val) : ((val) + (align) - (val) % (align)) typedef struct { /** * controlByte: * [0-3]: nRunLength * [4-7]: cRawBytes */ BYTE controlByte; BYTE* rawValues; } RDP6_RLE_SEGMENT; typedef struct { UINT32 cSegments; RDP6_RLE_SEGMENT* segments; } RDP6_RLE_SEGMENTS; typedef struct { /** * formatHeader: * [0-2]: Color Loss Level (CLL) * [3] : Chroma Subsampling (CS) * [4] : Run Length Encoding (RLE) * [5] : No Alpha (NA) * [6-7]: Reserved */ BYTE formatHeader; } RDP6_BITMAP_STREAM; struct S_BITMAP_PLANAR_CONTEXT { UINT32 maxWidth; UINT32 maxHeight; UINT32 maxPlaneSize; BOOL AllowSkipAlpha; BOOL AllowRunLengthEncoding; BOOL AllowColorSubsampling; BOOL AllowDynamicColorFidelity; UINT32 ColorLossLevel; BYTE* planes[4]; BYTE* planesBuffer; BYTE* deltaPlanes[4]; BYTE* deltaPlanesBuffer; BYTE* rlePlanes[4]; BYTE* rlePlanesBuffer; BYTE* pTempData; UINT32 nTempStep; BOOL bgr; BOOL topdown; }; static inline BYTE PLANAR_CONTROL_BYTE(UINT32 nRunLength, UINT32 cRawBytes) { return WINPR_ASSERTING_INT_CAST(UINT8, ((nRunLength & 0x0F) | ((cRawBytes & 0x0F) << 4))); } static inline BYTE PLANAR_CONTROL_BYTE_RUN_LENGTH(UINT32 controlByte) { return (controlByte & 0x0F); } static inline BYTE PLANAR_CONTROL_BYTE_RAW_BYTES(UINT32 controlByte) { return ((controlByte >> 4) & 0x0F); } static inline UINT32 planar_invert_format(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL alpha, UINT32 DstFormat) { WINPR_ASSERT(planar); if (planar->bgr && alpha) { switch (DstFormat) { case PIXEL_FORMAT_ARGB32: DstFormat = PIXEL_FORMAT_ABGR32; break; case PIXEL_FORMAT_XRGB32: DstFormat = PIXEL_FORMAT_XBGR32; break; case PIXEL_FORMAT_ABGR32: DstFormat = PIXEL_FORMAT_ARGB32; break; case PIXEL_FORMAT_XBGR32: DstFormat = PIXEL_FORMAT_XRGB32; break; case PIXEL_FORMAT_BGRA32: DstFormat = PIXEL_FORMAT_RGBA32; break; case PIXEL_FORMAT_BGRX32: DstFormat = PIXEL_FORMAT_RGBX32; break; case PIXEL_FORMAT_RGBA32: DstFormat = PIXEL_FORMAT_BGRA32; break; case PIXEL_FORMAT_RGBX32: DstFormat = PIXEL_FORMAT_BGRX32; break; case PIXEL_FORMAT_RGB24: DstFormat = PIXEL_FORMAT_BGR24; break; case PIXEL_FORMAT_BGR24: DstFormat = PIXEL_FORMAT_RGB24; break; case PIXEL_FORMAT_RGB16: DstFormat = PIXEL_FORMAT_BGR16; break; case PIXEL_FORMAT_BGR16: DstFormat = PIXEL_FORMAT_RGB16; break; case PIXEL_FORMAT_ARGB15: DstFormat = PIXEL_FORMAT_ABGR15; break; case PIXEL_FORMAT_RGB15: DstFormat = PIXEL_FORMAT_BGR15; break; case PIXEL_FORMAT_ABGR15: DstFormat = PIXEL_FORMAT_ARGB15; break; case PIXEL_FORMAT_BGR15: DstFormat = PIXEL_FORMAT_RGB15; break; default: break; } } return DstFormat; } static inline BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* WINPR_RESTRICT inPlane, UINT32 width, UINT32 height, BYTE* WINPR_RESTRICT outPlane, UINT32* WINPR_RESTRICT dstSize); static inline BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* WINPR_RESTRICT inPlane, UINT32 width, UINT32 height, BYTE* WINPR_RESTRICT outPlane); static inline INT32 planar_skip_plane_rle(const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize, UINT32 nWidth, UINT32 nHeight) { UINT32 used = 0; WINPR_ASSERT(pSrcData); for (UINT32 y = 0; y < nHeight; y++) { for (UINT32 x = 0; x < nWidth;) { if (used >= SrcSize) { WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRIu32, used, SrcSize); return -1; } const uint8_t controlByte = pSrcData[used++]; uint32_t nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte); uint32_t cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte); if (nRunLength == 1) { nRunLength = cRawBytes + 16; cRawBytes = 0; } else if (nRunLength == 2) { nRunLength = cRawBytes + 32; cRawBytes = 0; } used += cRawBytes; x += cRawBytes; x += nRunLength; if (x > nWidth) { WLog_ERR(TAG, "planar plane x %" PRIu32 " exceeds width %" PRIu32, x, nWidth); return -1; } if (used > SrcSize) { WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRId32, used, INT32_MAX); return -1; } } } if (used > INT32_MAX) { WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRIu32, used, SrcSize); return -1; } return (INT32)used; } static inline UINT8 clamp(INT16 val) { return (UINT8)val; } static inline INT32 planar_decompress_plane_rle_only(const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize, BYTE* WINPR_RESTRICT pDstData, UINT32 nWidth, UINT32 nHeight) { BYTE* previousScanline = nullptr; const BYTE* srcp = pSrcData; WINPR_ASSERT(nHeight <= INT32_MAX); WINPR_ASSERT(nWidth <= INT32_MAX); for (UINT32 y = 0; y < nHeight; y++) { BYTE* dstp = &pDstData[1ULL * (y)*nWidth]; INT16 pixel = 0; BYTE* currentScanline = dstp; for (UINT32 x = 0; x < nWidth;) { BYTE controlByte = *srcp; srcp++; if ((srcp - pSrcData) > SrcSize * 1ll) { WLog_ERR(TAG, "error reading input buffer"); return -1; } UINT32 nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte); UINT32 cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte); if (nRunLength == 1) { nRunLength = cRawBytes + 16; cRawBytes = 0; } else if (nRunLength == 2) { nRunLength = cRawBytes + 32; cRawBytes = 0; } if (((dstp + (cRawBytes + nRunLength)) - currentScanline) > nWidth * 1ll) { WLog_ERR(TAG, "too many pixels in scanline"); return -1; } if (!previousScanline) { /* first scanline, absolute values */ while (cRawBytes > 0) { pixel = *srcp; srcp++; *dstp = clamp(pixel); dstp++; x++; cRawBytes--; } while (nRunLength > 0) { *dstp = clamp(pixel); dstp++; x++; nRunLength--; } } else { /* delta values relative to previous scanline */ while (cRawBytes > 0) { UINT8 deltaValue = *srcp; srcp++; if (deltaValue & 1) { deltaValue = deltaValue >> 1; deltaValue = deltaValue + 1; pixel = WINPR_ASSERTING_INT_CAST(int16_t, -1 * (int16_t)deltaValue); } else { deltaValue = deltaValue >> 1; pixel = WINPR_ASSERTING_INT_CAST(INT16, deltaValue); } const INT16 delta = WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[x] + pixel); *dstp = clamp(delta); dstp++; x++; cRawBytes--; } while (nRunLength > 0) { const INT16 deltaValue = WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[x] + pixel); *dstp = clamp(deltaValue); dstp++; x++; nRunLength--; } } } previousScanline = currentScanline; } return (INT32)(srcp - pSrcData); } static inline INT32 planar_decompress_plane_rle(const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize, BYTE* WINPR_RESTRICT pDstData, UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst, UINT32 nWidth, UINT32 nHeight, UINT32 nChannel, BOOL vFlip) { INT32 beg = 0; INT32 end = 0; INT32 inc = 0; BYTE* previousScanline = nullptr; const BYTE* srcp = pSrcData; WINPR_ASSERT(nHeight <= INT32_MAX); WINPR_ASSERT(nWidth <= INT32_MAX); WINPR_ASSERT(nDstStep <= INT32_MAX); if (vFlip) { beg = (INT32)nHeight - 1; end = -1; inc = -1; } else { beg = 0; end = (INT32)nHeight; inc = 1; } for (INT32 y = beg; y != end; y += inc) { const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (4LL * nXDst) + nChannel * 1LL; BYTE* dstp = &pDstData[off]; BYTE* currentScanline = dstp; INT16 pixel = 0; for (INT32 x = 0; x < (INT32)nWidth;) { const BYTE controlByte = *srcp; srcp++; if ((srcp - pSrcData) > SrcSize * 1ll) { WLog_ERR(TAG, "error reading input buffer"); return -1; } UINT32 nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte); UINT32 cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte); if (nRunLength == 1) { nRunLength = cRawBytes + 16; cRawBytes = 0; } else if (nRunLength == 2) { nRunLength = cRawBytes + 32; cRawBytes = 0; } if (((dstp + (cRawBytes + nRunLength)) - currentScanline) > nWidth * 4ll) { WLog_ERR(TAG, "too many pixels in scanline"); return -1; } if (!previousScanline) { /* first scanline, absolute values */ while (cRawBytes > 0) { pixel = *srcp; srcp++; *dstp = WINPR_ASSERTING_INT_CAST(BYTE, pixel); dstp += 4; x++; cRawBytes--; } while (nRunLength > 0) { *dstp = WINPR_ASSERTING_INT_CAST(BYTE, pixel); dstp += 4; x++; nRunLength--; } } else { /* delta values relative to previous scanline */ while (cRawBytes > 0) { BYTE deltaValue = *srcp; srcp++; if (deltaValue & 1) { deltaValue = deltaValue >> 1; deltaValue = deltaValue + 1; pixel = WINPR_ASSERTING_INT_CAST(int16_t, -deltaValue); } else { deltaValue = deltaValue >> 1; pixel = deltaValue; } const INT16 delta = WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[4LL * x] + pixel); *dstp = clamp(delta); dstp += 4; x++; cRawBytes--; } while (nRunLength > 0) { const INT16 deltaValue = WINPR_ASSERTING_INT_CAST(int16_t, pixel + previousScanline[4LL * x]); *dstp = clamp(deltaValue); dstp += 4; x++; nRunLength--; } } } previousScanline = currentScanline; } return (INT32)(srcp - pSrcData); } static inline INT32 planar_set_plane(BYTE bValue, BYTE* pDstData, UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst, UINT32 nWidth, UINT32 nHeight, UINT32 nChannel, BOOL vFlip) { INT32 beg = 0; INT32 end = (INT32)nHeight; INT32 inc = 1; WINPR_ASSERT(nHeight <= INT32_MAX); WINPR_ASSERT(nWidth <= INT32_MAX); WINPR_ASSERT(nDstStep <= INT32_MAX); if (vFlip) { beg = (INT32)nHeight - 1; end = -1; inc = -1; } for (INT32 y = beg; y != end; y += inc) { const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (4LL * nXDst) + nChannel * 1LL; BYTE* dstp = &pDstData[off]; for (INT32 x = 0; x < (INT32)nWidth; ++x) { *dstp = bValue; dstp += 4; } } return 0; } static inline BOOL writeLine(BYTE** WINPR_RESTRICT ppRgba, UINT32 DstFormat, UINT32 width, const BYTE** WINPR_RESTRICT ppR, const BYTE** WINPR_RESTRICT ppG, const BYTE** WINPR_RESTRICT ppB, const BYTE** WINPR_RESTRICT ppA) { WINPR_ASSERT(ppRgba); WINPR_ASSERT(ppR); WINPR_ASSERT(ppG); WINPR_ASSERT(ppB); switch (DstFormat) { case PIXEL_FORMAT_BGRA32: for (UINT32 x = 0; x < width; x++) { *(*ppRgba)++ = *(*ppB)++; *(*ppRgba)++ = *(*ppG)++; *(*ppRgba)++ = *(*ppR)++; *(*ppRgba)++ = *(*ppA)++; } return TRUE; case PIXEL_FORMAT_BGRX32: for (UINT32 x = 0; x < width; x++) { *(*ppRgba)++ = *(*ppB)++; *(*ppRgba)++ = *(*ppG)++; *(*ppRgba)++ = *(*ppR)++; *(*ppRgba)++ = 0xFF; } return TRUE; default: if (ppA) { for (UINT32 x = 0; x < width; x++) { BYTE alpha = *(*ppA)++; UINT32 color = FreeRDPGetColor(DstFormat, *(*ppR)++, *(*ppG)++, *(*ppB)++, alpha); FreeRDPWriteColor(*ppRgba, DstFormat, color); *ppRgba += FreeRDPGetBytesPerPixel(DstFormat); } } else { const BYTE alpha = 0xFF; for (UINT32 x = 0; x < width; x++) { UINT32 color = FreeRDPGetColor(DstFormat, *(*ppR)++, *(*ppG)++, *(*ppB)++, alpha); FreeRDPWriteColor(*ppRgba, DstFormat, color); *ppRgba += FreeRDPGetBytesPerPixel(DstFormat); } } return TRUE; } } static inline BOOL planar_decompress_planes_raw(const BYTE* WINPR_RESTRICT pSrcData[4], BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat, UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst, UINT32 nWidth, UINT32 nHeight, BOOL vFlip, UINT32 totalHeight) { INT32 beg = 0; INT32 end = 0; INT32 inc = 0; const BYTE* pR = pSrcData[0]; const BYTE* pG = pSrcData[1]; const BYTE* pB = pSrcData[2]; const BYTE* pA = pSrcData[3]; const UINT32 bpp = FreeRDPGetBytesPerPixel(DstFormat); if (vFlip) { beg = WINPR_ASSERTING_INT_CAST(int32_t, nHeight - 1); end = -1; inc = -1; } else { beg = 0; end = WINPR_ASSERTING_INT_CAST(int32_t, nHeight); inc = 1; } if (nYDst + nHeight > totalHeight) { WLog_ERR(TAG, "planar plane destination Y %" PRIu32 " + height %" PRIu32 " exceeds totalHeight %" PRIu32, nYDst, nHeight, totalHeight); return FALSE; } if ((nXDst + nWidth) * bpp > nDstStep) { WLog_ERR(TAG, "planar plane destination (X %" PRIu32 " + width %" PRIu32 ") * bpp %" PRIu32 " exceeds stride %" PRIu32, nXDst, nWidth, bpp, nDstStep); return FALSE; } for (INT32 y = beg; y != end; y += inc) { BYTE* pRGB = nullptr; if (y > WINPR_ASSERTING_INT_CAST(INT64, nHeight)) { WLog_ERR(TAG, "planar plane destination Y %" PRId32 " exceeds height %" PRIu32, y, nHeight); return FALSE; } const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (1LL * nXDst * bpp); pRGB = &pDstData[off]; if (!writeLine(&pRGB, DstFormat, nWidth, &pR, &pG, &pB, &pA)) return FALSE; } return TRUE; } static BOOL planar_subsample_expand(const BYTE* WINPR_RESTRICT plane, size_t planeLength, UINT32 nWidth, UINT32 nHeight, UINT32 nPlaneWidth, UINT32 nPlaneHeight, BYTE* WINPR_RESTRICT deltaPlane) { size_t pos = 0; WINPR_UNUSED(planeLength); WINPR_ASSERT(plane); WINPR_ASSERT(deltaPlane); if (nWidth > nPlaneWidth * 2) { WLog_ERR(TAG, "planar subsample width %" PRIu32 " > PlaneWidth %" PRIu32 " * 2", nWidth, nPlaneWidth); return FALSE; } if (nHeight > nPlaneHeight * 2) { WLog_ERR(TAG, "planar subsample height %" PRIu32 " > PlaneHeight %" PRIu32 " * 2", nHeight, nPlaneHeight); return FALSE; } for (size_t y = 0; y < nHeight; y++) { const BYTE* src = plane + y / 2 * nPlaneWidth; for (UINT32 x = 0; x < nWidth; x++) { deltaPlane[pos++] = src[x / 2]; } } return TRUE; } #if !defined(WITHOUT_FREERDP_3x_DEPRECATED) BOOL planar_decompress(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize, UINT32 nSrcWidth, UINT32 nSrcHeight, BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat, UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst, UINT32 nDstWidth, UINT32 nDstHeight, BOOL vFlip) { return freerdp_bitmap_decompress_planar(planar, pSrcData, SrcSize, nSrcWidth, nSrcHeight, pDstData, DstFormat, nDstStep, nXDst, nYDst, nDstWidth, nDstHeight, vFlip); } #endif BOOL freerdp_bitmap_decompress_planar(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize, UINT32 nSrcWidth, UINT32 nSrcHeight, BYTE* WINPR_RESTRICT pDstData, UINT32 DstFormat, UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst, UINT32 nDstWidth, UINT32 nDstHeight, BOOL vFlip) { BOOL useAlpha = FALSE; INT32 status = 0; INT32 rleSizes[4] = { 0, 0, 0, 0 }; UINT32 rawSizes[4] = WINPR_C_ARRAY_INIT; UINT32 rawWidths[4] = WINPR_C_ARRAY_INIT; UINT32 rawHeights[4] = WINPR_C_ARRAY_INIT; const BYTE* planes[4] = WINPR_C_ARRAY_INIT; const UINT32 w = MIN(nSrcWidth, nDstWidth); const UINT32 h = MIN(nSrcHeight, nDstHeight); const primitives_t* prims = primitives_get(); WINPR_ASSERT(planar); WINPR_ASSERT(prims); if (planar->maxWidth < nSrcWidth) return FALSE; if (planar->maxHeight < nSrcHeight) return FALSE; const UINT32 bpp = FreeRDPGetBytesPerPixel(DstFormat); if (nDstStep <= 0) nDstStep = nDstWidth * bpp; const BYTE* srcp = pSrcData; if (!pSrcData) { WLog_ERR(TAG, "Invalid argument pSrcData=nullptr"); return FALSE; } if (!pDstData) { WLog_ERR(TAG, "Invalid argument pDstData=nullptr"); return FALSE; } const BYTE FormatHeader = *srcp++; const BYTE cll = (FormatHeader & PLANAR_FORMAT_HEADER_CLL_MASK); const BYTE cs = (FormatHeader & PLANAR_FORMAT_HEADER_CS) != 0; const BYTE rle = (FormatHeader & PLANAR_FORMAT_HEADER_RLE) != 0; const BYTE alpha = !(FormatHeader & PLANAR_FORMAT_HEADER_NA); DstFormat = planar_invert_format(planar, alpha, DstFormat); if (alpha) useAlpha = FreeRDPColorHasAlpha(DstFormat); // WLog_INFO(TAG, "CLL: %"PRIu32" CS: %"PRIu8" RLE: %"PRIu8" ALPHA: %"PRIu8"", cll, cs, rle, // alpha); if (!cll && cs) { WLog_ERR(TAG, "Chroma subsampling requires YCoCg and does not work with RGB data"); return FALSE; /* Chroma subsampling requires YCoCg */ } const UINT32 subWidth = (nSrcWidth / 2) + (nSrcWidth % 2); const UINT32 subHeight = (nSrcHeight / 2) + (nSrcHeight % 2); const UINT32 planeSize = nSrcWidth * nSrcHeight; const UINT32 subSize = subWidth * subHeight; if (!cs) { rawSizes[0] = planeSize; /* LumaOrRedPlane */ rawWidths[0] = nSrcWidth; rawHeights[0] = nSrcHeight; rawSizes[1] = planeSize; /* OrangeChromaOrGreenPlane */ rawWidths[1] = nSrcWidth; rawHeights[1] = nSrcHeight; rawSizes[2] = planeSize; /* GreenChromaOrBluePlane */ rawWidths[2] = nSrcWidth; rawHeights[2] = nSrcHeight; rawSizes[3] = planeSize; /* AlphaPlane */ rawWidths[3] = nSrcWidth; rawHeights[3] = nSrcHeight; } else /* Chroma Subsampling */ { rawSizes[0] = planeSize; /* LumaOrRedPlane */ rawWidths[0] = nSrcWidth; rawHeights[0] = nSrcHeight; rawSizes[1] = subSize; /* OrangeChromaOrGreenPlane */ rawWidths[1] = subWidth; rawHeights[1] = subHeight; rawSizes[2] = subSize; /* GreenChromaOrBluePlane */ rawWidths[2] = subWidth; rawHeights[2] = subHeight; rawSizes[3] = planeSize; /* AlphaPlane */ rawWidths[3] = nSrcWidth; rawHeights[3] = nSrcHeight; } const size_t diff = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(srcp - pSrcData)); if (SrcSize < diff) { WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff); return FALSE; } if (!rle) /* RAW */ { UINT32 base = planeSize * 3; if (cs) base = planeSize + planeSize / 2; if (alpha) { if ((SrcSize - diff) < (planeSize + base)) { WLog_ERR(TAG, "Alpha plane size mismatch %" PRIuz " < %" PRIu32, SrcSize - diff, (planeSize + base)); return FALSE; } planes[3] = srcp; /* AlphaPlane */ planes[0] = planes[3] + rawSizes[3]; /* LumaOrRedPlane */ planes[1] = planes[0] + rawSizes[0]; /* OrangeChromaOrGreenPlane */ planes[2] = planes[1] + rawSizes[1]; /* GreenChromaOrBluePlane */ if ((planes[2] + rawSizes[2]) > &pSrcData[SrcSize]) { WLog_ERR(TAG, "plane size mismatch %p + %" PRIu32 " > %p", WINPR_CXX_COMPAT_CAST(const void*, planes[2]), rawSizes[2], WINPR_CXX_COMPAT_CAST(const void*, &pSrcData[SrcSize])); return FALSE; } } else { if ((SrcSize - diff) < base) { WLog_ERR(TAG, "plane size mismatch %" PRIuz " < %" PRIu32, SrcSize - diff, base); return FALSE; } planes[0] = srcp; /* LumaOrRedPlane */ planes[1] = planes[0] + rawSizes[0]; /* OrangeChromaOrGreenPlane */ planes[2] = planes[1] + rawSizes[1]; /* GreenChromaOrBluePlane */ if ((planes[2] + rawSizes[2]) > &pSrcData[SrcSize]) { WLog_ERR(TAG, "plane size mismatch %p + %" PRIu32 " > %p", WINPR_CXX_COMPAT_CAST(const void*, planes[2]), rawSizes[2], WINPR_CXX_COMPAT_CAST(const void*, &pSrcData[SrcSize])); return FALSE; } } } else /* RLE */ { if (alpha) { planes[3] = srcp; rleSizes[3] = planar_skip_plane_rle(planes[3], (UINT32)(SrcSize - diff), rawWidths[3], rawHeights[3]); /* AlphaPlane */ if (rleSizes[3] < 0) return FALSE; planes[0] = planes[3] + rleSizes[3]; } else planes[0] = srcp; const size_t diff0 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[0] - pSrcData)); if (SrcSize < diff0) { WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff0); return FALSE; } rleSizes[0] = planar_skip_plane_rle(planes[0], (UINT32)(SrcSize - diff0), rawWidths[0], rawHeights[0]); /* RedPlane */ if (rleSizes[0] < 0) return FALSE; planes[1] = planes[0] + rleSizes[0]; const size_t diff1 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[1] - pSrcData)); if (SrcSize < diff1) { WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff1); return FALSE; } rleSizes[1] = planar_skip_plane_rle(planes[1], (UINT32)(SrcSize - diff1), rawWidths[1], rawHeights[1]); /* GreenPlane */ if (rleSizes[1] < 1) return FALSE; planes[2] = planes[1] + rleSizes[1]; const size_t diff2 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[2] - pSrcData)); if (SrcSize < diff2) { WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff); return FALSE; } rleSizes[2] = planar_skip_plane_rle(planes[2], (UINT32)(SrcSize - diff2), rawWidths[2], rawHeights[2]); /* BluePlane */ if (rleSizes[2] < 1) return FALSE; } if (!cll) /* RGB */ { UINT32 TempFormat = 0; BYTE* pTempData = pDstData; UINT32 nTempStep = nDstStep; UINT32 nTotalHeight = nYDst + nDstHeight; if (useAlpha) TempFormat = PIXEL_FORMAT_BGRA32; else TempFormat = PIXEL_FORMAT_BGRX32; TempFormat = planar_invert_format(planar, alpha, TempFormat); if ((TempFormat != DstFormat) || (nSrcWidth != nDstWidth) || (nSrcHeight != nDstHeight)) { pTempData = planar->pTempData; nTempStep = planar->nTempStep; nTotalHeight = planar->maxHeight; } if (!rle) /* RAW */ { if (!planar_decompress_planes_raw(planes, pTempData, TempFormat, nTempStep, nXDst, nYDst, nSrcWidth, nSrcHeight, vFlip, nTotalHeight)) return FALSE; if (alpha) srcp += rawSizes[0] + rawSizes[1] + rawSizes[2] + rawSizes[3]; else /* NoAlpha */ srcp += rawSizes[0] + rawSizes[1] + rawSizes[2]; if ((SrcSize - (srcp - pSrcData)) == 1) srcp++; /* pad */ } else /* RLE */ { if (nYDst + nSrcHeight > nTotalHeight) { WLog_ERR(TAG, "planar plane destination Y %" PRIu32 " + height %" PRIu32 " exceeds totalHeight %" PRIu32, nYDst, nSrcHeight, nTotalHeight); return FALSE; } if ((nXDst + nSrcWidth) * bpp > nDstStep) { WLog_ERR(TAG, "planar plane destination (X %" PRIu32 " + width %" PRIu32 ") * bpp %" PRIu32 " exceeds stride %" PRIu32, nXDst, nSrcWidth, bpp, nDstStep); return FALSE; } status = planar_decompress_plane_rle( planes[0], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[0]), pTempData, nTempStep, nXDst, nYDst, nSrcWidth, nSrcHeight, 2, vFlip); /* RedPlane */ if (status < 0) return FALSE; status = planar_decompress_plane_rle( planes[1], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[1]), pTempData, nTempStep, nXDst, nYDst, nSrcWidth, nSrcHeight, 1, vFlip); /* GreenPlane */ if (status < 0) return FALSE; status = planar_decompress_plane_rle( planes[2], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[2]), pTempData, nTempStep, nXDst, nYDst, nSrcWidth, nSrcHeight, 0, vFlip); /* BluePlane */ if (status < 0) return FALSE; srcp += rleSizes[0] + rleSizes[1] + rleSizes[2]; if (useAlpha) { status = planar_decompress_plane_rle( planes[3], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[3]), pTempData, nTempStep, nXDst, nYDst, nSrcWidth, nSrcHeight, 3, vFlip); /* AlphaPlane */ } else status = planar_set_plane(0xFF, pTempData, nTempStep, nXDst, nYDst, nSrcWidth, nSrcHeight, 3, vFlip); if (status < 0) return FALSE; if (alpha) srcp += rleSizes[3]; } if (pTempData != pDstData) { if (!freerdp_image_copy_no_overlap(pDstData, DstFormat, nDstStep, nXDst, nYDst, w, h, pTempData, TempFormat, nTempStep, nXDst, nYDst, nullptr, FREERDP_FLIP_NONE)) { WLog_ERR(TAG, "planar image copy failed"); return FALSE; } } } else /* YCoCg */ { UINT32 TempFormat = 0; BYTE* pTempData = planar->pTempData; UINT32 nTempStep = planar->nTempStep; UINT32 nTotalHeight = planar->maxHeight; BYTE* dst = &pDstData[nXDst * FreeRDPGetBytesPerPixel(DstFormat) + nYDst * nDstStep]; if (useAlpha) TempFormat = PIXEL_FORMAT_BGRA32; else TempFormat = PIXEL_FORMAT_BGRX32; if (!pTempData) return FALSE; if (rle) /* RLE encoded data. Decode and handle it like raw data. */ { BYTE* rleBuffer[4] = WINPR_C_ARRAY_INIT; if (!planar->rlePlanesBuffer) return FALSE; rleBuffer[3] = planar->rlePlanesBuffer; /* AlphaPlane */ rleBuffer[0] = rleBuffer[3] + planeSize; /* LumaOrRedPlane */ rleBuffer[1] = rleBuffer[0] + planeSize; /* OrangeChromaOrGreenPlane */ rleBuffer[2] = rleBuffer[1] + planeSize; /* GreenChromaOrBluePlane */ if (useAlpha) { status = planar_decompress_plane_rle_only( planes[3], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[3]), rleBuffer[3], rawWidths[3], rawHeights[3]); /* AlphaPlane */ if (status < 0) return FALSE; } if (alpha) srcp += rleSizes[3]; status = planar_decompress_plane_rle_only( planes[0], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[0]), rleBuffer[0], rawWidths[0], rawHeights[0]); /* LumaPlane */ if (status < 0) return FALSE; status = planar_decompress_plane_rle_only( planes[1], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[1]), rleBuffer[1], rawWidths[1], rawHeights[1]); /* OrangeChromaPlane */ if (status < 0) return FALSE; status = planar_decompress_plane_rle_only( planes[2], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[2]), rleBuffer[2], rawWidths[2], rawHeights[2]); /* GreenChromaPlane */ if (status < 0) return FALSE; planes[0] = rleBuffer[0]; planes[1] = rleBuffer[1]; planes[2] = rleBuffer[2]; planes[3] = rleBuffer[3]; } /* RAW */ { if (cs) { /* Chroma subsampling for Co and Cg: * Each pixel contains the value that should be expanded to * [2x,2y;2x+1,2y;2x+1,2y+1;2x;2y+1] */ if (!planar_subsample_expand(planes[1], rawSizes[1], nSrcWidth, nSrcHeight, rawWidths[1], rawHeights[1], planar->deltaPlanes[0])) return FALSE; planes[1] = planar->deltaPlanes[0]; rawSizes[1] = planeSize; /* OrangeChromaOrGreenPlane */ rawWidths[1] = nSrcWidth; rawHeights[1] = nSrcHeight; if (!planar_subsample_expand(planes[2], rawSizes[2], nSrcWidth, nSrcHeight, rawWidths[2], rawHeights[2], planar->deltaPlanes[1])) return FALSE; planes[2] = planar->deltaPlanes[1]; rawSizes[2] = planeSize; /* GreenChromaOrBluePlane */ rawWidths[2] = nSrcWidth; rawHeights[2] = nSrcHeight; } if (!planar_decompress_planes_raw(planes, pTempData, TempFormat, nTempStep, nXDst, nYDst, nSrcWidth, nSrcHeight, vFlip, nTotalHeight)) return FALSE; if (alpha) srcp += rawSizes[0] + rawSizes[1] + rawSizes[2] + rawSizes[3]; else /* NoAlpha */ srcp += rawSizes[0] + rawSizes[1] + rawSizes[2]; if ((SrcSize - (srcp - pSrcData)) == 1) srcp++; /* pad */ } WINPR_ASSERT(prims->YCoCgToRGB_8u_AC4R); int rc = prims->YCoCgToRGB_8u_AC4R( pTempData, WINPR_ASSERTING_INT_CAST(int32_t, nTempStep), dst, DstFormat, WINPR_ASSERTING_INT_CAST(int32_t, nDstStep), w, h, cll, useAlpha); if (rc != PRIMITIVES_SUCCESS) { WLog_ERR(TAG, "YCoCgToRGB_8u_AC4R failed with %d", rc); return FALSE; } } WINPR_UNUSED(srcp); return TRUE; } static inline BOOL freerdp_split_color_planes(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, const BYTE* WINPR_RESTRICT data, UINT32 format, UINT32 width, UINT32 height, UINT32 scanline, BYTE* WINPR_RESTRICT planes[4]) { WINPR_ASSERT(planar); if ((width > INT32_MAX) || (height > INT32_MAX) || (scanline > INT32_MAX)) return FALSE; if (scanline == 0) scanline = width * FreeRDPGetBytesPerPixel(format); if (planar->topdown) { UINT32 k = 0; for (UINT32 i = 0; i < height; i++) { const BYTE* pixel = &data[1ULL * scanline * i]; for (UINT32 j = 0; j < width; j++) { const UINT32 color = FreeRDPReadColor(pixel, format); pixel += FreeRDPGetBytesPerPixel(format); FreeRDPSplitColor(color, format, &planes[1][k], &planes[2][k], &planes[3][k], &planes[0][k], nullptr); k++; } } } else { UINT32 k = 0; for (INT64 i = (INT64)height - 1; i >= 0; i--) { const BYTE* pixel = &data[1ULL * scanline * (UINT32)i]; for (UINT32 j = 0; j < width; j++) { const UINT32 color = FreeRDPReadColor(pixel, format); pixel += FreeRDPGetBytesPerPixel(format); FreeRDPSplitColor(color, format, &planes[1][k], &planes[2][k], &planes[3][k], &planes[0][k], nullptr); k++; } } } return TRUE; } static inline UINT32 freerdp_bitmap_planar_write_rle_bytes(const BYTE* WINPR_RESTRICT pInBuffer, UINT32 cRawBytes, UINT32 nRunLength, BYTE* WINPR_RESTRICT pOutBuffer, UINT32 outBufferSize) { const BYTE* pInput = pInBuffer; BYTE* pOutput = pOutBuffer; BYTE controlByte = 0; UINT32 nBytesToWrite = 0; if (!cRawBytes && !nRunLength) return 0; if (nRunLength < 3) { cRawBytes += nRunLength; nRunLength = 0; } while (cRawBytes) { if (cRawBytes < 16) { if (nRunLength > 15) { if (nRunLength < 18) { controlByte = PLANAR_CONTROL_BYTE(13, cRawBytes); nRunLength -= 13; cRawBytes = 0; } else { controlByte = PLANAR_CONTROL_BYTE(15, cRawBytes); nRunLength -= 15; cRawBytes = 0; } } else { controlByte = PLANAR_CONTROL_BYTE(nRunLength, cRawBytes); nRunLength = 0; cRawBytes = 0; } } else { controlByte = PLANAR_CONTROL_BYTE(0, 15); cRawBytes -= 15; } if (outBufferSize < 1) return 0; outBufferSize--; *pOutput = controlByte; pOutput++; nBytesToWrite = (controlByte >> 4); if (nBytesToWrite) { if (outBufferSize < nBytesToWrite) return 0; outBufferSize -= nBytesToWrite; CopyMemory(pOutput, pInput, nBytesToWrite); pOutput += nBytesToWrite; pInput += nBytesToWrite; } } while (nRunLength) { if (nRunLength > 47) { if (nRunLength < 50) { controlByte = PLANAR_CONTROL_BYTE(2, 13); nRunLength -= 45; } else { controlByte = PLANAR_CONTROL_BYTE(2, 15); nRunLength -= 47; } } else if (nRunLength > 31) { controlByte = PLANAR_CONTROL_BYTE(2, (nRunLength - 32)); nRunLength = 0; } else if (nRunLength > 15) { controlByte = PLANAR_CONTROL_BYTE(1, (nRunLength - 16)); nRunLength = 0; } else { controlByte = PLANAR_CONTROL_BYTE(nRunLength, 0); nRunLength = 0; } if (outBufferSize < 1) return 0; --outBufferSize; *pOutput = controlByte; pOutput++; } const intptr_t diff = (pOutput - pOutBuffer); if ((diff < 0) || (diff > UINT32_MAX)) return 0; return (UINT32)diff; } static inline UINT32 freerdp_bitmap_planar_encode_rle_bytes(const BYTE* WINPR_RESTRICT pInBuffer, UINT32 inBufferSize, BYTE* WINPR_RESTRICT pOutBuffer, UINT32 outBufferSize) { BYTE symbol = 0; const BYTE* pInput = pInBuffer; BYTE* pOutput = pOutBuffer; const BYTE* pBytes = nullptr; UINT32 cRawBytes = 0; UINT32 nRunLength = 0; UINT32 nBytesWritten = 0; UINT32 nTotalBytesWritten = 0; if (!outBufferSize) return 0; do { if (!inBufferSize) break; const UINT32 bSymbolMatch = (symbol == *pInput) != 0; symbol = *pInput; pInput++; inBufferSize--; if (nRunLength && !bSymbolMatch) { if (nRunLength < 3) { cRawBytes += nRunLength; nRunLength = 0; } else { pBytes = pInput - (cRawBytes + nRunLength + 1); nBytesWritten = freerdp_bitmap_planar_write_rle_bytes(pBytes, cRawBytes, nRunLength, pOutput, outBufferSize); nRunLength = 0; if (!nBytesWritten || (nBytesWritten > outBufferSize)) return nRunLength; nTotalBytesWritten += nBytesWritten; outBufferSize -= nBytesWritten; pOutput += nBytesWritten; cRawBytes = 0; } } nRunLength += bSymbolMatch; cRawBytes += (!bSymbolMatch) != 0; } while (outBufferSize); if (cRawBytes || nRunLength) { pBytes = pInput - (cRawBytes + nRunLength); nBytesWritten = freerdp_bitmap_planar_write_rle_bytes(pBytes, cRawBytes, nRunLength, pOutput, outBufferSize); if (!nBytesWritten) return 0; nTotalBytesWritten += nBytesWritten; } if (inBufferSize) return 0; return nTotalBytesWritten; } BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* WINPR_RESTRICT inPlane, UINT32 width, UINT32 height, BYTE* WINPR_RESTRICT outPlane, UINT32* WINPR_RESTRICT dstSize) { if (!outPlane) return FALSE; UINT32 index = 0; const BYTE* pInput = inPlane; BYTE* pOutput = outPlane; UINT32 outBufferSize = *dstSize; UINT32 nTotalBytesWritten = 0; while (outBufferSize > 0) { const UINT32 nBytesWritten = freerdp_bitmap_planar_encode_rle_bytes(pInput, width, pOutput, outBufferSize); if ((!nBytesWritten) || (nBytesWritten > outBufferSize)) return FALSE; outBufferSize -= nBytesWritten; nTotalBytesWritten += nBytesWritten; pOutput += nBytesWritten; pInput += width; index++; if (index >= height) break; } *dstSize = nTotalBytesWritten; return TRUE; } static inline BOOL freerdp_bitmap_planar_compress_planes_rle(BYTE* WINPR_RESTRICT inPlanes[4], UINT32 width, UINT32 height, BYTE* WINPR_RESTRICT outPlanes, UINT32* WINPR_RESTRICT dstSizes, BOOL skipAlpha) { UINT32 outPlanesSize = width * height * 4; /* AlphaPlane */ if (skipAlpha) { dstSizes[0] = 0; } else { dstSizes[0] = outPlanesSize; if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[0], width, height, outPlanes, &dstSizes[0])) return FALSE; outPlanes += dstSizes[0]; outPlanesSize -= dstSizes[0]; } /* LumaOrRedPlane */ dstSizes[1] = outPlanesSize; if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[1], width, height, outPlanes, &dstSizes[1])) return FALSE; outPlanes += dstSizes[1]; outPlanesSize -= dstSizes[1]; /* OrangeChromaOrGreenPlane */ dstSizes[2] = outPlanesSize; if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[2], width, height, outPlanes, &dstSizes[2])) return FALSE; outPlanes += dstSizes[2]; outPlanesSize -= dstSizes[2]; /* GreenChromeOrBluePlane */ dstSizes[3] = outPlanesSize; return (freerdp_bitmap_planar_compress_plane_rle(inPlanes[3], width, height, outPlanes, &dstSizes[3])); } BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* WINPR_RESTRICT inPlane, UINT32 width, UINT32 height, BYTE* WINPR_RESTRICT outPlane) { if (!outPlane) { if (width * height == 0) return nullptr; outPlane = (BYTE*)calloc(height, width); if (!outPlane) return nullptr; } // first line is copied as is CopyMemory(outPlane, inPlane, width); for (UINT32 y = 1; y < height; y++) { const size_t off = 1ull * width * y; BYTE* outPtr = &outPlane[off]; const BYTE* srcPtr = &inPlane[off]; const BYTE* prevLinePtr = &inPlane[off - width]; for (UINT32 x = 0; x < width; x++) { const int delta = (int)srcPtr[x] - (int)prevLinePtr[x]; const int s2c1i = (delta >= 0) ? delta : (INT_MAX + delta) + 1; const int8_t s2c1 = WINPR_CXX_COMPAT_CAST(int8_t, s2c1i); const uint32_t s2c = (s2c1 >= 0) ? ((UINT32)s2c1 << 1) : (((UINT32)(~(s2c1) + 1) << 1) - 1); outPtr[x] = (BYTE)s2c; } } return outPlane; } static inline BOOL freerdp_bitmap_planar_delta_encode_planes(BYTE* WINPR_RESTRICT inPlanes[4], UINT32 width, UINT32 height, BYTE* WINPR_RESTRICT outPlanes[4]) { for (UINT32 i = 0; i < 4; i++) { outPlanes[i] = freerdp_bitmap_planar_delta_encode_plane(inPlanes[i], width, height, outPlanes[i]); if (!outPlanes[i]) return FALSE; } return TRUE; } BYTE* freerdp_bitmap_compress_planar(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT context, const BYTE* WINPR_RESTRICT data, UINT32 format, UINT32 width, UINT32 height, UINT32 scanline, BYTE* WINPR_RESTRICT dstData, UINT32* WINPR_RESTRICT pDstSize) { UINT32 size = 0; BYTE* dstp = nullptr; UINT32 dstSizes[4] = WINPR_C_ARRAY_INIT; BYTE FormatHeader = 0; if (!context || !context->rlePlanesBuffer) return nullptr; if (context->AllowSkipAlpha) FormatHeader |= PLANAR_FORMAT_HEADER_NA; const UINT32 planeSize = width * height; if (!context->AllowSkipAlpha) format = planar_invert_format(context, TRUE, format); if (!freerdp_split_color_planes(context, data, format, width, height, scanline, context->planes)) return nullptr; if (context->AllowRunLengthEncoding) { if (!freerdp_bitmap_planar_delta_encode_planes(context->planes, width, height, context->deltaPlanes)) return nullptr; if (!freerdp_bitmap_planar_compress_planes_rle(context->deltaPlanes, width, height, context->rlePlanesBuffer, dstSizes, context->AllowSkipAlpha)) return nullptr; { uint32_t offset = 0; FormatHeader |= PLANAR_FORMAT_HEADER_RLE; context->rlePlanes[0] = &context->rlePlanesBuffer[offset]; offset += dstSizes[0]; context->rlePlanes[1] = &context->rlePlanesBuffer[offset]; offset += dstSizes[1]; context->rlePlanes[2] = &context->rlePlanesBuffer[offset]; offset += dstSizes[2]; context->rlePlanes[3] = &context->rlePlanesBuffer[offset]; #if defined(WITH_DEBUG_CODECS) WLog_DBG(TAG, "R: [%" PRIu32 "/%" PRIu32 "] G: [%" PRIu32 "/%" PRIu32 "] B: [%" PRIu32 " / %" PRIu32 "] ", dstSizes[1], planeSize, dstSizes[2], planeSize, dstSizes[3], planeSize); #endif } } if (FormatHeader & PLANAR_FORMAT_HEADER_RLE) { if (!context->AllowRunLengthEncoding) return nullptr; if (context->rlePlanes[0] == nullptr) return nullptr; if (context->rlePlanes[1] == nullptr) return nullptr; if (context->rlePlanes[2] == nullptr) return nullptr; if (context->rlePlanes[3] == nullptr) return nullptr; } if (!dstData) { size = 1; if (!(FormatHeader & PLANAR_FORMAT_HEADER_NA)) { if (FormatHeader & PLANAR_FORMAT_HEADER_RLE) size += dstSizes[0]; else size += planeSize; } if (FormatHeader & PLANAR_FORMAT_HEADER_RLE) size += (dstSizes[1] + dstSizes[2] + dstSizes[3]); else size += (planeSize * 3); if (!(FormatHeader & PLANAR_FORMAT_HEADER_RLE)) size++; dstData = malloc(size); if (!dstData) return nullptr; *pDstSize = size; } dstp = dstData; *dstp = FormatHeader; /* FormatHeader */ dstp++; /* AlphaPlane */ if (!(FormatHeader & PLANAR_FORMAT_HEADER_NA)) { if (FormatHeader & PLANAR_FORMAT_HEADER_RLE) { CopyMemory(dstp, context->rlePlanes[0], dstSizes[0]); /* Alpha */ dstp += dstSizes[0]; } else { CopyMemory(dstp, context->planes[0], planeSize); /* Alpha */ dstp += planeSize; } } /* LumaOrRedPlane */ if (FormatHeader & PLANAR_FORMAT_HEADER_RLE) { CopyMemory(dstp, context->rlePlanes[1], dstSizes[1]); /* Red */ dstp += dstSizes[1]; } else { CopyMemory(dstp, context->planes[1], planeSize); /* Red */ dstp += planeSize; } /* OrangeChromaOrGreenPlane */ if (FormatHeader & PLANAR_FORMAT_HEADER_RLE) { CopyMemory(dstp, context->rlePlanes[2], dstSizes[2]); /* Green */ dstp += dstSizes[2]; } else { CopyMemory(dstp, context->planes[2], planeSize); /* Green */ dstp += planeSize; } /* GreenChromeOrBluePlane */ if (FormatHeader & PLANAR_FORMAT_HEADER_RLE) { CopyMemory(dstp, context->rlePlanes[3], dstSizes[3]); /* Blue */ dstp += dstSizes[3]; } else { CopyMemory(dstp, context->planes[3], planeSize); /* Blue */ dstp += planeSize; } /* Pad1 (1 byte) */ if (!(FormatHeader & PLANAR_FORMAT_HEADER_RLE)) { *dstp = 0; dstp++; } const intptr_t diff = (dstp - dstData); if ((diff < 0) || (diff > UINT32_MAX)) { free(dstData); return nullptr; } size = (UINT32)diff; *pDstSize = size; return dstData; } BOOL freerdp_bitmap_planar_context_reset(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT context, UINT32 width, UINT32 height) { if (!context) return FALSE; context->bgr = FALSE; context->maxWidth = PLANAR_ALIGN(width, 4); context->maxHeight = PLANAR_ALIGN(height, 4); { const UINT64 tmp = (UINT64)context->maxWidth * context->maxHeight; if (tmp > UINT32_MAX) return FALSE; context->maxPlaneSize = (UINT32)tmp; } if (context->maxWidth > UINT32_MAX / 4) return FALSE; context->nTempStep = context->maxWidth * 4; memset((void*)context->planes, 0, sizeof(context->planes)); memset((void*)context->rlePlanes, 0, sizeof(context->rlePlanes)); memset((void*)context->deltaPlanes, 0, sizeof(context->deltaPlanes)); if (context->maxPlaneSize > 0) { void* tmp = winpr_aligned_recalloc(context->planesBuffer, context->maxPlaneSize, 4, 32); if (!tmp) return FALSE; context->planesBuffer = tmp; tmp = winpr_aligned_recalloc(context->pTempData, context->maxPlaneSize, 6, 32); if (!tmp) return FALSE; context->pTempData = tmp; tmp = winpr_aligned_recalloc(context->deltaPlanesBuffer, context->maxPlaneSize, 4, 32); if (!tmp) return FALSE; context->deltaPlanesBuffer = tmp; tmp = winpr_aligned_recalloc(context->rlePlanesBuffer, context->maxPlaneSize, 4, 32); if (!tmp) return FALSE; context->rlePlanesBuffer = tmp; context->planes[0] = &context->planesBuffer[0ULL * context->maxPlaneSize]; context->planes[1] = &context->planesBuffer[1ULL * context->maxPlaneSize]; context->planes[2] = &context->planesBuffer[2ULL * context->maxPlaneSize]; context->planes[3] = &context->planesBuffer[3ULL * context->maxPlaneSize]; context->deltaPlanes[0] = &context->deltaPlanesBuffer[0ULL * context->maxPlaneSize]; context->deltaPlanes[1] = &context->deltaPlanesBuffer[1ULL * context->maxPlaneSize]; context->deltaPlanes[2] = &context->deltaPlanesBuffer[2ULL * context->maxPlaneSize]; context->deltaPlanes[3] = &context->deltaPlanesBuffer[3ULL * context->maxPlaneSize]; } return TRUE; } BITMAP_PLANAR_CONTEXT* freerdp_bitmap_planar_context_new(DWORD flags, UINT32 maxWidth, UINT32 maxHeight) { BITMAP_PLANAR_CONTEXT* context = (BITMAP_PLANAR_CONTEXT*)winpr_aligned_calloc(1, sizeof(BITMAP_PLANAR_CONTEXT), 32); if (!context) return nullptr; if (flags & PLANAR_FORMAT_HEADER_NA) context->AllowSkipAlpha = TRUE; if (flags & PLANAR_FORMAT_HEADER_RLE) context->AllowRunLengthEncoding = TRUE; if (flags & PLANAR_FORMAT_HEADER_CS) context->AllowColorSubsampling = TRUE; context->ColorLossLevel = flags & PLANAR_FORMAT_HEADER_CLL_MASK; if (context->ColorLossLevel) context->AllowDynamicColorFidelity = TRUE; if (!freerdp_bitmap_planar_context_reset(context, maxWidth, maxHeight)) { WINPR_PRAGMA_DIAG_PUSH WINPR_PRAGMA_DIAG_IGNORED_MISMATCHED_DEALLOC freerdp_bitmap_planar_context_free(context); WINPR_PRAGMA_DIAG_POP return nullptr; } return context; } void freerdp_bitmap_planar_context_free(BITMAP_PLANAR_CONTEXT* context) { if (!context) return; winpr_aligned_free(context->pTempData); winpr_aligned_free(context->planesBuffer); winpr_aligned_free(context->deltaPlanesBuffer); winpr_aligned_free(context->rlePlanesBuffer); winpr_aligned_free(context); } void freerdp_planar_switch_bgr(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL bgr) { WINPR_ASSERT(planar); planar->bgr = bgr; } void freerdp_planar_topdown_image(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL topdown) { WINPR_ASSERT(planar); planar->topdown = topdown; }