tjDecompressToYUV2: Use scaled dims for plane calc

The documented behavior of the function is to use decompression scaling
to generate the largest possible image that will fit within the desired
image dimensions.  Thus, if the desired image dimensions are larger than
the scaled image dimensions, then tjDecompressToYUV2() should use the
scaled image dimensions when computing the plane pointers and strides to
pass to tjDecompressToYUVPlanes().

Note that this bug was not previously detected, because tjunittest and
tjbench always passed the scaled image dimensions to
tjDecompressToYUV2().

ChangeLog.md

@@ -34,6 +34,13 @@ Fixed a similar issue in `tjCompressFromYUV()` whereby it generated a corrupt
 JPEG image in certain cases, rather than throwing an error, if the `align`
 parameter was not a power of 2.
 
+6. Fixed an issue whereby `tjDecompressToYUV2()`, which is a wrapper for
+`tjDecompressToYUVPlanes()`, used the desired YUV image dimensions rather than
+the actual scaled image dimensions when computing the plane pointers and
+strides to pass to `tjDecompressToYUVPlanes()`.  This caused a buffer overrun
+and subsequent segfault if the desired image dimensions exceeded the scaled
+image dimensions.
+
 
 2.1.4
 =====

tjunittest.c

@@ -456,8 +456,12 @@ static void _decompTest(tjhandle handle, unsigned char *jpegBuf,
     if (sf.num != 1 || sf.denom != 1)
       printf("%d/%d ... ", sf.num, sf.denom);
     else printf("... ");
-    TRY_TJ(tjDecompressToYUV2(handle, jpegBuf, jpegSize, yuvBuf, scaledWidth,
-                              yuvAlign, scaledHeight, flags));
+    /* We pass scaledWidth + 1 and scaledHeight + 1 to validate that
+       tjDecompressToYUV2() generates the largest possible scaled image that
+       fits within the desired dimensions, as documented. */
+    TRY_TJ(tjDecompressToYUV2(handle, jpegBuf, jpegSize, yuvBuf,
+                              scaledWidth + 1, yuvAlign, scaledHeight + 1,
+                              flags));
     if (checkBufYUV(yuvBuf, scaledWidth, scaledHeight, subsamp, sf))
       printf("Passed.\n");
     else printf("FAILED!\n");
@@ -473,8 +477,11 @@ static void _decompTest(tjhandle handle, unsigned char *jpegBuf,
     if (sf.num != 1 || sf.denom != 1)
       printf("%d/%d ... ", sf.num, sf.denom);
     else printf("... ");
-    TRY_TJ(tjDecompress2(handle, jpegBuf, jpegSize, dstBuf, scaledWidth, 0,
-                         scaledHeight, pf, flags));
+    /* We pass scaledWidth + 1 and scaledHeight + 1 to validate that
+       tjDecompress2() generates the largest possible scaled image that fits
+       within the desired dimensions, as documented. */
+    TRY_TJ(tjDecompress2(handle, jpegBuf, jpegSize, dstBuf, scaledWidth + 1, 0,
+                         scaledHeight + 1, pf, flags));
   }
 
   if (checkBuf(dstBuf, scaledWidth, scaledHeight, pf, subsamp, sf, flags))

turbojpeg.c

@@ -1879,6 +1879,8 @@ DLLEXPORT int tjDecompressToYUV2(tjhandle handle, const unsigned char *jpegBuf,
   if (i >= NUMSF)
     THROW("tjDecompressToYUV2(): Could not scale down to desired image dimensions");
 
+  width = scaledw;  height = scaledh;
+
   pw0 = tjPlaneWidth(0, width, jpegSubsamp);
   ph0 = tjPlaneHeight(0, height, jpegSubsamp);
   dstPlanes[0] = dstBuf;