TurboJPEG: Update JPEG buf ptrs on comp/xform err

When using the in-memory destination manager, it is necessary to
explicitly call the destination manager's term_destination() method if
an error occurs.  That method is called by jpeg_finish_compress() but
not by jpeg_abort_compress().

This fixes a potential double free() that could occur if tjCompress*()
or tjTransform() returned an error and the calling application tried to
clean up a JPEG buffer that was dynamically re-allocated by one of those
functions.

ChangeLog.md

@@ -45,6 +45,11 @@ a 16-bit binary PGM file into an RGB image buffer.
 generated when using the `tjLoadImage()` function to load a 16-bit binary PPM
 file into an extended RGB image buffer.
 
+9. Fixed an issue whereby, if a JPEG buffer was automatically re-allocated by
+one of the TurboJPEG compression or transform functions and an error
+subsequently occurred during compression or transformation, the JPEG buffer
+pointer passed by the application was not updated when the function returned.
+
 
 2.0.90 (2.1 beta1)
 ==================

fuzz/transform.cc

@@ -92,21 +92,37 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
   if (!dstBufs[2])
     goto bailout;
 
-  tjTransform(handle, data, size, NUMXFORMS, dstBufs, dstSizes, transforms,
-              TJFLAG_LIMITSCANS | TJFLAG_NOREALLOC);
-
   maxBufSize = tjBufSize(width, height, jpegSubsamp);
 
-  /* Touch all of the output pixels in order to catch uninitialized reads
+  if (tjTransform(handle, data, size, NUMXFORMS, dstBufs, dstSizes, transforms,
-     when using MemorySanitizer. */
+                  TJFLAG_LIMITSCANS | TJFLAG_NOREALLOC) == 0) {
-  for (t = 0; t < NUMXFORMS; t++) {
+    /* Touch all of the output pixels in order to catch uninitialized reads
+       when using MemorySanitizer. */
+    for (t = 0; t < NUMXFORMS; t++) {
+      int sum = 0;
+
+      for (i = 0; i < dstSizes[t]; i++)
+        sum += dstBufs[t][i];
+
+      /* Prevent the code above from being optimized out.  This test should
+         never be true, but the compiler doesn't know that. */
+      if (sum > 255 * maxBufSize)
+        goto bailout;
+    }
+  }
+
+  transforms[0].options &= ~TJXOPT_COPYNONE;
+  free(dstBufs[0]);
+  dstBufs[0] = NULL;
+  dstSizes[0] = 0;
+
+  if (tjTransform(handle, data, size, 1, dstBufs, dstSizes, transforms,
+                  TJFLAG_LIMITSCANS) == 0) {
     int sum = 0;
 
-    for (i = 0; i < dstSizes[t]; i++)
+    for (i = 0; i < dstSizes[0]; i++)
-      sum += dstBufs[t][i];
+      sum += dstBufs[0][i];
 
-    /* Prevent the code above from being optimized out.  This test should never
-       be true, but the compiler doesn't know that. */
     if (sum > 255 * maxBufSize)
       goto bailout;
   }

turbojpeg.c

@@ -720,7 +720,10 @@ DLLEXPORT int tjCompress2(tjhandle handle, const unsigned char *srcBuf,
   jpeg_finish_compress(cinfo);
 
 bailout:
-  if (cinfo->global_state > CSTATE_START) jpeg_abort_compress(cinfo);
+  if (cinfo->global_state > CSTATE_START) {
+    if (alloc) (*cinfo->dest->term_destination) (cinfo);
+    jpeg_abort_compress(cinfo);
+  }
   free(row_pointer);
   if (this->jerr.warning) retval = -1;
   this->jerr.stopOnWarning = FALSE;
@@ -1088,7 +1091,10 @@ DLLEXPORT int tjCompressFromYUVPlanes(tjhandle handle,
   jpeg_finish_compress(cinfo);
 
 bailout:
-  if (cinfo->global_state > CSTATE_START) jpeg_abort_compress(cinfo);
+  if (cinfo->global_state > CSTATE_START) {
+    if (alloc) (*cinfo->dest->term_destination) (cinfo);
+    jpeg_abort_compress(cinfo);
+  }
   for (i = 0; i < MAX_COMPONENTS; i++) {
     free(tmpbuf[i]);
     free(inbuf[i]);
@@ -1886,7 +1892,7 @@ DLLEXPORT int tjTransform(tjhandle handle, const unsigned char *jpegBuf,
 {
   jpeg_transform_info *xinfo = NULL;
   jvirt_barray_ptr *srccoefs, *dstcoefs;
-  int retval = 0, i, jpegSubsamp, saveMarkers = 0;
+  int retval = 0, alloc = 1, i, jpegSubsamp, saveMarkers = 0;
   struct my_progress_mgr progress;
 
   GET_INSTANCE(handle);
@@ -1974,7 +1980,7 @@ DLLEXPORT int tjTransform(tjhandle handle, const unsigned char *jpegBuf,
   srccoefs = jpeg_read_coefficients(dinfo);
 
   for (i = 0; i < n; i++) {
-    int w, h, alloc = 1;
+    int w, h;
 
     if (!xinfo[i].crop) {
       w = dinfo->image_width;  h = dinfo->image_height;
@@ -2032,7 +2038,10 @@ DLLEXPORT int tjTransform(tjhandle handle, const unsigned char *jpegBuf,
   jpeg_finish_decompress(dinfo);
 
 bailout:
-  if (cinfo->global_state > CSTATE_START) jpeg_abort_compress(cinfo);
+  if (cinfo->global_state > CSTATE_START) {
+    if (alloc) (*cinfo->dest->term_destination) (cinfo);
+    jpeg_abort_compress(cinfo);
+  }
   if (dinfo->global_state > DSTATE_START) jpeg_abort_decompress(dinfo);
   free(xinfo);
   if (this->jerr.warning) retval = -1;