OSS-Fuzz: Check img size b4 readers allocate mem

After the completion of the start_input() method, it's too late to check
the image size, because the image readers may have already tried to
allocate memory for the image.  If the width and height are excessively
large, then attempting to allocate memory for the image could slow
performance or lead to out-of-memory errors prior to the fuzz target
checking the image size.

NOTE: Specifically, the aforementioned OOM errors and slow units were
observed with the compression fuzz targets when using MSan.

cdjpeg.h

@@ -5,7 +5,7 @@
  * Copyright (C) 1994-1997, Thomas G. Lane.
  * Modified 2019 by Guido Vollbeding.
  * libjpeg-turbo Modifications:
- * Copyright (C) 2017, 2019, D. R. Commander.
+ * Copyright (C) 2017, 2019, 2021, D. R. Commander.
  * For conditions of distribution and use, see the accompanying README.ijg
  * file.
  *
@@ -36,6 +36,9 @@ struct cjpeg_source_struct {
 
   JSAMPARRAY buffer;
   JDIMENSION buffer_height;
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+  JDIMENSION max_pixels;
+#endif
 };
 
 

cjpeg.c

@@ -690,6 +690,9 @@ main(int argc, char **argv)
   /* Figure out the input file format, and set up to read it. */
   src_mgr = select_file_type(&cinfo, input_file);
   src_mgr->input_file = input_file;
+#ifdef CJPEG_FUZZER
+  src_mgr->max_pixels = 1048576;
+#endif
 
   /* Read the input file header to obtain file size & colorspace. */
   (*src_mgr->start_input) (&cinfo, src_mgr);
@@ -709,9 +712,7 @@ main(int argc, char **argv)
     jpeg_stdio_dest(&cinfo, output_file);
 
 #ifdef CJPEG_FUZZER
-  if (cinfo.image_width < 1 || cinfo.image_height < 1 ||
-      (unsigned long long)cinfo.image_width * cinfo.image_height > 1048576 ||
-      setjmp(myerr.setjmp_buffer))
+  if (setjmp(myerr.setjmp_buffer))
     HANDLE_ERROR()
 #endif
 

rdbmp.c

@@ -522,6 +522,11 @@ start_input_bmp(j_compress_ptr cinfo, cjpeg_source_ptr sinfo)
 
   if (biWidth <= 0 || biHeight <= 0)
     ERREXIT(cinfo, JERR_BMP_EMPTY);
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+  if (sinfo->max_pixels &&
+      (unsigned long long)biWidth * biHeight > sinfo->max_pixels)
+    ERREXIT(cinfo, JERR_WIDTH_OVERFLOW);
+#endif
   if (biPlanes != 1)
     ERREXIT(cinfo, JERR_BMP_BADPLANES);
 
@@ -672,6 +677,9 @@ jinit_read_bmp(j_compress_ptr cinfo, boolean use_inversion_array)
   /* Fill in method ptrs, except get_pixel_rows which start_input sets */
   source->pub.start_input = start_input_bmp;
   source->pub.finish_input = finish_input_bmp;
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+  source->pub.max_pixels = 0;
+#endif
 
   source->use_inversion_array = use_inversion_array;
 

rdgif.c

@@ -408,6 +408,11 @@ start_input_gif(j_compress_ptr cinfo, cjpeg_source_ptr sinfo)
   height = LM_to_uint(hdrbuf, 2);
   if (width == 0 || height == 0)
     ERREXIT(cinfo, JERR_GIF_EMPTY);
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+  if (sinfo->max_pixels &&
+      (unsigned long long)width * height > sinfo->max_pixels)
+    ERREXIT(cinfo, JERR_WIDTH_OVERFLOW);
+#endif
   /* we ignore the color resolution, sort flag, and background color index */
   aspectRatio = UCH(hdrbuf[6]);
   if (aspectRatio != 0 && aspectRatio != 49)
@@ -452,6 +457,11 @@ start_input_gif(j_compress_ptr cinfo, cjpeg_source_ptr sinfo)
     height = LM_to_uint(hdrbuf, 6);
     if (width == 0 || height == 0)
       ERREXIT(cinfo, JERR_GIF_EMPTY);
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    if (sinfo->max_pixels &&
+        (unsigned long long)width * height > sinfo->max_pixels)
+      ERREXIT(cinfo, JERR_WIDTH_OVERFLOW);
+#endif
     source->is_interlaced = (BitSet(hdrbuf[8], INTERLACE) != 0);
 
     /* Read local colormap if header indicates it is present */
@@ -675,6 +685,9 @@ jinit_read_gif(j_compress_ptr cinfo)
   /* Fill in method ptrs, except get_pixel_rows which start_input sets */
   source->pub.start_input = start_input_gif;
   source->pub.finish_input = finish_input_gif;
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+  source->pub.max_pixels = 0;
+#endif
 
   return (cjpeg_source_ptr)source;
 }

rdppm.c

@@ -586,6 +586,10 @@ start_input_ppm(j_compress_ptr cinfo, cjpeg_source_ptr sinfo)
 
   if (w <= 0 || h <= 0 || maxval <= 0) /* error check */
     ERREXIT(cinfo, JERR_PPM_NOT);
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+  if (sinfo->max_pixels && (unsigned long long)w * h > sinfo->max_pixels)
+    ERREXIT(cinfo, JERR_WIDTH_OVERFLOW);
+#endif
 
   cinfo->data_precision = BITS_IN_JSAMPLE; /* we always rescale data to this */
   cinfo->image_width = (JDIMENSION)w;
@@ -765,6 +769,9 @@ jinit_read_ppm(j_compress_ptr cinfo)
   /* Fill in method ptrs, except get_pixel_rows which start_input sets */
   source->pub.start_input = start_input_ppm;
   source->pub.finish_input = finish_input_ppm;
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+  source->pub.max_pixels = 0;
+#endif
 
   return (cjpeg_source_ptr)source;
 }

rdtarga.c

@@ -5,7 +5,7 @@
  * Copyright (C) 1991-1996, Thomas G. Lane.
  * Modified 2017 by Guido Vollbeding.
  * libjpeg-turbo Modifications:
- * Copyright (C) 2018, D. R. Commander.
+ * Copyright (C) 2018, 2021, D. R. Commander.
  * For conditions of distribution and use, see the accompanying README.ijg
  * file.
  *
@@ -363,6 +363,11 @@ start_input_tga(j_compress_ptr cinfo, cjpeg_source_ptr sinfo)
       interlace_type != 0 ||      /* currently don't allow interlaced image */
       width == 0 || height == 0)  /* image width/height must be non-zero */
     ERREXIT(cinfo, JERR_TGA_BADPARMS);
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+  if (sinfo->max_pixels &&
+      (unsigned long long)width * height > sinfo->max_pixels)
+    ERREXIT(cinfo, JERR_WIDTH_OVERFLOW);
+#endif
 
   if (subtype > 8) {
     /* It's an RLE-coded file */
@@ -493,6 +498,9 @@ jinit_read_targa(j_compress_ptr cinfo)
   /* Fill in method ptrs, except get_pixel_rows which start_input sets */
   source->pub.start_input = start_input_tga;
   source->pub.finish_input = finish_input_tga;
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+  source->pub.max_pixels = 0;
+#endif
 
   return (cjpeg_source_ptr)source;
 }

turbojpeg.c

@@ -2092,22 +2092,17 @@ DLLEXPORT unsigned char *tjLoadImage(const char *filename, int *width,
     THROWG("tjLoadImage(): Unsupported file type");
 
   src->input_file = file;
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+  /* Ignore images larger than 1 Megapixel when fuzzing. */
+  if (flags & TJFLAG_FUZZING)
+    src->max_pixels = 1048576;
+#endif
   (*src->start_input) (cinfo, src);
   (*cinfo->mem->realize_virt_arrays) ((j_common_ptr)cinfo);
 
   *width = cinfo->image_width;  *height = cinfo->image_height;
   *pixelFormat = cs2pf[cinfo->in_color_space];
 
-#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
-  /* Ignore 0-pixel images and images larger than 1 Megapixel when fuzzing.
-     Casting *width to (unsigned long long) prevents integer overflow if
-     (*width) * (*height) > INT_MAX. */
-  if (flags & TJFLAG_FUZZING &&
-      (*width < 1 || *height < 1 ||
-       (unsigned long long)(*width) * (*height) > 1048576))
-    THROWG("tjLoadImage(): Uncompressed image is too large");
-#endif
-
   pitch = PAD((*width) * tjPixelSize[*pixelFormat], align);
   if ((unsigned long long)pitch * (unsigned long long)(*height) >
       (unsigned long long)((size_t)-1) ||