Work around valgrind/MSan SIMD false positives

Referring to
https://sourceforge.net/p/libjpeg-turbo/bugs/48,
https://sourceforge.net/p/libjpeg-turbo/bugs/82,
 #15, #238, #253, and #619,
valgrind and MSan have failed to properly detect data initialization by
libjpeg-turbo's x86 SIMD extensions for the entire 14 years that
libjpeg-turbo has been a project, resulting in false positives unless
libjpeg-turbo is built with WITH_SIMD=0 or run with JSIMD_FORCENONE=1.
This commit introduces a new C preprocessor macro (ZERO_BUFFERS) that,
if set, causes libjpeg-turbo to zero certain buffers in order to work
around the specific valgrind/MSan test failures caused by the
aforementioned false positives.  This allows us to more closely
approximate the production configuration of libjpeg-turbo when testing
with valgrind or MSan.

Closes #781

.github/workflows/build.yml

@@ -167,6 +167,7 @@ jobs:
         uses: actions/checkout@v4
       - name: Set up build
         run: |
+          sudo apt install -y nasm
           sudo sysctl vm.mmap_rnd_bits=28
       - name: Build
         env:
@@ -174,7 +175,7 @@ jobs:
         run: |
           mkdir build
           pushd build
-          cmake -G"Unix Makefiles" -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_C_COMPILER=clang -DCMAKE_C_FLAGS_RELWITHDEBINFO="-O0 -g -fsanitize=memory -fsanitize-memory-param-retval -fno-sanitize-recover=all -fPIE" -DWITH_SIMD=0 ..
+          cmake -G"Unix Makefiles" -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_C_COMPILER=clang -DCMAKE_C_FLAGS_RELWITHDEBINFO="-O0 -g -fsanitize=memory -fsanitize-memory-param-retval -fno-sanitize-recover=all -fPIE -DZERO_BUFFERS=1" -DREQUIRE_SIMD=1 ..
           export NUMCPUS=`grep -c '^processor' /proc/cpuinfo`
           make -j$NUMCPUS --load-average=$NUMCPUS
           make test

fuzz/cjpeg.cc

@@ -57,13 +57,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
     (char *)"-targa", NULL
   };
   int fd = -1;
-#if defined(__has_feature) && __has_feature(memory_sanitizer)
-  char env[18] = "JSIMD_FORCENONE=1";
-
-  /* The libjpeg-turbo SIMD extensions produce false positives with
-     MemorySanitizer. */
-  putenv(env);
-#endif
 
   snprintf(filename, FILENAME_MAX, "/tmp/libjpeg-turbo_cjpeg_fuzz.XXXXXX");
   if ((fd = mkstemp(filename)) < 0 || write(fd, data, size) < 0)

fuzz/compress.cc

@@ -59,13 +59,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
     { TJPF_GRAY, TJSAMP_GRAY, 50 },
     { TJPF_CMYK, TJSAMP_440, 40 }
   };
-#if defined(__has_feature) && __has_feature(memory_sanitizer)
-  char env[18] = "JSIMD_FORCENONE=1";
-
-  /* The libjpeg-turbo SIMD extensions produce false positives with
-     MemorySanitizer. */
-  putenv(env);
-#endif
 
   snprintf(filename, FILENAME_MAX, "/tmp/libjpeg-turbo_compress_fuzz.XXXXXX");
   if ((fd = mkstemp(filename)) < 0 || write(fd, data, size) < 0)

fuzz/compress12.cc

@@ -60,13 +60,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
     { TJPF_GRAY, TJSAMP_GRAY, 50 },
     { TJPF_CMYK, TJSAMP_440, 40 }
   };
-#if defined(__has_feature) && __has_feature(memory_sanitizer)
-  char env[18] = "JSIMD_FORCENONE=1";
-
-  /* The libjpeg-turbo SIMD extensions produce false positives with
-     MemorySanitizer. */
-  putenv(env);
-#endif
 
   snprintf(filename, FILENAME_MAX, "/tmp/libjpeg-turbo_compress12_fuzz.XXXXXX");
   if ((fd = mkstemp(filename)) < 0 || write(fd, data, size) < 0)

fuzz/compress12_lossless.cc

@@ -59,13 +59,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
     { TJPF_GRAY, 6, 3 },
     { TJPF_CMYK, 7, 0 }
   };
-#if defined(__has_feature) && __has_feature(memory_sanitizer)
-  char env[18] = "JSIMD_FORCENONE=1";
-
-  /* The libjpeg-turbo SIMD extensions produce false positives with
-     MemorySanitizer. */
-  putenv(env);
-#endif
 
   snprintf(filename, FILENAME_MAX, "/tmp/libjpeg-turbo_compress_fuzz.XXXXXX");
   if ((fd = mkstemp(filename)) < 0 || write(fd, data, size) < 0)

fuzz/compress16_lossless.cc

@@ -59,13 +59,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
     { TJPF_GRAY, 6, 3 },
     { TJPF_CMYK, 7, 0 }
   };
-#if defined(__has_feature) && __has_feature(memory_sanitizer)
-  char env[18] = "JSIMD_FORCENONE=1";
-
-  /* The libjpeg-turbo SIMD extensions produce false positives with
-     MemorySanitizer. */
-  putenv(env);
-#endif
 
   snprintf(filename, FILENAME_MAX, "/tmp/libjpeg-turbo_compress_fuzz.XXXXXX");
   if ((fd = mkstemp(filename)) < 0 || write(fd, data, size) < 0)

fuzz/compress_lossless.cc

@@ -58,13 +58,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
     { TJPF_GRAY, 6, 3 },
     { TJPF_CMYK, 7, 0 }
   };
-#if defined(__has_feature) && __has_feature(memory_sanitizer)
-  char env[18] = "JSIMD_FORCENONE=1";
-
-  /* The libjpeg-turbo SIMD extensions produce false positives with
-     MemorySanitizer. */
-  putenv(env);
-#endif
 
   snprintf(filename, FILENAME_MAX, "/tmp/libjpeg-turbo_compress_fuzz.XXXXXX");
   if ((fd = mkstemp(filename)) < 0 || write(fd, data, size) < 0)

fuzz/compress_yuv.cc

@@ -58,13 +58,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
     { TJPF_BGR, TJSAMP_GRAY, 60 },
     { TJPF_GRAY, TJSAMP_GRAY, 50 }
   };
-#if defined(__has_feature) && __has_feature(memory_sanitizer)
-  char env[18] = "JSIMD_FORCENONE=1";
-
-  /* The libjpeg-turbo SIMD extensions produce false positives with
-     MemorySanitizer. */
-  putenv(env);
-#endif
 
   snprintf(filename, FILENAME_MAX, "/tmp/libjpeg-turbo_compress_yuv_fuzz.XXXXXX");
   if ((fd = mkstemp(filename)) < 0 || write(fd, data, size) < 0)

fuzz/decompress.cc

@@ -44,13 +44,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
      necessary to achieve full coverage. */
   enum TJPF pixelFormats[NUMPF] =
     { TJPF_RGB, TJPF_BGRX, TJPF_GRAY, TJPF_CMYK };
-#if defined(__has_feature) && __has_feature(memory_sanitizer)
-  char env[18] = "JSIMD_FORCENONE=1";
-
-  /* The libjpeg-turbo SIMD extensions produce false positives with
-     MemorySanitizer. */
-  putenv(env);
-#endif
 
   if ((handle = tj3Init(TJINIT_DECOMPRESS)) == NULL)
     goto bailout;

fuzz/decompress_yuv.cc

@@ -44,13 +44,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
      necessary to achieve full coverage. */
   enum TJPF pixelFormats[NUMPF] =
     { TJPF_BGR, TJPF_XRGB, TJPF_GRAY };
-#if defined(__has_feature) && __has_feature(memory_sanitizer)
-  char env[18] = "JSIMD_FORCENONE=1";
-
-  /* The libjpeg-turbo SIMD extensions produce false positives with
-     MemorySanitizer. */
-  putenv(env);
-#endif
 
   if ((handle = tj3Init(TJINIT_DECOMPRESS)) == NULL)
     goto bailout;

fuzz/transform.cc

@@ -39,13 +39,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
   size_t dstSizes[1] = { 0 }, maxBufSize;
   int width = 0, height = 0, jpegSubsamp, i;
   tjtransform transforms[1];
-#if defined(__has_feature) && __has_feature(memory_sanitizer)
-  char env[18] = "JSIMD_FORCENONE=1";
-
-  /* The libjpeg-turbo SIMD extensions produce false positives with
-     MemorySanitizer. */
-  putenv(env);
-#endif
 
   if ((handle = tj3Init(TJINIT_TRANSFORM)) == NULL)
     goto bailout;

jchuff.c

@@ -542,6 +542,10 @@ encode_one_block_simd(working_state *state, JCOEFPTR block, int last_dc_val,
   JOCTET _buffer[BUFSIZE], *buffer;
   int localbuf = 0;
 
+#ifdef ZERO_BUFFERS
+  memset(_buffer, 0, sizeof(_buffer));
+#endif
+
   LOAD_BUFFER()
 
   buffer = jsimd_huff_encode_one_block(state, buffer, block, last_dc_val,

jcphuff.c

@@ -650,6 +650,11 @@ encode_mcu_AC_first(j_compress_ptr cinfo, JBLOCKROW *MCU_data)
   size_t bits[8 / SIZEOF_SIZE_T];
   int max_coef_bits = cinfo->data_precision + 2;
 
+#ifdef ZERO_BUFFERS
+  memset(values_unaligned, 0, sizeof(values_unaligned));
+  memset(bits, 0, sizeof(bits));
+#endif
+
   entropy->next_output_byte = cinfo->dest->next_output_byte;
   entropy->free_in_buffer = cinfo->dest->free_in_buffer;
 
@@ -915,6 +920,11 @@ encode_mcu_AC_refine(j_compress_ptr cinfo, JBLOCKROW *MCU_data)
   size_t zerobits, signbits;
   size_t bits[16 / SIZEOF_SIZE_T];
 
+#ifdef ZERO_BUFFERS
+  memset(absvalues_unaligned, 0, sizeof(absvalues_unaligned));
+  memset(bits, 0, sizeof(bits));
+#endif
+
   entropy->next_output_byte = cinfo->dest->next_output_byte;
   entropy->free_in_buffer = cinfo->dest->free_in_buffer;
 

jdatadst-tj.c

@@ -19,6 +19,7 @@
  */
 
 /* this is not a core library module, so it doesn't define JPEG_INTERNALS */
+#define JPEG_INTERNALS
 #include "jinclude.h"
 #include "jpeglib.h"
 #include "jerror.h"
@@ -92,7 +93,7 @@ empty_mem_output_buffer(j_compress_ptr cinfo)
 
   /* Try to allocate new buffer with double size */
   nextsize = dest->bufsize * 2;
-  nextbuffer = (JOCTET *)malloc(nextsize);
+  nextbuffer = (JOCTET *)MALLOC(nextsize);
 
   if (nextbuffer == NULL)
     ERREXIT1(cinfo, JERR_OUT_OF_MEMORY, 10);
@@ -183,7 +184,7 @@ jpeg_mem_dest_tj(j_compress_ptr cinfo, unsigned char **outbuffer,
   if (*outbuffer == NULL || *outsize == 0) {
     if (alloc) {
       /* Allocate initial buffer */
-      dest->newbuffer = *outbuffer = (unsigned char *)malloc(OUTPUT_BUF_SIZE);
+      dest->newbuffer = *outbuffer = (unsigned char *)MALLOC(OUTPUT_BUF_SIZE);
       if (dest->newbuffer == NULL)
         ERREXIT1(cinfo, JERR_OUT_OF_MEMORY, 10);
       *outsize = OUTPUT_BUF_SIZE;

jmemnobs.c

@@ -4,7 +4,7 @@
  * This file was part of the Independent JPEG Group's software:
  * Copyright (C) 1992-1996, Thomas G. Lane.
  * libjpeg-turbo Modifications:
- * Copyright (C) 2017-2018, D. R. Commander.
+ * Copyright (C) 2017-2018, 2024, D. R. Commander.
  * For conditions of distribution and use, see the accompanying README.ijg
  * file.
  *
@@ -31,7 +31,7 @@
 GLOBAL(void *)
 jpeg_get_small(j_common_ptr cinfo, size_t sizeofobject)
 {
-  return (void *)malloc(sizeofobject);
+  return (void *)MALLOC(sizeofobject);
 }
 
 GLOBAL(void)
@@ -48,7 +48,7 @@ jpeg_free_small(j_common_ptr cinfo, void *object, size_t sizeofobject)
 GLOBAL(void *)
 jpeg_get_large(j_common_ptr cinfo, size_t sizeofobject)
 {
-  return (void *)malloc(sizeofobject);
+  return (void *)MALLOC(sizeofobject);
 }
 
 GLOBAL(void)

jpegint.h

@@ -446,6 +446,12 @@ struct jpeg_color_quantizer {
 #undef MIN
 #define MIN(a, b)       ((a) < (b) ? (a) : (b))
 
+#ifdef ZERO_BUFFERS
+#define MALLOC(size)  calloc(1, size)
+#else
+#define MALLOC(size)  malloc(size)
+#endif
+
 
 /* We assume that right shift corresponds to signed division by 2 with
  * rounding towards minus infinity.  This is correct for typical "arithmetic

turbojpeg.c

@@ -879,7 +879,7 @@ DLLEXPORT void tjFree(unsigned char *buf)
 /* TurboJPEG 3+ */
 DLLEXPORT void *tj3Alloc(size_t bytes)
 {
-  return malloc(bytes);
+  return MALLOC(bytes);
 }
 
 /* TurboJPEG 1.2+ */
@@ -1292,7 +1292,7 @@ DLLEXPORT int tj3EncodeYUVPlanes8(tjhandle handle, const unsigned char *srcBuf,
 
   for (i = 0; i < cinfo->num_components; i++) {
     compptr = &cinfo->comp_info[i];
-    _tmpbuf[i] = (JSAMPLE *)malloc(
+    _tmpbuf[i] = (JSAMPLE *)MALLOC(
       PAD((compptr->width_in_blocks * cinfo->max_h_samp_factor * DCTSIZE) /
           compptr->h_samp_factor, 32) *
       cinfo->max_v_samp_factor + 32);
@@ -1311,7 +1311,7 @@ DLLEXPORT int tj3EncodeYUVPlanes8(tjhandle handle, const unsigned char *srcBuf,
             compptr->h_samp_factor, 32) * row];
     }
     _tmpbuf2[i] =
-      (JSAMPLE *)malloc(PAD(compptr->width_in_blocks * DCTSIZE, 32) *
+      (JSAMPLE *)MALLOC(PAD(compptr->width_in_blocks * DCTSIZE, 32) *
                         compptr->v_samp_factor + 32);
     if (!_tmpbuf2[i])
       THROW("Memory allocation failure");
@@ -2399,7 +2399,7 @@ DLLEXPORT int tj3DecompressToYUVPlanes8(tjhandle handle,
     }
   }
   if (usetmpbuf) {
-    if ((_tmpbuf = (JSAMPLE *)malloc(sizeof(JSAMPLE) * tmpbufsize)) == NULL)
+    if ((_tmpbuf = (JSAMPLE *)MALLOC(sizeof(JSAMPLE) * tmpbufsize)) == NULL)
       THROW("Memory allocation failure");
     ptr = _tmpbuf;
     for (i = 0; i < dinfo->num_components; i++) {