diff --git a/.travis.yml b/.travis.yml index 69c0e4e9..73001cf3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,7 +49,8 @@ matrix: before_install: - if [ "$TRAVIS_OS_NAME" = "osx" ]; then - brew install nasm homebrew/versions/gcc5 md5sha1sum Caskroom/versions/java6 gpg && + brew install nasm homebrew/versions/gcc5 md5sha1sum Caskroom/versions/java6 && + ln -fs /usr/local/bin/gpg1 /usr/local/bin/gpg && git clone --depth=1 https://github.com/libjpeg-turbo/gas-preprocessor.git ~/src/gas-preprocessor && ln -fs /Applications/Xcode.app /Applications/Xcode72.app; fi diff --git a/BUILDING.md b/BUILDING.md index 24d3bb91..6894a162 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -539,7 +539,7 @@ needs. export LDFLAGS=-pie TOOLCHAIN=${NDK_PATH}/toolchains/${HOST}-${TOOLCHAIN_VERSION}/prebuilt/${BUILD_PLATFORM} - cat <android.cmake + cat <toolchain.cmake set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_PROCESSOR arm) set(CMAKE_C_COMPILER ${TOOLCHAIN}/bin/${HOST}-gcc) @@ -570,7 +570,7 @@ needs. # It should not be necessary to modify the rest HOST=aarch64-linux-android SYSROOT=${NDK_PATH}/platforms/android-${ANDROID_VERSION}/arch-arm64 - export CFLAGS="--sysroot ${SYSROOT}" + export CFLAGS="--sysroot=${SYSROOT}" export LDFLAGS=-pie TOOLCHAIN=${NDK_PATH}/toolchains/${HOST}-${TOOLCHAIN_VERSION}/prebuilt/${BUILD_PLATFORM} @@ -587,6 +587,77 @@ needs. [additional CMake flags] {source_directory} make + +### x86 (32-bit) + +The following is a general recipe script that can be modified for your specific +needs. + + # Set these variables to suit your needs + NDK_PATH={full path to the "ndk" directory-- for example, /opt/android/sdk/ndk-bundle} + BUILD_PLATFORM={the platform name for the NDK package you installed-- + for example, "windows-x86" or "linux-x86_64" or "darwin-x86_64"} + TOOLCHAIN_VERSION={"4.8", "4.9", "clang3.5", etc. This corresponds to a + toolchain directory under ${NDK_PATH}/toolchains/.} + ANDROID_VERSION={The minimum version of Android to support-- for example, + "16", "19", etc.} + + # It should not be necessary to modify the rest + HOST=i686-linux-android + SYSROOT=${NDK_PATH}/platforms/android-${ANDROID_VERSION}/arch-x86 + export CFLAGS="--sysroot=${SYSROOT}" + export LDFLAGS=-pie + TOOLCHAIN=${NDK_PATH}/toolchains/x86-${TOOLCHAIN_VERSION}/prebuilt/${BUILD_PLATFORM} + + cat <toolchain.cmake + set(CMAKE_SYSTEM_NAME Linux) + set(CMAKE_SYSTEM_PROCESSOR i386) + set(CMAKE_C_COMPILER ${TOOLCHAIN}/bin/${HOST}-gcc) + set(CMAKE_FIND_ROOT_PATH ${TOOLCHAIN}/${HOST}) + EOF + + cd {build_directory} + cmake -G"Unix Makefiles" -DCMAKE_TOOLCHAIN_FILE=toolchain.cmake \ + -DCMAKE_POSITION_INDEPENDENT_CODE=1 \ + [additional CMake flags] {source_directory} + make + + +### x86-64 (64-bit) + +The following is a general recipe script that can be modified for your specific +needs. + + # Set these variables to suit your needs + NDK_PATH={full path to the "ndk" directory-- for example, /opt/android/sdk/ndk-bundle} + BUILD_PLATFORM={the platform name for the NDK package you installed-- + for example, "windows-x86" or "linux-x86_64" or "darwin-x86_64"} + TOOLCHAIN_VERSION={"4.8", "4.9", "clang3.5", etc. This corresponds to a + toolchain directory under ${NDK_PATH}/toolchains/.} + ANDROID_VERSION={The minimum version of Android to support. "21" or later + is required for a 64-bit build.} + + # It should not be necessary to modify the rest + HOST=x86_64-linux-android + SYSROOT=${NDK_PATH}/platforms/android-${ANDROID_VERSION}/arch-x86_64 + export CFLAGS="--sysroot=${SYSROOT}" + export LDFLAGS=-pie + TOOLCHAIN=${NDK_PATH}/toolchains/x86_64-${TOOLCHAIN_VERSION}/prebuilt/${BUILD_PLATFORM} + + cat <toolchain.cmake + set(CMAKE_SYSTEM_NAME Linux) + set(CMAKE_SYSTEM_PROCESSOR x86_64) + set(CMAKE_C_COMPILER ${TOOLCHAIN}/bin/${HOST}-gcc) + set(CMAKE_FIND_ROOT_PATH ${TOOLCHAIN}/${HOST}) + EOF + + cd {build_directory} + cmake -G"Unix Makefiles" -DCMAKE_TOOLCHAIN_FILE=toolchain.cmake \ + -DCMAKE_POSITION_INDEPENDENT_CODE=1 \ + [additional CMake flags] {source_directory} + make + + If building for Android 4.0.x (API level < 16) or earlier, remove `-DCMAKE_POSITION_INDEPENDENT_CODE=1` from the CMake arguments and `-pie` from `LDFLAGS`. diff --git a/ChangeLog.md b/ChangeLog.md index 7e0494f9..ef745739 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -81,6 +81,27 @@ API/ABI emulation, since the behavior is entirely internal. Note that `-copy all` must be passed to jpegtran in order to transfer the EXIF tags from the source image to the destination image. +8. Fixed several memory leaks in the TurboJPEG API library that could occur +if the library was built with certain compilers and optimization levels +(known to occur with GCC 4.x and clang with `-O1` and higher but not with +GCC 5.x or 6.x) and one of the underlying libjpeg API functions threw an error +after a TurboJPEG API function allocated a local buffer. + +9. The libjpeg-turbo memory manager will now honor the `max_memory_to_use` +structure member in jpeg\_memory\_mgr, which can be set to the maximum amount +of memory (in bytes) that libjpeg-turbo should use during decompression or +multi-pass (including progressive) compression. This limit can also be set +using the `JPEGMEM` environment variable or using the `-maxmemory` switch in +cjpeg/djpeg/jpegtran (refer to the respective man pages for more details.) +This has been a documented feature of libjpeg since v5, but the +`malloc()`/`free()` implementation of the memory manager (jmemnobs.c) never +implemented the feature. Restricting libjpeg-turbo's memory usage is useful +for two reasons: it allows testers to more easily work around the 2 GB limit +in libFuzzer, and it allows developers of security-sensitive applications to +more easily defend against one of the progressive JPEG exploits (LJT-01-004) +identified in +[this report](http://www.libjpeg-turbo.org/pmwiki/uploads/About/TwoIssueswiththeJPEGStandard.pdf). + 1.5.1 ===== diff --git a/appveyor.yml b/appveyor.yml index 95dde03e..0acc3292 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -24,7 +24,9 @@ install: build_script: - cmd: >- - bash c:/buildscripts/buildljt -r file://%CD:\=/% -b /c/ljt.nightly %APPVEYOR_REPO_BRANCH% -v + for /f %%i in ('"cygpath %CD%"') do set MINGWPATH=%%i + + bash c:/buildscripts/buildljt -r file://%MINGWPATH% -b /c/ljt.nightly %APPVEYOR_REPO_BRANCH% -v move c:\ljt.nightly\files\*.tar.gz . diff --git a/cjpeg.1 b/cjpeg.1 index 36805ea2..557b0b0a 100644 --- a/cjpeg.1 +++ b/cjpeg.1 @@ -1,4 +1,4 @@ -.TH CJPEG 1 "19 January 2017" +.TH CJPEG 1 "18 March 2017" .SH NAME cjpeg \- compress an image file to a JPEG file .SH SYNOPSIS @@ -205,7 +205,7 @@ Set limit for amount of memory to use in processing large images. Value is in thousands of bytes, or millions of bytes if "M" is attached to the number. For example, .B \-max 4m -selects 4000000 bytes. If more space is needed, temporary files will be used. +selects 4000000 bytes. If more space is needed, an error will occur. .TP .BI \-outfile " name" Send output image to the named file, not to standard output. diff --git a/djpeg.1 b/djpeg.1 index ed392866..1b13b7e3 100644 --- a/djpeg.1 +++ b/djpeg.1 @@ -1,4 +1,4 @@ -.TH DJPEG 1 "19 January 2017" +TH DJPEG 1 "18 March 2017" .SH NAME djpeg \- decompress a JPEG file to an image file .SH SYNOPSIS @@ -188,7 +188,7 @@ Set limit for amount of memory to use in processing large images. Value is in thousands of bytes, or millions of bytes if "M" is attached to the number. For example, .B \-max 4m -selects 4000000 bytes. If more space is needed, temporary files will be used. +selects 4000000 bytes. If more space is needed, an error will occur. .TP .BI \-outfile " name" Send output image to the named file, not to standard output. diff --git a/java/TJBench.java b/java/TJBench.java index 23fab612..618cc977 100644 --- a/java/TJBench.java +++ b/java/TJBench.java @@ -1,5 +1,5 @@ /* - * Copyright (C)2009-2014, 2016 D. R. Commander. All Rights Reserved. + * Copyright (C)2009-2014, 2016-2017 D. R. Commander. All Rights Reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -582,8 +582,8 @@ class TJBench { System.out.print("N/A N/A "); jpegBuf = new byte[1][TJ.bufSize(_tilew, _tileh, subsamp)]; jpegSize = new int[1]; + jpegBuf[0] = srcBuf; jpegSize[0] = srcSize; - System.arraycopy(srcBuf, 0, jpegBuf[0], 0, srcSize); } if (w == tilew) diff --git a/jmemnobs.c b/jmemnobs.c index 5797198d..ac12afa5 100644 --- a/jmemnobs.c +++ b/jmemnobs.c @@ -3,8 +3,8 @@ * * This file was part of the Independent JPEG Group's software: * Copyright (C) 1992-1996, Thomas G. Lane. - * It was modified by The libjpeg-turbo Project to include only code and - * information relevant to libjpeg-turbo. + * libjpeg-turbo Modifications: + * Copyright (C) 2017, D. R. Commander. * For conditions of distribution and use, see the accompanying README.ijg * file. * @@ -15,7 +15,6 @@ * This is very portable in the sense that it'll compile on almost anything, * but you'd better have lots of main memory (or virtual memory) if you want * to process big images. - * Note that the max_memory_to_use option is ignored by this implementation. */ #define JPEG_INTERNALS @@ -66,14 +65,21 @@ jpeg_free_large (j_common_ptr cinfo, void *object, size_t sizeofobject) /* * This routine computes the total memory space available for allocation. - * Here we always say, "we got all you want bud!" */ GLOBAL(size_t) jpeg_mem_available (j_common_ptr cinfo, size_t min_bytes_needed, size_t max_bytes_needed, size_t already_allocated) { - return max_bytes_needed; + if (cinfo->mem->max_memory_to_use) { + if (cinfo->mem->max_memory_to_use > already_allocated) + return cinfo->mem->max_memory_to_use - already_allocated; + else + return 0; + } else { + /* Here we always say, "we got all you want bud!" */ + return max_bytes_needed; + } } diff --git a/jpegtran.1 b/jpegtran.1 index 1357d5f4..2efb2647 100644 --- a/jpegtran.1 +++ b/jpegtran.1 @@ -1,4 +1,4 @@ -.TH JPEGTRAN 1 "19 January 2017" +.TH JPEGTRAN 1 "18 March 2017" .SH NAME jpegtran \- lossless transformation of JPEG files .SH SYNOPSIS @@ -227,7 +227,7 @@ Set limit for amount of memory to use in processing large images. Value is in thousands of bytes, or millions of bytes if "M" is attached to the number. For example, .B \-max 4m -selects 4000000 bytes. If more space is needed, temporary files will be used. +selects 4000000 bytes. If more space is needed, an error will occur. .TP .BI \-outfile " name" Send output image to the named file, not to standard output. diff --git a/libjpeg.txt b/libjpeg.txt index a35faf4a..13094d49 100644 --- a/libjpeg.txt +++ b/libjpeg.txt @@ -2981,13 +2981,6 @@ Some operating modes (eg, two-pass color quantization) require full-image buffers. Such buffers are treated as "virtual arrays": only the current strip need be in memory, and the rest can be swapped out to a temporary file. -If you use the simplest memory manager back end (jmemnobs.c), then no -temporary files are used; virtual arrays are simply malloc()'d. Images bigger -than memory can be processed only if your system supports virtual memory. -The other memory manager back ends support temporary files of various flavors -and thus work in machines without virtual memory. They may also be useful on -Unix machines if you need to process images that exceed available swap space. - When using temporary files, the library will make the in-memory buffers for its virtual arrays just big enough to stay within a "maximum memory" setting. Your application can set this limit by setting cinfo->mem->max_memory_to_use @@ -3000,6 +2993,11 @@ that space allocated with alloc_small() is ignored, on the assumption that it's too small to be worth worrying about; so a reasonable safety margin should be left when setting max_memory_to_use. +NOTE: Unless you develop your own memory manager back end, then temporary files +will never be used. The back end provided in libjpeg-turbo (jmemnobs.c) simply +malloc()s and free()s virtual arrays, and an error occurs if the required +memory exceeds the limit specified in cinfo->mem->max_memory_to_use. + Memory usage ------------ diff --git a/simd/arm/jsimd.c b/simd/arm/jsimd.c index 77c54dcd..19633492 100644 --- a/simd/arm/jsimd.c +++ b/simd/arm/jsimd.c @@ -2,6 +2,7 @@ * jsimd_arm.c * * Copyright 2009 Pierre Ossman for Cendio AB + * Copyright (C) 2011, Nokia Corporation and/or its subsidiary(-ies). * Copyright (C) 2009-2011, 2013-2014, 2016, D. R. Commander. * Copyright (C) 2015-2016, Matthieu Darbois. * diff --git a/simd/arm64/jsimd.c b/simd/arm64/jsimd.c index 43ed87fa..12a78226 100644 --- a/simd/arm64/jsimd.c +++ b/simd/arm64/jsimd.c @@ -2,6 +2,7 @@ * jsimd_arm64.c * * Copyright 2009 Pierre Ossman for Cendio AB + * Copyright (C) 2011, Nokia Corporation and/or its subsidiary(-ies). * Copyright (C) 2009-2011, 2013-2014, 2016, D. R. Commander. * Copyright (C) 2015-2016, Matthieu Darbois. * diff --git a/structure.txt b/structure.txt index 296d1250..f69c9d87 100644 --- a/structure.txt +++ b/structure.txt @@ -832,21 +832,19 @@ read_backing_store, manipulate a backing-store object write_backing_store, close_backing_store -On some systems there will be more than one type of backing-store object -(specifically, in MS-DOS a backing store file might be an area of extended -memory as well as a disk file). jpeg_open_backing_store is responsible for -choosing how to implement a given object. The read/write/close routines -are method pointers in the structure that describes a given object; this -lets them be different for different object types. +On some systems there will be more than one type of backing-store object. +jpeg_open_backing_store is responsible for choosing how to implement a given +object. The read/write/close routines are method pointers in the structure +that describes a given object; this lets them be different for different object +types. It may be necessary to ensure that backing store objects are explicitly -released upon abnormal program termination. For example, MS-DOS won't free -extended memory by itself. To support this, we will expect the main program -or surrounding application to arrange to call self_destruct (typically via -jpeg_destroy) upon abnormal termination. This may require a SIGINT signal -handler or equivalent. We don't want to have the back end module install its -own signal handler, because that would pre-empt the surrounding application's -ability to control signal handling. +released upon abnormal program termination. To support this, we will expect +the main program or surrounding application to arrange to call self_destruct +(typically via jpeg_destroy) upon abnormal termination. This may require a +SIGINT signal handler or equivalent. We don't want to have the back end module +install its own signal handler, because that would pre-empt the surrounding +application's ability to control signal handling. The IJG distribution includes several memory manager back end implementations. Usually the same back end should be suitable for all applications on a given diff --git a/tjbench.c b/tjbench.c index 9db1968c..12b4efe5 100644 --- a/tjbench.c +++ b/tjbench.c @@ -1,5 +1,5 @@ /* - * Copyright (C)2009-2016 D. R. Commander. All Rights Reserved. + * Copyright (C)2009-2017 D. R. Commander. All Rights Reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -40,7 +40,7 @@ #define _throw(op, err) { \ printf("ERROR in line %d while %s:\n%s\n", __LINE__, op, err); \ - retval=-1; goto bailout;} + retval=-1; goto bailout;} #define _throwunix(m) _throw(m, strerror(errno)) #define _throwtj(m) _throw(m, tjGetErrorStr()) #define _throwbmp(m) _throw(m, bmpgeterr()) @@ -207,7 +207,7 @@ int decomp(unsigned char *srcbuf, unsigned char **jpegbuf, (double)(w*h)/1000000.*(double)iter/elapsed); if(doyuv) { - printf("YUV Decode --> Frame rate:  %f fps\n", + printf("YUV Decode --> Frame rate: %f fps\n", (double)iter/elapsedDecode); printf(" Throughput: %f Megapixels/sec\n", (double)(w*h)/1000000.*(double)iter/elapsedDecode); @@ -668,7 +668,9 @@ int decompTest(char *filename) { if(quiet==1) printf("N/A N/A "); jpegsize[0]=srcsize; - memcpy(jpegbuf[0], srcbuf, srcsize); + free(jpegbuf[0]); + jpegbuf[0]=srcbuf; + srcbuf=NULL; } if(w==tilew) _tilew=_w; diff --git a/turbojpeg.c b/turbojpeg.c index 5fa6a7af..f3c99224 100644 --- a/turbojpeg.c +++ b/turbojpeg.c @@ -1,5 +1,5 @@ /* - * Copyright (C)2009-2016 D. R. Commander. All Rights Reserved. + * Copyright (C)2009-2017 D. R. Commander. All Rights Reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -772,13 +772,6 @@ DLLEXPORT int DLLCALL tjCompress2(tjhandle handle, const unsigned char *srcBuf, || jpegSubsamp<0 || jpegSubsamp>=NUMSUBOPT || jpegQual<0 || jpegQual>100) _throw("tjCompress2(): Invalid argument"); - if(setjmp(this->jerr.setjmp_buffer)) - { - /* If we get here, the JPEG code has signaled an error. */ - retval=-1; - goto bailout; - } - if(pitch==0) pitch=width*tjPixelSize[pixelFormat]; #ifndef JCS_EXTENSIONS @@ -791,6 +784,16 @@ DLLEXPORT int DLLCALL tjCompress2(tjhandle handle, const unsigned char *srcBuf, } #endif + if((row_pointer=(JSAMPROW *)malloc(sizeof(JSAMPROW)*height))==NULL) + _throw("tjCompress2(): Memory allocation failure"); + + if(setjmp(this->jerr.setjmp_buffer)) + { + /* If we get here, the JPEG code has signaled an error. */ + retval=-1; + goto bailout; + } + cinfo->image_width=width; cinfo->image_height=height; @@ -807,8 +810,6 @@ DLLEXPORT int DLLCALL tjCompress2(tjhandle handle, const unsigned char *srcBuf, return -1; jpeg_start_compress(cinfo, TRUE); - if((row_pointer=(JSAMPROW *)malloc(sizeof(JSAMPROW)*height))==NULL) - _throw("tjCompress2(): Memory allocation failure"); for(i=0; ijerr.setjmp_buffer)) - { - /* If we get here, the JPEG code has signaled an error. */ - retval=-1; - goto bailout; - } - if(pixelFormat==TJPF_CMYK) _throw("tjEncodeYUVPlanes(): Cannot generate YUV images from CMYK pixels"); @@ -910,6 +904,13 @@ DLLEXPORT int DLLCALL tjEncodeYUVPlanes(tjhandle handle, } #endif + if(setjmp(this->jerr.setjmp_buffer)) + { + /* If we get here, the JPEG code has signaled an error. */ + retval=-1; + goto bailout; + } + cinfo->image_width=width; cinfo->image_height=height; @@ -986,6 +987,13 @@ DLLEXPORT int DLLCALL tjEncodeYUVPlanes(tjhandle handle, } } + if(setjmp(this->jerr.setjmp_buffer)) + { + /* If we get here, the JPEG code has signaled an error. */ + retval=-1; + goto bailout; + } + for(row=0; rowmax_v_samp_factor) { (*cinfo->cconvert->color_convert)(cinfo, &row_pointer[row], tmpbuf, 0, @@ -1160,6 +1168,13 @@ DLLEXPORT int DLLCALL tjCompressFromYUVPlanes(tjhandle handle, } } + if(setjmp(this->jerr.setjmp_buffer)) + { + /* If we get here, the JPEG code has signaled an error. */ + retval=-1; + goto bailout; + } + for(row=0; row<(int)cinfo->image_height; row+=cinfo->max_v_samp_factor*DCTSIZE) { @@ -1438,6 +1453,12 @@ DLLEXPORT int DLLCALL tjDecompress2(tjhandle handle, if((row_pointer=(JSAMPROW *)malloc(sizeof(JSAMPROW) *dinfo->output_height))==NULL) _throw("tjDecompress2(): Memory allocation failure"); + if(setjmp(this->jerr.setjmp_buffer)) + { + /* If we get here, the JPEG code has signaled an error. */ + retval=-1; + goto bailout; + } for(i=0; i<(int)dinfo->output_height; i++) { if(flags&TJFLAG_BOTTOMUP) @@ -1660,6 +1681,13 @@ DLLEXPORT int DLLCALL tjDecodeYUVPlanes(tjhandle handle, } } + if(setjmp(this->jerr.setjmp_buffer)) + { + /* If we get here, the JPEG code has signaled an error. */ + retval=-1; + goto bailout; + } + for(row=0; rowmax_v_samp_factor) { JDIMENSION inrow=0, outrow=0; @@ -1840,6 +1868,13 @@ DLLEXPORT int DLLCALL tjDecompressToYUVPlanes(tjhandle handle, } } + if(setjmp(this->jerr.setjmp_buffer)) + { + /* If we get here, the JPEG code has signaled an error. */ + retval=-1; + goto bailout; + } + if(flags&TJFLAG_FASTUPSAMPLE) dinfo->do_fancy_upsampling=FALSE; if(flags&TJFLAG_FASTDCT) dinfo->dct_method=JDCT_FASTEST; dinfo->raw_data_out=TRUE; @@ -2017,6 +2052,11 @@ DLLEXPORT int DLLCALL tjTransform(tjhandle handle, else if(flags&TJFLAG_FORCESSE) putenv("JSIMD_FORCESSE=1"); else if(flags&TJFLAG_FORCESSE2) putenv("JSIMD_FORCESSE2=1"); + if((xinfo=(jpeg_transform_info *)malloc(sizeof(jpeg_transform_info)*n)) + ==NULL) + _throw("tjTransform(): Memory allocation failure"); + MEMZERO(xinfo, sizeof(jpeg_transform_info)*n); + if(setjmp(this->jerr.setjmp_buffer)) { /* If we get here, the JPEG code has signaled an error. */ @@ -2026,11 +2066,6 @@ DLLEXPORT int DLLCALL tjTransform(tjhandle handle, jpeg_mem_src_tj(dinfo, jpegBuf, jpegSize); - if((xinfo=(jpeg_transform_info *)malloc(sizeof(jpeg_transform_info)*n)) - ==NULL) - _throw("tjTransform(): Memory allocation failure"); - MEMZERO(xinfo, sizeof(jpeg_transform_info)*n); - for(i=0; i