ryan/mozjpeg
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TJBench: Improve consistency of results

Browse Source 
Given that libjpeg-turbo can often process hundreds of megapixels/second
on modern hardware, the default of one warmup iteration was essentially
meaningless.  Furthermore, the -warmup option was a bit clunky, since
it required some foreknowledge of how fast the benchmarks were going to
execute.

This commit introduces a 1-second warmup interval for each benchmark by
default, and the -warmup option has been retasked to control the length
of that interval.
This commit is contained in:
DRC
2017-06-27 14:22:39 -05:00
parent d0c3aa900b
commit 1db1ce45da
3 changed files with 66 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
ChangeLog.md
View File

@@ -59,6 +59,11 @@ more easily defend against one of the progressive JPEG exploits (LJT-01-004)
identified in identified in
[this report](http://www.libjpeg-turbo.org/pmwiki/uploads/About/TwoIssueswiththeJPEGStandard.pdf). [this report](http://www.libjpeg-turbo.org/pmwiki/uploads/About/TwoIssueswiththeJPEGStandard.pdf).
10. TJBench will now run each benchmark for 1 second prior to starting the
timer, in order to improve the consistency of the results. Furthermore, the
`-warmup` option is now used to specify the amount of warmup time rather than
the number of warmup iterations.
1.5.1 1.5.1
===== =====

42
java/TJBench.java
View File

@@ -34,7 +34,7 @@ import org.libjpegturbo.turbojpeg.*;
class TJBench { class TJBench {
static int flags = 0, quiet = 0, pf = TJ.PF_BGR, yuvpad = 1, warmup = 1; static int flags = 0, quiet = 0, pf = TJ.PF_BGR, yuvpad = 1;
static boolean compOnly, decompOnly, doTile, doYUV, write = true; static boolean compOnly, decompOnly, doTile, doYUV, write = true;
static final String[] pixFormatStr = { static final String[] pixFormatStr = {
@@ -55,7 +55,7 @@ class TJBench {
static TJScalingFactor sf; static TJScalingFactor sf;
static int xformOp = TJTransform.OP_NONE, xformOpt = 0; static int xformOp = TJTransform.OP_NONE, xformOpt = 0;
static double benchTime = 5.0; static double benchTime = 5.0, warmup = 1.0;
static final double getTime() { static final double getTime() {
@@ -162,7 +162,7 @@ class TJBench {
} }
/* Benchmark */ /* Benchmark */
iter -= warmup; iter = -1;
elapsed = elapsedDecode = 0.0; elapsed = elapsedDecode = 0.0;
while (true) { while (true) {
int tile = 0; int tile = 0;
@@ -184,11 +184,14 @@ class TJBench {
tjd.decompress(dstBuf, x, y, width, pitch, height, pf, flags); tjd.decompress(dstBuf, x, y, width, pitch, height, pf, flags);
} }
} }
iter++; elapsed += getTime() - start;
if (iter >= 1) { if (iter >= 0) {
elapsed += getTime() - start; iter++;
if (elapsed >= benchTime) if (elapsed >= benchTime)
break; break;
} else if (elapsed >= warmup) {
iter = 0;
elapsed = elapsedDecode = 0.0;
} }
} }
if(doYUV) if(doYUV)
@@ -321,7 +324,7 @@ class TJBench {
} }
/* Benchmark */ /* Benchmark */
iter = -warmup; iter = -1;
elapsed = elapsedEncode = 0.0; elapsed = elapsedEncode = 0.0;
while (true) { while (true) {
int tile = 0; int tile = 0;
@@ -346,11 +349,14 @@ class TJBench {
totalJpegSize += jpegSize[tile]; totalJpegSize += jpegSize[tile];
} }
} }
iter++; elapsed += getTime() - start;
if (iter >= 1) { if (iter >= 0) {
elapsed += getTime() - start; iter++;
if (elapsed >= benchTime) if (elapsed >= benchTime)
break; break;
} else if (elapsed >= warmup) {
iter = 0;
elapsed = elapsedEncode = 0.0;
} }
} }
if (doYUV) if (doYUV)
@@ -541,17 +547,20 @@ class TJBench {
} }
} }
iter = -warmup; iter = -1;
elapsed = 0.; elapsed = 0.;
while (true) { while (true) {
start = getTime(); start = getTime();
tjt.transform(jpegBuf, t, flags); tjt.transform(jpegBuf, t, flags);
jpegSize = tjt.getTransformedSizes(); jpegSize = tjt.getTransformedSizes();
iter++; elapsed += getTime() - start;
if (iter >= 1) { if (iter >= 0) {
elapsed += getTime() - start; iter++;
if (elapsed >= benchTime) if (elapsed >= benchTime)
break; break;
} else if (elapsed >= warmup) {
iter = 0;
elapsed = 0.0;
} }
} }
t = null; t = null;
@@ -659,8 +668,9 @@ class TJBench {
System.out.println("-grayscale = Perform lossless grayscale conversion prior to decompression"); System.out.println("-grayscale = Perform lossless grayscale conversion prior to decompression");
System.out.println(" test (can be combined with the other transforms above)"); System.out.println(" test (can be combined with the other transforms above)");
System.out.println("-benchtime <t> = Run each benchmark for at least <t> seconds (default = 5.0)"); System.out.println("-benchtime <t> = Run each benchmark for at least <t> seconds (default = 5.0)");
System.out.println("-warmup <w> = Execute each benchmark <w> times to prime the cache before"); System.out.println("-warmup <t> = Run each benchmark for <t> seconds (default = 1.0) prior to");
System.out.println(" taking performance measurements (default = 1)"); System.out.println(" starting the timer, in order to prime the caches and thus improve the");
System.out.println(" consistency of the results.");
System.out.println("-componly = Stop after running compression tests. Do not test decompression."); System.out.println("-componly = Stop after running compression tests. Do not test decompression.");
System.out.println("-nowrite = Do not write reference or output images (improves consistency"); System.out.println("-nowrite = Do not write reference or output images (improves consistency");
System.out.println(" of performance measurements.)\n"); System.out.println(" of performance measurements.)\n");

57
tjbench.c
View File

@@ -46,7 +46,7 @@
#define _throwbmp(m) _throw(m, bmpgeterr()) #define _throwbmp(m) _throw(m, bmpgeterr())
int flags=TJFLAG_NOREALLOC, componly=0, decomponly=0, doyuv=0, quiet=0, int flags=TJFLAG_NOREALLOC, componly=0, decomponly=0, doyuv=0, quiet=0,
dotile=0, pf=TJPF_BGR, yuvpad=1, warmup=1, dowrite=1; dotile=0, pf=TJPF_BGR, yuvpad=1, dowrite=1;
char *ext="ppm"; char *ext="ppm";
const char *pixFormatStr[TJ_NUMPF]= const char *pixFormatStr[TJ_NUMPF]=
{ {
@@ -64,7 +64,7 @@ const char *subName[TJ_NUMSAMP]={"444", "422", "420", "GRAY", "440", "411"};
tjscalingfactor *scalingfactors=NULL, sf={1, 1}; int nsf=0; tjscalingfactor *scalingfactors=NULL, sf={1, 1}; int nsf=0;
int xformop=TJXOP_NONE, xformopt=0; int xformop=TJXOP_NONE, xformopt=0;
int (*customFilter)(short *, tjregion, tjregion, int, int, tjtransform *); int (*customFilter)(short *, tjregion, tjregion, int, int, tjtransform *);
double benchtime=5.0; double benchtime=5.0, warmup=1.0;
char *formatName(int subsamp, int cs, char *buf) char *formatName(int subsamp, int cs, char *buf)
@@ -146,7 +146,7 @@ int decomp(unsigned char *srcbuf, unsigned char **jpegbuf,
} }
/* Benchmark */ /* Benchmark */
iter=-warmup; iter=-1;
elapsed=elapsedDecode=0.; elapsed=elapsedDecode=0.;
while(1) while(1)
{ {
@@ -176,12 +176,17 @@ int decomp(unsigned char *srcbuf, unsigned char **jpegbuf,
_throwtj("executing tjDecompress2()"); _throwtj("executing tjDecompress2()");
} }
} }
iter++; elapsed+=gettime()-start;
if(iter>=1) if(iter>=0)
{ {
elapsed+=gettime()-start; iter++;
if(elapsed>=benchtime) break; if(elapsed>=benchtime) break;
} }
else if(elapsed>=warmup)
{
iter=0;
elapsed=elapsedDecode=0.;
}
} }
if(doyuv) elapsed-=elapsedDecode; if(doyuv) elapsed-=elapsedDecode;
@@ -340,7 +345,7 @@ int fullTest(unsigned char *srcbuf, int w, int h, int subsamp, int jpegqual,
} }
/* Benchmark */ /* Benchmark */
iter=-warmup; iter=-1;
elapsed=elapsedEncode=0.; elapsed=elapsedEncode=0.;
while(1) while(1)
{ {
@@ -374,12 +379,17 @@ int fullTest(unsigned char *srcbuf, int w, int h, int subsamp, int jpegqual,
totaljpegsize+=jpegsize[tile]; totaljpegsize+=jpegsize[tile];
} }
} }
iter++; elapsed+=gettime()-start;
if(iter>=1) if(iter>=0)
{ {
elapsed+=gettime()-start; iter++;
if(elapsed>=benchtime) break; if(elapsed>=benchtime) break;
} }
else if(elapsed>=warmup)
{
iter=0;
elapsed=elapsedEncode=0.;
}
} }
if(doyuv) elapsed-=elapsedEncode; if(doyuv) elapsed-=elapsedEncode;
@@ -623,7 +633,7 @@ int decompTest(char *filename)
} }
} }
iter=-warmup; iter=-1;
elapsed=0.; elapsed=0.;
while(1) while(1)
{ {
@@ -631,12 +641,17 @@ int decompTest(char *filename)
if(tjTransform(handle, srcbuf, srcsize, _ntilesw*_ntilesh, jpegbuf, if(tjTransform(handle, srcbuf, srcsize, _ntilesw*_ntilesh, jpegbuf,
jpegsize, t, flags)==-1) jpegsize, t, flags)==-1)
_throwtj("executing tjTransform()"); _throwtj("executing tjTransform()");
iter++; elapsed+=gettime()-start;
if(iter>=1) if(iter>=0)
{ {
elapsed+=gettime()-start; iter++;
if(elapsed>=benchtime) break; if(elapsed>=benchtime) break;
} }
else if(elapsed>=warmup)
{
iter=0;
elapsed=0.;
}
} }
free(t); t=NULL; free(t); t=NULL;
@@ -765,8 +780,9 @@ void usage(char *progname)
printf("-grayscale = Perform lossless grayscale conversion prior to decompression\n"); printf("-grayscale = Perform lossless grayscale conversion prior to decompression\n");
printf(" test (can be combined with the other transforms above)\n"); printf(" test (can be combined with the other transforms above)\n");
printf("-benchtime <t> = Run each benchmark for at least <t> seconds (default = 5.0)\n"); printf("-benchtime <t> = Run each benchmark for at least <t> seconds (default = 5.0)\n");
printf("-warmup <w> = Execute each benchmark <w> times to prime the cache before\n"); printf("-warmup <t> = Run each benchmark for <t> seconds (default = 1.0) prior to\n");
printf(" taking performance measurements (default = 1)\n"); printf(" starting the timer, in order to prime the caches and thus improve the\n");
printf(" consistency of the results.\n");
printf("-componly = Stop after running compression tests. Do not test decompression.\n"); printf("-componly = Stop after running compression tests. Do not test decompression.\n");
printf("-nowrite = Do not write reference or output images (improves consistency of\n"); printf("-nowrite = Do not write reference or output images (improves consistency of\n");
printf(" performance measurements.)\n\n"); printf(" performance measurements.)\n\n");
@@ -880,13 +896,10 @@ int main(int argc, char *argv[])
} }
if(!strcasecmp(argv[i], "-warmup") && i<argc-1) if(!strcasecmp(argv[i], "-warmup") && i<argc-1)
{ {
int temp=atoi(argv[++i]); double temp=atof(argv[++i]);
if(temp>=0) if(temp>=0.0) warmup=temp;
{
warmup=temp;
printf("Warmup runs = %d\n\n", warmup);
}
else usage(argv[0]); else usage(argv[0]);
printf("Warmup time = %.1f seconds\n\n", warmup);
} }
if(!strcmp(argv[i], "-?")) usage(argv[0]); if(!strcmp(argv[i], "-?")) usage(argv[0]);
if(!strcasecmp(argv[i], "-alloc")) flags&=(~TJFLAG_NOREALLOC); if(!strcasecmp(argv[i], "-alloc")) flags&=(~TJFLAG_NOREALLOC);