ryan/mozjpeg
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TJBench: Improve consistency of results

Browse Source 
Given that libjpeg-turbo can often process hundreds of megapixels/second
on modern hardware, the default of one warmup iteration was essentially
meaningless.  Furthermore, the -warmup option was a bit clunky, since
it required some foreknowledge of how fast the benchmarks were going to
execute.

This commit introduces a 1-second warmup interval for each benchmark by
default, and the -warmup option has been retasked to control the length
of that interval.
This commit is contained in:
DRC
2017-06-27 14:22:39 -05:00
parent d0c3aa900b
commit 1db1ce45da
3 changed files with 66 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
ChangeLog.md
View File

@@ -59,6 +59,11 @@ more easily defend against one of the progressive JPEG exploits (LJT-01-004)
identified in
[this report](http://www.libjpeg-turbo.org/pmwiki/uploads/About/TwoIssueswiththeJPEGStandard.pdf).
10. TJBench will now run each benchmark for 1 second prior to starting the
timer, in order to improve the consistency of the results. Furthermore, the
`-warmup` option is now used to specify the amount of warmup time rather than
the number of warmup iterations.
1.5.1
=====

42
java/TJBench.java
View File

@@ -34,7 +34,7 @@ import org.libjpegturbo.turbojpeg.*;
class TJBench {
static int flags = 0, quiet = 0, pf = TJ.PF_BGR, yuvpad = 1, warmup = 1;
static int flags = 0, quiet = 0, pf = TJ.PF_BGR, yuvpad = 1;
static boolean compOnly, decompOnly, doTile, doYUV, write = true;
static final String[] pixFormatStr = {
@@ -55,7 +55,7 @@ class TJBench {
static TJScalingFactor sf;
static int xformOp = TJTransform.OP_NONE, xformOpt = 0;
static double benchTime = 5.0;
static double benchTime = 5.0, warmup = 1.0;
static final double getTime() {
@@ -162,7 +162,7 @@ class TJBench {
}
/* Benchmark */
iter -= warmup;
iter = -1;
elapsed = elapsedDecode = 0.0;
while (true) {
int tile = 0;
@@ -184,11 +184,14 @@ class TJBench {
tjd.decompress(dstBuf, x, y, width, pitch, height, pf, flags);
}
}
iter++;
if (iter >= 1) {
elapsed += getTime() - start;
elapsed += getTime() - start;
if (iter >= 0) {
iter++;
if (elapsed >= benchTime)
break;
} else if (elapsed >= warmup) {
iter = 0;
elapsed = elapsedDecode = 0.0;
}
}
if(doYUV)
@@ -321,7 +324,7 @@ class TJBench {
}
/* Benchmark */
iter = -warmup;
iter = -1;
elapsed = elapsedEncode = 0.0;
while (true) {
int tile = 0;
@@ -346,11 +349,14 @@ class TJBench {
totalJpegSize += jpegSize[tile];
}
}
iter++;
if (iter >= 1) {
elapsed += getTime() - start;
elapsed += getTime() - start;
if (iter >= 0) {
iter++;
if (elapsed >= benchTime)
break;
} else if (elapsed >= warmup) {
iter = 0;
elapsed = elapsedEncode = 0.0;
}
}
if (doYUV)
@@ -541,17 +547,20 @@ class TJBench {
}
}
iter = -warmup;
iter = -1;
elapsed = 0.;
while (true) {
start = getTime();
tjt.transform(jpegBuf, t, flags);
jpegSize = tjt.getTransformedSizes();
iter++;
if (iter >= 1) {
elapsed += getTime() - start;
elapsed += getTime() - start;
if (iter >= 0) {
iter++;
if (elapsed >= benchTime)
break;
} else if (elapsed >= warmup) {
iter = 0;
elapsed = 0.0;
}
}
t = null;
@@ -659,8 +668,9 @@ class TJBench {
System.out.println("-grayscale = Perform lossless grayscale conversion prior to decompression");
System.out.println(" test (can be combined with the other transforms above)");
System.out.println("-benchtime <t> = Run each benchmark for at least <t> seconds (default = 5.0)");
System.out.println("-warmup <w> = Execute each benchmark <w> times to prime the cache before");
System.out.println(" taking performance measurements (default = 1)");
System.out.println("-warmup <t> = Run each benchmark for <t> seconds (default = 1.0) prior to");
System.out.println(" starting the timer, in order to prime the caches and thus improve the");
System.out.println(" consistency of the results.");
System.out.println("-componly = Stop after running compression tests. Do not test decompression.");
System.out.println("-nowrite = Do not write reference or output images (improves consistency");
System.out.println(" of performance measurements.)\n");

57
tjbench.c
View File

@@ -46,7 +46,7 @@
#define _throwbmp(m) _throw(m, bmpgeterr())
int flags=TJFLAG_NOREALLOC, componly=0, decomponly=0, doyuv=0, quiet=0,
dotile=0, pf=TJPF_BGR, yuvpad=1, warmup=1, dowrite=1;
dotile=0, pf=TJPF_BGR, yuvpad=1, dowrite=1;
char *ext="ppm";
const char *pixFormatStr[TJ_NUMPF]=
{
@@ -64,7 +64,7 @@ const char *subName[TJ_NUMSAMP]={"444", "422", "420", "GRAY", "440", "411"};
tjscalingfactor *scalingfactors=NULL, sf={1, 1}; int nsf=0;
int xformop=TJXOP_NONE, xformopt=0;
int (*customFilter)(short *, tjregion, tjregion, int, int, tjtransform *);
double benchtime=5.0;
double benchtime=5.0, warmup=1.0;
char *formatName(int subsamp, int cs, char *buf)
@@ -146,7 +146,7 @@ int decomp(unsigned char *srcbuf, unsigned char **jpegbuf,
}
/* Benchmark */
iter=-warmup;
iter=-1;
elapsed=elapsedDecode=0.;
while(1)
{
@@ -176,12 +176,17 @@ int decomp(unsigned char *srcbuf, unsigned char **jpegbuf,
_throwtj("executing tjDecompress2()");
}
}
iter++;
if(iter>=1)
elapsed+=gettime()-start;
if(iter>=0)
{
elapsed+=gettime()-start;
iter++;
if(elapsed>=benchtime) break;
}
else if(elapsed>=warmup)
{
iter=0;
elapsed=elapsedDecode=0.;
}
}
if(doyuv) elapsed-=elapsedDecode;
@@ -340,7 +345,7 @@ int fullTest(unsigned char *srcbuf, int w, int h, int subsamp, int jpegqual,
}
/* Benchmark */
iter=-warmup;
iter=-1;
elapsed=elapsedEncode=0.;
while(1)
{
@@ -374,12 +379,17 @@ int fullTest(unsigned char *srcbuf, int w, int h, int subsamp, int jpegqual,
totaljpegsize+=jpegsize[tile];
}
}
iter++;
if(iter>=1)
elapsed+=gettime()-start;
if(iter>=0)
{
elapsed+=gettime()-start;
iter++;
if(elapsed>=benchtime) break;
}
else if(elapsed>=warmup)
{
iter=0;
elapsed=elapsedEncode=0.;
}
}
if(doyuv) elapsed-=elapsedEncode;
@@ -623,7 +633,7 @@ int decompTest(char *filename)
}
}
iter=-warmup;
iter=-1;
elapsed=0.;
while(1)
{
@@ -631,12 +641,17 @@ int decompTest(char *filename)
if(tjTransform(handle, srcbuf, srcsize, _ntilesw*_ntilesh, jpegbuf,
jpegsize, t, flags)==-1)
_throwtj("executing tjTransform()");
iter++;
if(iter>=1)
elapsed+=gettime()-start;
if(iter>=0)
{
elapsed+=gettime()-start;
iter++;
if(elapsed>=benchtime) break;
}
else if(elapsed>=warmup)
{
iter=0;
elapsed=0.;
}
}
free(t); t=NULL;
@@ -765,8 +780,9 @@ void usage(char *progname)
printf("-grayscale = Perform lossless grayscale conversion prior to decompression\n");
printf(" test (can be combined with the other transforms above)\n");
printf("-benchtime <t> = Run each benchmark for at least <t> seconds (default = 5.0)\n");
printf("-warmup <w> = Execute each benchmark <w> times to prime the cache before\n");
printf(" taking performance measurements (default = 1)\n");
printf("-warmup <t> = Run each benchmark for <t> seconds (default = 1.0) prior to\n");
printf(" starting the timer, in order to prime the caches and thus improve the\n");
printf(" consistency of the results.\n");
printf("-componly = Stop after running compression tests. Do not test decompression.\n");
printf("-nowrite = Do not write reference or output images (improves consistency of\n");
printf(" performance measurements.)\n\n");
@@ -880,13 +896,10 @@ int main(int argc, char *argv[])
}
if(!strcasecmp(argv[i], "-warmup") && i<argc-1)
{
int temp=atoi(argv[++i]);
if(temp>=0)
{
warmup=temp;
printf("Warmup runs = %d\n\n", warmup);
}
double temp=atof(argv[++i]);
if(temp>=0.0) warmup=temp;
else usage(argv[0]);
printf("Warmup time = %.1f seconds\n\n", warmup);
}
if(!strcmp(argv[i], "-?")) usage(argv[0]);
if(!strcasecmp(argv[i], "-alloc")) flags&=(~TJFLAG_NOREALLOC);