From b5eb30229e9b9c4a09917e7f563317c380031a22 Mon Sep 17 00:00:00 2001 From: DRC Date: Thu, 11 Jul 2019 15:30:04 -0500 Subject: [PATCH 1/2] TurboJPEG: Properly handle gigapixel images Prevent several integer overflow issues and subsequent segfaults that occurred when attempting to compress or decompress gigapixel images with the TurboJPEG API: - Modify tjBufSize(), tjBufSizeYUV2(), and tjPlaneSizeYUV() to avoid integer overflow when computing the return values and to return an error if such an overflow is unavoidable. - Modify tjunittest to validate the above. - Modify tjCompress2(), tjEncodeYUVPlanes(), tjDecompress2(), and tjDecodeYUVPlanes() to avoid integer overflow when computing the row pointers in the 64-bit TurboJPEG C API. - Modify TJBench (both C and Java versions) to avoid overflowing the size argument to malloc()/new and to fail gracefully if such an overflow is unavoidable. In general, this allows gigapixel images to be accommodated by the 64-bit TurboJPEG C API when using automatic JPEG buffer (re)allocation. Such images cannot currently be accommodated without automatic JPEG buffer (re)allocation, due to the fact that tjAlloc() accepts a 32-bit integer argument (oops.) Such images cannot be accommodated in the TurboJPEG Java API due to the fact that Java always uses a signed 32-bit integer as an array index. Fixes #361 (cherry picked from commit 2a9e3bd7430cfda1bc812d139e0609c6aca0b884) --- java/TJBench.java | 11 ++++++++++- tjbench.c | 44 +++++++++++++++++++++++++++++------------ tjunittest.c | 38 +++++++++++++++++++++++++++++++++++ turbojpeg.c | 50 ++++++++++++++++++++++++++++------------------- 4 files changed, 109 insertions(+), 34 deletions(-) diff --git a/java/TJBench.java b/java/TJBench.java index ddc414c..9b1ff81 100644 --- a/java/TJBench.java +++ b/java/TJBench.java @@ -96,6 +96,8 @@ class TJBench { int rindex = TJ.getRedOffset(pixelFormat); int gindex = TJ.getGreenOffset(pixelFormat); int bindex = TJ.getBlueOffset(pixelFormat); + if ((long)w[0] * (long)h[0] * (long)ps > (long)Integer.MAX_VALUE) + throw new Exception("Image is too large"); byte[] dstBuf = new byte[w[0] * h[0] * ps]; int pixels = w[0] * h[0], dstPtr = 0, rgbPtr = 0; while (pixels-- > 0) { @@ -147,8 +149,11 @@ class TJBench { tjd = new TJDecompressor(); - if (dstBuf == null) + if (dstBuf == null) { + if ((long)pitch * (long)scaledh > (long)Integer.MAX_VALUE) + throw new Exception("Image is too large"); dstBuf = new byte[pitch * scaledh]; + } /* Set the destination buffer to gray so we know whether the decompressor attempted to write to it */ @@ -287,6 +292,8 @@ class TJBench { String pfStr = pixFormatStr[pf]; YUVImage yuvImage = null; + if ((long)pitch * (long)h > (long)Integer.MAX_VALUE) + throw new Exception("Image is too large"); tmpBuf = new byte[pitch * h]; if (quiet == 0) @@ -435,6 +442,8 @@ class TJBench { int ps = TJ.getPixelSize(pf), tile; FileInputStream fis = new FileInputStream(fileName); + if (fis.getChannel().size() > (long)Integer.MAX_VALUE) + throw new Exception("Image is too large"); int srcSize = (int)fis.getChannel().size(); srcBuf = new byte[srcSize]; fis.read(srcBuf, 0, srcSize); diff --git a/tjbench.c b/tjbench.c index 0187b75..e90ba3d 100644 --- a/tjbench.c +++ b/tjbench.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include "./bmp.h" #include "./tjutil.h" @@ -127,7 +128,10 @@ int decomp(unsigned char *srcbuf, unsigned char **jpegbuf, if(dstbuf==NULL) { - if((dstbuf=(unsigned char *)malloc(pitch*scaledh))==NULL) + if ((unsigned long long)pitch * (unsigned long long)scaledh > + (unsigned long long)((size_t)-1)) + _throw("allocating destination buffer", "Image is too large"); + if ((dstbuf = (unsigned char *)malloc((size_t)pitch * scaledh)) == NULL) _throwunix("allocating destination buffer"); dstbufalloc=1; } @@ -139,7 +143,10 @@ int decomp(unsigned char *srcbuf, unsigned char **jpegbuf, { int width=dotile? tilew:scaledw; int height=dotile? tileh:scaledh; - int yuvsize=tjBufSizeYUV2(width, yuvpad, height, subsamp); + unsigned long yuvsize=tjBufSizeYUV2(width, yuvpad, height, subsamp); + + if (yuvsize == (unsigned long)-1) + _throwtj("allocating YUV buffer"); if((yuvbuf=(unsigned char *)malloc(yuvsize))==NULL) _throwunix("allocating YUV buffer"); memset(yuvbuf, 127, yuvsize); @@ -242,14 +249,14 @@ int decomp(unsigned char *srcbuf, unsigned char **jpegbuf, if(!quiet) printf("Compression error written to %s.\n", tempstr); if(subsamp==TJ_GRAYSCALE) { - int index, index2; + unsigned long index, index2; for(row=0, index=0; row + (unsigned long long)((size_t)-1)) + _throw("allocating temporary image buffer", "Image is too large"); + if((tmpbuf = (unsigned char *)malloc((size_t)pitch * h)) == NULL) _throwunix("allocating temporary image buffer"); if(!quiet) @@ -322,6 +332,8 @@ int fullTest(unsigned char *srcbuf, int w, int h, int subsamp, int jpegqual, if((flags&TJFLAG_NOREALLOC)!=0) for(i=0; i (unsigned long)INT_MAX) + _throw("getting buffer size", "Image is too large"); if((jpegbuf[i]=(unsigned char *)tjAlloc(tjBufSize(tilew, tileh, subsamp)))==NULL) _throwunix("allocating JPEG tiles"); @@ -339,6 +351,8 @@ int fullTest(unsigned char *srcbuf, int w, int h, int subsamp, int jpegqual, if(doyuv) { yuvsize=tjBufSizeYUV2(tilew, yuvpad, tileh, subsamp); + if(yuvsize == (unsigned long)-1) + _throwtj("allocating YUV buffer"); if((yuvbuf=(unsigned char *)malloc(yuvsize))==NULL) _throwunix("allocating YUV buffer"); memset(yuvbuf, 127, yuvsize); @@ -418,7 +432,7 @@ int fullTest(unsigned char *srcbuf, int w, int h, int subsamp, int jpegqual, { printf("Encode YUV --> Frame rate: %f fps\n", (double)iter/elapsedEncode); - printf(" Output image size: %d bytes\n", yuvsize); + printf(" Output image size: %lu bytes\n", yuvsize); printf(" Compression ratio: %f:1\n", (double)(w*h*ps)/(double)yuvsize); printf(" Throughput: %f Megapixels/sec\n", @@ -561,9 +575,12 @@ int decompTest(char *filename) _throwunix("allocating JPEG size array"); memset(jpegsize, 0, sizeof(unsigned long)*ntilesw*ntilesh); - if((flags&TJFLAG_NOREALLOC)!=0 || !dotile) + if ((flags & TJFLAG_NOREALLOC) != 0 && + (dotile || xformop != TJXOP_NONE || xformopt != 0 || customFilter)) for(i=0; i (unsigned long)INT_MAX) + _throw("getting buffer size", "Image is too large"); if((jpegbuf[i]=(unsigned char *)tjAlloc(tjBufSize(tilew, tileh, subsamp)))==NULL) _throwunix("allocating JPEG tiles"); @@ -684,7 +701,7 @@ int decompTest(char *filename) else { if(quiet==1) printf("N/A N/A "); - tjFree(jpegbuf[0]); + if(jpegbuf[0]) tjFree(jpegbuf[0]); jpegbuf[0]=NULL; decompsrc=1; } @@ -701,7 +718,8 @@ int decompTest(char *filename) for(i=0; i #define random() rand() #endif +#include "config.h" /* for SIZEOF_SIZE_T */ void usage(char *progName) @@ -593,6 +594,42 @@ void doTest(int w, int h, const int *formats, int nformats, int subsamp, } +#if SIZEOF_SIZE_T == 8 +#define CHECKSIZE(function) { \ + if ((unsigned long long)size < (unsigned long long)0xFFFFFFFF) \ + _throw(#function " overflow"); \ +} +#else +#define CHECKSIZE(function) { \ + if (size != (unsigned long)(-1) || \ + !strcmp(tjGetErrorStr(), "No error")) \ + _throw(#function " overflow"); \ +} +#endif + +static void overflowTest(void) +{ + /* Ensure that the various buffer size functions don't overflow */ + unsigned long size; + + size = tjBufSize(26755, 26755, TJSAMP_444); + CHECKSIZE(tjBufSize()); + size = TJBUFSIZE(26755, 26755); + CHECKSIZE(TJBUFSIZE()); + size = tjBufSizeYUV2(37838, 1, 37838, TJSAMP_444); + CHECKSIZE(tjBufSizeYUV2()); + size = TJBUFSIZEYUV(37838, 37838, TJSAMP_444); + CHECKSIZE(TJBUFSIZEYUV()); + size = tjBufSizeYUV(37838, 37838, TJSAMP_444); + CHECKSIZE(tjBufSizeYUV()); + size = tjPlaneSizeYUV(0, 65536, 0, 65536, TJSAMP_444); + CHECKSIZE(tjPlaneSizeYUV()); + +bailout: + return; +} + + void bufSizeTest(void) { int w, h, i, subsamp; @@ -704,6 +741,7 @@ int main(int argc, char *argv[]) } if(alloc) printf("Testing automatic buffer allocation\n"); if(doyuv) num4bf=4; + overflowTest(); doTest(35, 39, _3byteFormats, 2, TJSAMP_444, "test"); doTest(39, 41, _4byteFormats, num4bf, TJSAMP_444, "test"); doTest(41, 35, _3byteFormats, 2, TJSAMP_422, "test"); diff --git a/turbojpeg.c b/turbojpeg.c index 330a004..be03482 100644 --- a/turbojpeg.c +++ b/turbojpeg.c @@ -644,7 +644,8 @@ DLLEXPORT tjhandle DLLCALL tjInitCompress(void) DLLEXPORT unsigned long DLLCALL tjBufSize(int width, int height, int jpegSubsamp) { - unsigned long retval=0; int mcuw, mcuh, chromasf; + unsigned long long retval=0; + int mcuw, mcuh, chromasf; if(width<1 || height<1 || jpegSubsamp<0 || jpegSubsamp>=NUMSUBOPT) _throw("tjBufSize(): Invalid argument"); @@ -654,32 +655,37 @@ DLLEXPORT unsigned long DLLCALL tjBufSize(int width, int height, mcuw=tjMCUWidth[jpegSubsamp]; mcuh=tjMCUHeight[jpegSubsamp]; chromasf=jpegSubsamp==TJSAMP_GRAY? 0: 4*64/(mcuw*mcuh); - retval=PAD(width, mcuw) * PAD(height, mcuh) * (2 + chromasf) + 2048; + retval = PAD(width, mcuw) * PAD(height, mcuh) * (2ULL + chromasf) + 2048ULL; + if (retval > (unsigned long long)((unsigned long)-1)) + _throw("tjBufSize(): Image is too large"); bailout: - return retval; + return (unsigned long)retval; } DLLEXPORT unsigned long DLLCALL TJBUFSIZE(int width, int height) { - unsigned long retval=0; + unsigned long long retval = 0; if(width<1 || height<1) _throw("TJBUFSIZE(): Invalid argument"); /* This allows for rare corner cases in which a JPEG image can actually be larger than the uncompressed input (we wouldn't mention it if it hadn't happened before.) */ - retval=PAD(width, 16) * PAD(height, 16) * 6 + 2048; + retval = PAD(width, 16) * PAD(height, 16) * 6ULL + 2048ULL; + if (retval > (unsigned long long)((unsigned long)-1)) + _throw("TJBUFSIZE(): Image is too large"); bailout: - return retval; + return (unsigned long)retval; } DLLEXPORT unsigned long DLLCALL tjBufSizeYUV2(int width, int pad, int height, int subsamp) { - int retval=0, nc, i; + unsigned long long retval=0; + int nc, i; if(subsamp<0 || subsamp>=NUMSUBOPT) _throw("tjBufSizeYUV2(): Invalid argument"); @@ -691,11 +697,13 @@ DLLEXPORT unsigned long DLLCALL tjBufSizeYUV2(int width, int pad, int height, int stride=PAD(pw, pad); int ph=tjPlaneHeight(i, height, subsamp); if(pw<0 || ph<0) return -1; - else retval+=stride*ph; + else retval+=(unsigned long long)stride*ph; } + if (retval > (unsigned long long)((unsigned long)-1)) + _throw("tjBufSizeYUV2(): Image is too large"); bailout: - return retval; + return (unsigned long) retval; } DLLEXPORT unsigned long DLLCALL tjBufSizeYUV(int width, int height, @@ -756,7 +764,7 @@ DLLEXPORT int tjPlaneHeight(int componentID, int height, int subsamp) DLLEXPORT unsigned long DLLCALL tjPlaneSizeYUV(int componentID, int width, int stride, int height, int subsamp) { - unsigned long retval=0; + unsigned long long retval=0; int pw, ph; if(width<1 || height<1 || subsamp<0 || subsamp>=NUMSUBOPT) @@ -769,10 +777,12 @@ DLLEXPORT unsigned long DLLCALL tjPlaneSizeYUV(int componentID, int width, if(stride==0) stride=pw; else stride=abs(stride); - retval=stride*(ph-1)+pw; + retval=(unsigned long long)stride*(ph-1)+pw; + if (retval > (unsigned long long)((unsigned long)-1)) + _throw("tjPlaneSizeYUV(): Image is too large"); bailout: - return retval; + return (unsigned long)retval; } @@ -836,8 +846,8 @@ DLLEXPORT int DLLCALL tjCompress2(tjhandle handle, const unsigned char *srcBuf, for(i=0; inext_scanlineimage_height) { @@ -964,8 +974,8 @@ DLLEXPORT int DLLCALL tjEncodeYUVPlanes(tjhandle handle, for(i=0; ioutput_height; i++) { if(flags&TJFLAG_BOTTOMUP) - row_pointer[i]=&dstBuf[(dinfo->output_height-i-1)*pitch]; - else row_pointer[i]=&dstBuf[i*pitch]; + row_pointer[i]=&dstBuf[(dinfo->output_height-i-1)*(size_t)pitch]; + else row_pointer[i]=&dstBuf[i*(size_t)pitch]; } while(dinfo->output_scanlineoutput_height) { @@ -1672,8 +1682,8 @@ DLLEXPORT int DLLCALL tjDecodeYUVPlanes(tjhandle handle, _throw("tjDecodeYUVPlanes(): Memory allocation failure"); for(i=0; i Date: Tue, 12 Nov 2019 12:27:22 -0600 Subject: [PATCH 2/2] 64-bit tjbench: Fix signed int overflow/segfault ... that occurred when attempting to decompress images with more than 715827882 (2048*1024*1024 / 3) pixels. Fixes #388 (cherry picked from commit c30b1e72dac76343ef9029833d1561de07d29bad) --- tjbench.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tjbench.c b/tjbench.c index e90ba3d..858471c 100644 --- a/tjbench.c +++ b/tjbench.c @@ -137,7 +137,7 @@ int decomp(unsigned char *srcbuf, unsigned char **jpegbuf, } /* Set the destination buffer to gray so we know whether the decompressor attempted to write to it */ - memset(dstbuf, 127, pitch*scaledh); + memset(dstbuf, 127, (size_t)pitch * scaledh); if(doyuv) { @@ -159,7 +159,7 @@ int decomp(unsigned char *srcbuf, unsigned char **jpegbuf, { int tile=0; double start=gettime(); - for(row=0, dstptr=dstbuf; row