From f42765c206cc2e4a77f5e3c107a9a258d981c98e Mon Sep 17 00:00:00 2001

From: =?UTF-8?q?Nikola=20Forr=C3=B3?= <nforro@redhat.com>

Date: Mon, 16 Jan 2017 18:13:13 +0100

Subject: [PATCH 2/5] Fix CVE-2016-9535

---

libtiff/tif_fax3.c | 3 +-

libtiff/tif_lzw.c | 9 +-

libtiff/tif_predict.c | 285 +++++++++++++++++++++++++++++++++-----------------

libtiff/tif_predict.h | 6 +-

4 files changed, 203 insertions(+), 100 deletions(-)

diff --git a/libtiff/tif_fax3.c b/libtiff/tif_fax3.c

index 2b2dccd..777ef8f 100644

--- a/libtiff/tif_fax3.c

+++ b/libtiff/tif_fax3.c

@@ -442,8 +442,9 @@ _TIFFFax3fillruns(unsigned char* buf, uint32* runs, uint32* erun, uint32 lastx)

FILL(n, cp);

run &= 7;

}

+ /* Explicit 0xff masking to make icc -check=conversions happy */

if (run)

- cp[0] |= 0xff00 >> run;

+ cp[0] = (unsigned char)((cp[0] | (0xff00 >> run))&0xff);

} else

cp[0] |= _fillmasks[run]>>bx;

x += runs[1];

diff --git a/libtiff/tif_lzw.c b/libtiff/tif_lzw.c

index fd9c7a0..d7fbe74 100644

--- a/libtiff/tif_lzw.c

+++ b/libtiff/tif_lzw.c

@@ -830,13 +830,15 @@ LZWPreEncode(TIFF* tif, uint16 s)

} else \

rat = (incount<<8) / outcount; \

}

+

+/* Explicit 0xff masking to make icc -check=conversions happy */

#define PutNextCode(op, c) { \

nextdata = (nextdata << nbits) | c; \

nextbits += nbits; \

- *op++ = (unsigned char)(nextdata >> (nextbits-8)); \

+ *op++ = (unsigned char)((nextdata >> (nextbits-8))&0xff); \

nextbits -= 8; \

if (nextbits >= 8) { \

- *op++ = (unsigned char)(nextdata >> (nextbits-8)); \

+ *op++ = (unsigned char)((nextdata >> (nextbits-8))&0xff); \

nextbits -= 8; \

} \

outcount += nbits; \

@@ -1042,8 +1044,9 @@ LZWPostEncode(TIFF* tif)

sp->enc_oldcode = (hcode_t) -1;

}

PutNextCode(op, CODE_EOI);

+ /* Explicit 0xff masking to make icc -check=conversions happy */

if (nextbits > 0)

- *op++ = (unsigned char)(nextdata << (8-nextbits));

+ *op++ = (unsigned char)((nextdata << (8-nextbits))&0xff);

tif->tif_rawcc = (tmsize_t)(op - tif->tif_rawdata);

return (1);

}

diff --git a/libtiff/tif_predict.c b/libtiff/tif_predict.c

index f93c664..39d764f 100644

--- a/libtiff/tif_predict.c

+++ b/libtiff/tif_predict.c

@@ -34,16 +34,18 @@

#define PredictorState(tif) ((TIFFPredictorState*) (tif)->tif_data)

-static void horAcc8(TIFF* tif, uint8* cp0, tmsize_t cc);

-static void horAcc16(TIFF* tif, uint8* cp0, tmsize_t cc);

-static void horAcc32(TIFF* tif, uint8* cp0, tmsize_t cc);

-static void swabHorAcc16(TIFF* tif, uint8* cp0, tmsize_t cc);

-static void swabHorAcc32(TIFF* tif, uint8* cp0, tmsize_t cc);

-static void horDiff8(TIFF* tif, uint8* cp0, tmsize_t cc);

-static void horDiff16(TIFF* tif, uint8* cp0, tmsize_t cc);

-static void horDiff32(TIFF* tif, uint8* cp0, tmsize_t cc);

-static void fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc);

-static void fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc);

+static int horAcc8(TIFF* tif, uint8* cp0, tmsize_t cc);

+static int horAcc16(TIFF* tif, uint8* cp0, tmsize_t cc);

+static int horAcc32(TIFF* tif, uint8* cp0, tmsize_t cc);

+static int swabHorAcc16(TIFF* tif, uint8* cp0, tmsize_t cc);

+static int swabHorAcc32(TIFF* tif, uint8* cp0, tmsize_t cc);

+static int horDiff8(TIFF* tif, uint8* cp0, tmsize_t cc);

+static int horDiff16(TIFF* tif, uint8* cp0, tmsize_t cc);

+static int horDiff32(TIFF* tif, uint8* cp0, tmsize_t cc);

+static int swabHorDiff16(TIFF* tif, uint8* cp0, tmsize_t cc);

+static int swabHorDiff32(TIFF* tif, uint8* cp0, tmsize_t cc);

+static int fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc);

+static int fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc);

static int PredictorDecodeRow(TIFF* tif, uint8* op0, tmsize_t occ0, uint16 s);

static int PredictorDecodeTile(TIFF* tif, uint8* op0, tmsize_t occ0, uint16 s);

static int PredictorEncodeRow(TIFF* tif, uint8* bp, tmsize_t cc, uint16 s);

@@ -207,7 +209,24 @@ PredictorSetupEncode(TIFF* tif)

sp->encodetile = tif->tif_encodetile;

tif->tif_encodetile = PredictorEncodeTile;

}

- }

+

+ /*

+ * If the data is horizontally differenced 16-bit data that

+ * requires byte-swapping, then it must be byte swapped after

+ * the differenciation step. We do this with a special-purpose

+ * routine and override the normal post decoding logic that

+ * the library setup when the directory was read.

+ */

+ if (tif->tif_flags & TIFF_SWAB) {

+ if (sp->encodepfunc == horDiff16) {

+ sp->encodepfunc = swabHorDiff16;

+ tif->tif_postdecode = _TIFFNoPostDecode;

+ } else if (sp->encodepfunc == horDiff32) {

+ sp->encodepfunc = swabHorDiff32;

+ tif->tif_postdecode = _TIFFNoPostDecode;

+ }

+ }

+ }

else if (sp->predictor == 3) {

sp->encodepfunc = fpDiff;

@@ -239,13 +258,25 @@ PredictorSetupEncode(TIFF* tif)

case 0: ; \

}

-static void

+/* Remarks related to C standard compliance in all below functions : */

+/* - to avoid any undefined behaviour, we only operate on unsigned types */

+/* since the behaviour of "overflows" is defined (wrap over) */

+/* - when storing into the byte stream, we explicitly mask with 0xff so */

+/* as to make icc -check=conversions happy (not necessary by the standard) */

+

+static int

horAcc8(TIFF* tif, uint8* cp0, tmsize_t cc)

{

tmsize_t stride = PredictorState(tif)->stride;

- char* cp = (char*) cp0;

- assert((cc%stride)==0);

+ unsigned char* cp = (unsigned char*) cp0;

+ if((cc%stride)!=0)

+ {

+ TIFFErrorExt(tif->tif_clientdata, "horAcc8",

+ "%s", "(cc%stride)!=0");

+ return 0;

+ }

+

if (cc > stride) {

/*

* Pipeline the most common cases.

@@ -257,9 +288,9 @@ horAcc8(TIFF* tif, uint8* cp0, tmsize_t cc)

cc -= 3;

cp += 3;

while (cc>0) {

- cp[0] = (char) (cr += cp[0]);

- cp[1] = (char) (cg += cp[1]);

- cp[2] = (char) (cb += cp[2]);

+ cp[0] = (unsigned char) ((cr += cp[0]) & 0xff);

+ cp[1] = (unsigned char) ((cg += cp[1]) & 0xff);

+ cp[2] = (unsigned char) ((cb += cp[2]) & 0xff);

cc -= 3;

cp += 3;

}

@@ -271,10 +302,10 @@ horAcc8(TIFF* tif, uint8* cp0, tmsize_t cc)

cc -= 4;

cp += 4;

while (cc>0) {

- cp[0] = (char) (cr += cp[0]);

- cp[1] = (char) (cg += cp[1]);

- cp[2] = (char) (cb += cp[2]);

- cp[3] = (char) (ca += cp[3]);

+ cp[0] = (unsigned char) ((cr += cp[0]) & 0xff);

+ cp[1] = (unsigned char) ((cg += cp[1]) & 0xff);

+ cp[2] = (unsigned char) ((cb += cp[2]) & 0xff);

+ cp[3] = (unsigned char) ((ca += cp[3]) & 0xff);

cc -= 4;

cp += 4;

}

@@ -282,77 +313,71 @@ horAcc8(TIFF* tif, uint8* cp0, tmsize_t cc)

cc -= stride;

do {

REPEAT4(stride, cp[stride] =

- (char) (cp[stride] + *cp); cp++)

+ (unsigned char) ((cp[stride] + *cp) & 0xff); cp++)

cc -= stride;

} while (cc>0);

}

}

+ return 1;

}

-static void

+static int

swabHorAcc16(TIFF* tif, uint8* cp0, tmsize_t cc)

{

- tmsize_t stride = PredictorState(tif)->stride;

uint16* wp = (uint16*) cp0;

tmsize_t wc = cc / 2;

- assert((cc%(2*stride))==0);

-

- if (wc > stride) {

- TIFFSwabArrayOfShort(wp, wc);

- wc -= stride;

- do {

- REPEAT4(stride, wp[stride] += wp[0]; wp++)

- wc -= stride;

- } while (wc > 0);

- }

+ TIFFSwabArrayOfShort(wp, wc);

+ return horAcc16(tif, cp0, cc);

}

-static void

+static int

horAcc16(TIFF* tif, uint8* cp0, tmsize_t cc)

{

tmsize_t stride = PredictorState(tif)->stride;

uint16* wp = (uint16*) cp0;

tmsize_t wc = cc / 2;

- assert((cc%(2*stride))==0);

+ if((cc%(2*stride))!=0)

+ {

+ TIFFErrorExt(tif->tif_clientdata, "horAcc16",

+ "%s", "cc%(2*stride))!=0");

+ return 0;

+ }

if (wc > stride) {

wc -= stride;

do {

- REPEAT4(stride, wp[stride] += wp[0]; wp++)

+ REPEAT4(stride, wp[stride] = (uint16)(((unsigned int)wp[stride] + (unsigned int)wp[0]) & 0xffff); wp++)

wc -= stride;

} while (wc > 0);

}

+ return 1;

}

-static void

+static int

swabHorAcc32(TIFF* tif, uint8* cp0, tmsize_t cc)

{

- tmsize_t stride = PredictorState(tif)->stride;

uint32* wp = (uint32*) cp0;

tmsize_t wc = cc / 4;

- assert((cc%(4*stride))==0);

-

- if (wc > stride) {

- TIFFSwabArrayOfLong(wp, wc);

- wc -= stride;

- do {

- REPEAT4(stride, wp[stride] += wp[0]; wp++)

- wc -= stride;

- } while (wc > 0);

- }

+ TIFFSwabArrayOfLong(wp, wc);

+ return horAcc32(tif, cp0, cc);

}

-static void

+static int

horAcc32(TIFF* tif, uint8* cp0, tmsize_t cc)

{

tmsize_t stride = PredictorState(tif)->stride;

uint32* wp = (uint32*) cp0;

tmsize_t wc = cc / 4;

- assert((cc%(4*stride))==0);

+ if((cc%(4*stride))!=0)

+ {

+ TIFFErrorExt(tif->tif_clientdata, "horAcc32",

+ "%s", "cc%(4*stride))!=0");

+ return 0;

+ }

if (wc > stride) {

wc -= stride;

@@ -361,12 +386,13 @@ horAcc32(TIFF* tif, uint8* cp0, tmsize_t cc)

wc -= stride;

} while (wc > 0);

}

+ return 1;

}

/*

* Floating point predictor accumulation routine.

*/

-static void

+static int

fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc)

{

tmsize_t stride = PredictorState(tif)->stride;

@@ -374,15 +400,22 @@ fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc)

tmsize_t wc = cc / bps;

tmsize_t count = cc;

uint8 *cp = (uint8 *) cp0;

- uint8 *tmp = (uint8 *)_TIFFmalloc(cc);

+ uint8 *tmp;

- assert((cc%(bps*stride))==0);

+ if(cc%(bps*stride)!=0)

+ {

+ TIFFErrorExt(tif->tif_clientdata, "fpAcc",

+ "%s", "cc%(bps*stride))!=0");

+ return 0;

+ }

+ tmp = (uint8 *)_TIFFmalloc(cc);

if (!tmp)

- return;

+ return 0;

while (count > stride) {

- REPEAT4(stride, cp[stride] += cp[0]; cp++)

+ REPEAT4(stride, cp[stride] =

+ (unsigned char) ((cp[stride] + cp[0]) & 0xff); cp++)

count -= stride;

}

@@ -400,6 +433,7 @@ fpAcc(TIFF* tif, uint8* cp0, tmsize_t cc)

}

}

_TIFFfree(tmp);

+ return 1;

}

/*

@@ -415,8 +449,7 @@ PredictorDecodeRow(TIFF* tif, uint8* op0, tmsize_t occ0, uint16 s)

assert(sp->decodepfunc != NULL);

if ((*sp->decoderow)(tif, op0, occ0, s)) {

- (*sp->decodepfunc)(tif, op0, occ0);

- return 1;

+ return (*sp->decodepfunc)(tif, op0, occ0);

} else

return 0;

}

@@ -439,10 +472,16 @@ PredictorDecodeTile(TIFF* tif, uint8* op0, tmsize_t occ0, uint16 s)

if ((*sp->decodetile)(tif, op0, occ0, s)) {

tmsize_t rowsize = sp->rowsize;

assert(rowsize > 0);

- assert((occ0%rowsize)==0);

+ if((occ0%rowsize) !=0)

+ {

+ TIFFErrorExt(tif->tif_clientdata, "PredictorDecodeTile",

+ "%s", "occ0%rowsize != 0");

+ return 0;

+ }

assert(sp->decodepfunc != NULL);

while (occ0 > 0) {

- (*sp->decodepfunc)(tif, op0, rowsize);

+ if( !(*sp->decodepfunc)(tif, op0, rowsize) )

+ return 0;

occ0 -= rowsize;

op0 += rowsize;

}

@@ -451,14 +490,19 @@ PredictorDecodeTile(TIFF* tif, uint8* op0, tmsize_t occ0, uint16 s)

return 0;

}

-static void

+static int

horDiff8(TIFF* tif, uint8* cp0, tmsize_t cc)

{

TIFFPredictorState* sp = PredictorState(tif);

tmsize_t stride = sp->stride;

- char* cp = (char*) cp0;

+ unsigned char* cp = (unsigned char*) cp0;

- assert((cc%stride)==0);

+ if((cc%stride)!=0)

+ {

+ TIFFErrorExt(tif->tif_clientdata, "horDiff8",

+ "%s", "(cc%stride)!=0");

+ return 0;

+ }

if (cc > stride) {

cc -= stride;

@@ -466,67 +510,92 @@ horDiff8(TIFF* tif, uint8* cp0, tmsize_t cc)

* Pipeline the most common cases.

*/

if (stride == 3) {

- int r1, g1, b1;

- int r2 = cp[0];

- int g2 = cp[1];

- int b2 = cp[2];

+ unsigned int r1, g1, b1;

+ unsigned int r2 = cp[0];

+ unsigned int g2 = cp[1];

+ unsigned int b2 = cp[2];

do {

- r1 = cp[3]; cp[3] = r1-r2; r2 = r1;

- g1 = cp[4]; cp[4] = g1-g2; g2 = g1;

- b1 = cp[5]; cp[5] = b1-b2; b2 = b1;

+ r1 = cp[3]; cp[3] = (unsigned char)((r1-r2)&0xff); r2 = r1;

+ g1 = cp[4]; cp[4] = (unsigned char)((g1-g2)&0xff); g2 = g1;

+ b1 = cp[5]; cp[5] = (unsigned char)((b1-b2)&0xff); b2 = b1;

cp += 3;

} while ((cc -= 3) > 0);

} else if (stride == 4) {

- int r1, g1, b1, a1;

- int r2 = cp[0];

- int g2 = cp[1];

- int b2 = cp[2];

- int a2 = cp[3];

+ unsigned int r1, g1, b1, a1;

+ unsigned int r2 = cp[0];

+ unsigned int g2 = cp[1];

+ unsigned int b2 = cp[2];

+ unsigned int a2 = cp[3];

do {

- r1 = cp[4]; cp[4] = r1-r2; r2 = r1;

- g1 = cp[5]; cp[5] = g1-g2; g2 = g1;

- b1 = cp[6]; cp[6] = b1-b2; b2 = b1;

- a1 = cp[7]; cp[7] = a1-a2; a2 = a1;

+ r1 = cp[4]; cp[4] = (unsigned char)((r1-r2)&0xff); r2 = r1;

+ g1 = cp[5]; cp[5] = (unsigned char)((g1-g2)&0xff); g2 = g1;

+ b1 = cp[6]; cp[6] = (unsigned char)((b1-b2)&0xff); b2 = b1;

+ a1 = cp[7]; cp[7] = (unsigned char)((a1-a2)&0xff); a2 = a1;

cp += 4;

} while ((cc -= 4) > 0);

} else {

cp += cc - 1;

do {

- REPEAT4(stride, cp[stride] -= cp[0]; cp--)

+ REPEAT4(stride, cp[stride] = (unsigned char)((cp[stride] - cp[0])&0xff); cp--)

} while ((cc -= stride) > 0);

}

}

+ return 1;

}

-static void

+static int

horDiff16(TIFF* tif, uint8* cp0, tmsize_t cc)

{

TIFFPredictorState* sp = PredictorState(tif);

tmsize_t stride = sp->stride;

- int16 *wp = (int16*) cp0;

+ uint16 *wp = (uint16*) cp0;

tmsize_t wc = cc/2;

- assert((cc%(2*stride))==0);

+ if((cc%(2*stride))!=0)

+ {

+ TIFFErrorExt(tif->tif_clientdata, "horDiff8",

+ "%s", "(cc%(2*stride))!=0");

+ return 0;

+ }

if (wc > stride) {

wc -= stride;

wp += wc - 1;

do {

- REPEAT4(stride, wp[stride] -= wp[0]; wp--)

+ REPEAT4(stride, wp[stride] = (uint16)(((unsigned int)wp[stride] - (unsigned int)wp[0]) & 0xffff); wp--)

wc -= stride;

} while (wc > 0);

}

+ return 1;

}

-static void

+static int

+swabHorDiff16(TIFF* tif, uint8* cp0, tmsize_t cc)

+{

+ uint16* wp = (uint16*) cp0;

+ tmsize_t wc = cc / 2;

+

+ if( !horDiff16(tif, cp0, cc) )

+ return 0;

+

+ TIFFSwabArrayOfShort(wp, wc);

+ return 1;

+}

+

+static int

horDiff32(TIFF* tif, uint8* cp0, tmsize_t cc)

{

TIFFPredictorState* sp = PredictorState(tif);

tmsize_t stride = sp->stride;

- int32 *wp = (int32*) cp0;

+ uint32 *wp = (uint32*) cp0;

tmsize_t wc = cc/4;

- assert((cc%(4*stride))==0);

+ if((cc%(4*stride))!=0)

+ {

+ TIFFErrorExt(tif->tif_clientdata, "horDiff32",

+ "%s", "(cc%(4*stride))!=0");

+ return 0;

+ }

if (wc > stride) {

wc -= stride;

@@ -536,12 +605,26 @@ horDiff32(TIFF* tif, uint8* cp0, tmsize_t cc)

wc -= stride;

} while (wc > 0);

}

+ return 1;

+}

+

+static int

+swabHorDiff32(TIFF* tif, uint8* cp0, tmsize_t cc)

+{

+ uint32* wp = (uint32*) cp0;

+ tmsize_t wc = cc / 4;

+

+ if( !horDiff32(tif, cp0, cc) )

+ return 0;

+

+ TIFFSwabArrayOfLong(wp, wc);

+ return 1;

}

/*

* Floating point predictor differencing routine.

*/

-static void

+static int

fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc)

{

tmsize_t stride = PredictorState(tif)->stride;

@@ -549,12 +632,18 @@ fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc)

tmsize_t wc = cc / bps;

tmsize_t count;

uint8 *cp = (uint8 *) cp0;

- uint8 *tmp = (uint8 *)_TIFFmalloc(cc);

+ uint8 *tmp;

- assert((cc%(bps*stride))==0);

+ if((cc%(bps*stride))!=0)

+ {

+ TIFFErrorExt(tif->tif_clientdata, "fpDiff",

+ "%s", "(cc%(bps*stride))!=0");

+ return 0;

+ }

+ tmp = (uint8 *)_TIFFmalloc(cc);

if (!tmp)

- return;

+ return 0;

_TIFFmemcpy(tmp, cp0, cc);

for (count = 0; count < wc; count++) {

@@ -573,7 +662,8 @@ fpDiff(TIFF* tif, uint8* cp0, tmsize_t cc)

cp = (uint8 *) cp0;

cp += cc - stride - 1;

for (count = cc; count > stride; count -= stride)

- REPEAT4(stride, cp[stride] -= cp[0]; cp--)

+ REPEAT4(stride, cp[stride] = (unsigned char)((cp[stride] - cp[0])&0xff); cp--)

+ return 1;

}

static int

@@ -586,7 +676,8 @@ PredictorEncodeRow(TIFF* tif, uint8* bp, tmsize_t cc, uint16 s)

assert(sp->encoderow != NULL);

/* XXX horizontal differencing alters user's data XXX */

- (*sp->encodepfunc)(tif, bp, cc);

+ if( !(*sp->encodepfunc)(tif, bp, cc) )

+ return 0;

return (*sp->encoderow)(tif, bp, cc, s);

}

@@ -621,7 +712,13 @@ PredictorEncodeTile(TIFF* tif, uint8* bp0, tmsize_t cc0, uint16 s)

rowsize = sp->rowsize;

assert(rowsize > 0);

- assert((cc0%rowsize)==0);

+ if((cc0%rowsize)!=0)

+ {

+ TIFFErrorExt(tif->tif_clientdata, "PredictorEncodeTile",

+ "%s", "(cc0%rowsize)!=0");

+ _TIFFfree( working_copy );

+ return 0;

+ }

while (cc > 0) {

(*sp->encodepfunc)(tif, bp, rowsize);

cc -= rowsize;

diff --git a/libtiff/tif_predict.h b/libtiff/tif_predict.h

index dc7144c..1585a58 100644

--- a/libtiff/tif_predict.h

+++ b/libtiff/tif_predict.h

@@ -30,6 +30,8 @@

* ``Library-private'' Support for the Predictor Tag

*/

+typedef int (*TIFFEncodeDecodeMethod)(TIFF* tif, uint8* buf, tmsize_t size);

+

/*

* Codecs that want to support the Predictor tag must place

* this structure first in their private state block so that

@@ -43,12 +45,12 @@ typedef struct {

TIFFCodeMethod encoderow; /* parent codec encode/decode row */

TIFFCodeMethod encodestrip; /* parent codec encode/decode strip */

TIFFCodeMethod encodetile; /* parent codec encode/decode tile */

- TIFFPostMethod encodepfunc; /* horizontal differencer */

+ TIFFEncodeDecodeMethod encodepfunc; /* horizontal differencer */

TIFFCodeMethod decoderow; /* parent codec encode/decode row */

TIFFCodeMethod decodestrip; /* parent codec encode/decode strip */

TIFFCodeMethod decodetile; /* parent codec encode/decode tile */

- TIFFPostMethod decodepfunc; /* horizontal accumulator */

+ TIFFEncodeDecodeMethod decodepfunc; /* horizontal accumulator */

TIFFVGetMethod vgetparent; /* super-class method */

TIFFVSetMethod vsetparent; /* super-class method */

--

2.7.4