Merge pull request #1375 from rouault/fix_ossfuzz_11700_30646

Avoid integer overflows in DWT.
uclouvain · Sep 8, 2021 · 0c19626 · 0c19626
2 parents 172583a + badbd93
commit 0c19626
Show file tree

Hide file tree

Showing 2 changed files with 55 additions and 7 deletions.
diff --git a/src/lib/openjp2/dwt.c b/src/lib/openjp2/dwt.c
@@ -385,7 +385,8 @@ static void  opj_idwt53_h_cas0(OPJ_INT32* tmp,
         s0n = s1n - ((d1c + d1n + 2) >> 2);
 
         tmp[i  ] = s0c;
-        tmp[i + 1] = d1c + ((s0c + s0n) >> 1);
+        tmp[i + 1] = opj_int_add_no_overflow(d1c, opj_int_add_no_overflow(s0c,
+                                             s0n) >> 1);
     }
 
     tmp[i] = s0n;
@@ -450,7 +451,7 @@ static void  opj_idwt53_h_cas1(OPJ_INT32* tmp,
 
         dn = in_odd[j] - ((s1 + s2 + 2) >> 2);
         tmp[i  ] = dc;
-        tmp[i + 1] = s1 + ((dn + dc) >> 1);
+        tmp[i + 1] = opj_int_add_no_overflow(s1, opj_int_add_no_overflow(dn, dc) >> 1);
 
         dc = dn;
         s1 = s2;
@@ -796,7 +797,8 @@ static void opj_idwt3_v_cas0(OPJ_INT32* tmp,
         s1n = tiledp_col[(OPJ_SIZE_T)(j + 1) * stride];
         d1n = tiledp_col[(OPJ_SIZE_T)(sn + j + 1) * stride];
 
-        s0n = s1n - ((d1c + d1n + 2) >> 2);
+        s0n = opj_int_sub_no_overflow(s1n,
+                                      opj_int_add_no_overflow(opj_int_add_no_overflow(d1c, d1n), 2) >> 2);
 
         tmp[i  ] = s0c;
         tmp[i + 1] = d1c + ((s0c + s0n) >> 1);
@@ -2343,10 +2345,13 @@ static void opj_dwt_decode_partial_1(OPJ_INT32 *a, OPJ_INT32 dn, OPJ_INT32 sn,
             OPJ_S(0) /= 2;
         } else {
             for (i = win_l_x0; i < win_l_x1; i++) {
-                OPJ_D(i) -= (OPJ_SS_(i) + OPJ_SS_(i + 1) + 2) >> 2;
+                OPJ_D(i) = opj_int_sub_no_overflow(OPJ_D(i),
+                                                   opj_int_add_no_overflow(opj_int_add_no_overflow(OPJ_SS_(i), OPJ_SS_(i + 1)),
+                                                           2) >> 2);
             }
             for (i = win_h_x0; i < win_h_x1; i++) {
-                OPJ_S(i) += (OPJ_DD_(i) + OPJ_DD_(i - 1)) >> 1;
+                OPJ_S(i) = opj_int_add_no_overflow(OPJ_S(i),
+                                                   opj_int_add_no_overflow(OPJ_DD_(i), OPJ_DD_(i - 1)) >> 1);
             }
         }
     }
@@ -2484,12 +2489,17 @@ static void opj_dwt_decode_partial_1_parallel(OPJ_INT32 *a,
         } else {
             for (i = win_l_x0; i < win_l_x1; i++) {
                 for (off = 0; off < 4; off++) {
-                    OPJ_D_off(i, off) -= (OPJ_SS__off(i, off) + OPJ_SS__off(i + 1, off) + 2) >> 2;
+                    OPJ_D_off(i, off) = opj_int_sub_no_overflow(
+                                            OPJ_D_off(i, off),
+                                            opj_int_add_no_overflow(
+                                                opj_int_add_no_overflow(OPJ_SS__off(i, off), OPJ_SS__off(i + 1, off)), 2) >> 2);
                 }
             }
             for (i = win_h_x0; i < win_h_x1; i++) {
                 for (off = 0; off < 4; off++) {
-                    OPJ_S_off(i, off) += (OPJ_DD__off(i, off) + OPJ_DD__off(i - 1, off)) >> 1;
+                    OPJ_S_off(i, off) = opj_int_add_no_overflow(
+                                            OPJ_S_off(i, off),
+                                            opj_int_add_no_overflow(OPJ_DD__off(i, off), OPJ_DD__off(i - 1, off)) >> 1);
                 }
             }
         }

diff --git a/src/lib/openjp2/opj_intmath.h b/src/lib/openjp2/opj_intmath.h
@@ -276,6 +276,44 @@ static INLINE OPJ_INT32 opj_int_fix_mul_t1(OPJ_INT32 a, OPJ_INT32 b)
     return (OPJ_INT32)(temp >> (13 + 11 - T1_NMSEDEC_FRACBITS)) ;
 }
 
+/**
+Addtion two signed integers with a wrap-around behaviour.
+Assumes complement-to-two signed integers.
+@param a
+@param b
+@return Returns a + b
+*/
+static INLINE OPJ_INT32 opj_int_add_no_overflow(OPJ_INT32 a, OPJ_INT32 b)
+{
+    void* pa = &a;
+    void* pb = &b;
+    OPJ_UINT32* upa = (OPJ_UINT32*)pa;
+    OPJ_UINT32* upb = (OPJ_UINT32*)pb;
+    OPJ_UINT32 ures = *upa + *upb;
+    void* pures = &ures;
+    OPJ_INT32* ipres = (OPJ_INT32*)pures;
+    return *ipres;
+}
+
+/**
+Subtract two signed integers with a wrap-around behaviour.
+Assumes complement-to-two signed integers.
+@param a
+@param b
+@return Returns a - b
+*/
+static INLINE OPJ_INT32 opj_int_sub_no_overflow(OPJ_INT32 a, OPJ_INT32 b)
+{
+    void* pa = &a;
+    void* pb = &b;
+    OPJ_UINT32* upa = (OPJ_UINT32*)pa;
+    OPJ_UINT32* upb = (OPJ_UINT32*)pb;
+    OPJ_UINT32 ures = *upa - *upb;
+    void* pures = &ures;
+    OPJ_INT32* ipres = (OPJ_INT32*)pures;
+    return *ipres;
+}
+
 /* ----------------------------------------------------------------------- */
 /*@}*/