Prevent overflow by large arguments of opj_calloc

calloc allocates (num * size) bytes, possibly without handling an
overflow when doing the multiplication. This is fatal because the
allocated memory would be too small in that case.

Signed-off-by: Stefan Weil <[email protected]>

src/lib/openjp2/opj_malloc.c

@@ -32,6 +32,8 @@
 #define OPJ_SKIP_POISON
 #include "opj_includes.h"
 
+#include <errno.h>
+
 #if defined(OPJ_HAVE_MALLOC_H) && defined(OPJ_HAVE_MEMALIGN)
 # include <malloc.h>
 #endif
@@ -201,6 +203,11 @@ void * opj_calloc(size_t num, size_t size)
         /* prevent implementation defined behavior of realloc */
         return NULL;
     }
+    if (num > SIZE_MAX / size) {
+        /* prevent overflow */
+        errno = ENOMEM;
+        return NULL;
+    }
     return calloc(num, size);
 }