From 32333a46981d0779f987040cfac795f4c9a59b64 Mon Sep 17 00:00:00 2001 From: Harry Mallon Date: Fri, 16 Oct 2020 10:59:31 +0100 Subject: [PATCH 01/15] Revert "Disable OPENEXR_IMF_HAVE_GCC_INLINE_ASM_AVX when building on arm64 macOS" This reverts commit 67053ebf1882ed93e74dbb7b5dcdcf0b2420c2c9. Signed-off-by: Harry Mallon --- OpenEXR/config/CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/OpenEXR/config/CMakeLists.txt b/OpenEXR/config/CMakeLists.txt index 6e14e10b66..b52ad0d556 100644 --- a/OpenEXR/config/CMakeLists.txt +++ b/OpenEXR/config/CMakeLists.txt @@ -47,9 +47,6 @@ set(OPENEXR_IMF_HAVE_COMPLETE_IOMANIP TRUE) if(APPLE) set(OPENEXR_IMF_HAVE_DARWIN TRUE) - if (${CMAKE_OSX_ARCHITECTURES} MATCHES arm64) - set(OPENEXR_IMF_HAVE_GCC_INLINE_ASM_AVX FALSE) - endif() endif() configure_file(OpenEXRConfig.h.in_cmake ${CMAKE_CURRENT_BINARY_DIR}/OpenEXRConfig.h) From 2d89f0b5e2bca2d37d15bebbb1608fda9841d63c Mon Sep 17 00:00:00 2001 From: Harry Mallon Date: Fri, 16 Oct 2020 11:44:33 +0100 Subject: [PATCH 02/15] Fix Apple Universal 2 (arm64/x86_64) builds * In these types of builds we want arm64 and x86_64 (with AVX optimisations). However the way cmake works (with `CMAKE_OSX_ARCHITECTURES="arm64;x86_64"` means that we share one OpenEXRConfigInternal.h between both builds. So we have to have OPENEXR_IMF_HAVE_GCC_INLINE_ASM_AVX mean "AVX GCC asm is available if platform is x86", rather than "AVX GCC asm is available". Then we decide on AVX optimisations based on that #define and also the platform defines. Signed-off-by: Harry Mallon --- OpenEXR/IlmImf/ImfDwaCompressorSimd.h | 28 +++++++-------------------- OpenEXR/IlmImf/ImfSystemSpecific.cpp | 6 +++--- OpenEXR/IlmImf/ImfSystemSpecific.h | 15 ++++++++++++++ OpenEXR/config/CMakeLists.txt | 4 +++- 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/OpenEXR/IlmImf/ImfDwaCompressorSimd.h b/OpenEXR/IlmImf/ImfDwaCompressorSimd.h index b4c4107c3f..8a8f93ab89 100644 --- a/OpenEXR/IlmImf/ImfDwaCompressorSimd.h +++ b/OpenEXR/IlmImf/ImfDwaCompressorSimd.h @@ -60,20 +60,6 @@ OPENEXR_IMF_INTERNAL_NAMESPACE_HEADER_ENTER #define _SSE_ALIGNMENT_MASK 0x0F #define _AVX_ALIGNMENT_MASK 0x1F -// -// Test if we should enable GCC inline asm paths for AVX -// - -#ifdef OPENEXR_IMF_HAVE_GCC_INLINE_ASM_AVX - - #define IMF_HAVE_GCC_INLINEASM - - #ifdef __LP64__ - #define IMF_HAVE_GCC_INLINEASM_64 - #endif /* __LP64__ */ - -#endif /* OPENEXR_IMF_HAVE_GCC_INLINE_ASM_AVX */ - // // A simple 64-element array, aligned properly for SIMD access. // @@ -467,7 +453,7 @@ convertFloatToHalf64_f16c (unsigned short *dst, float *src) // I'll take the asm. // - #if defined IMF_HAVE_GCC_INLINEASM + #if defined IMF_HAVE_GCC_INLINEASM_X86 __asm__ ("vmovaps (%0), %%ymm0 \n" "vmovaps 0x20(%0), %%ymm1 \n" @@ -506,7 +492,7 @@ convertFloatToHalf64_f16c (unsigned short *dst, float *src) ); #else convertFloatToHalf64_scalar (dst, src); - #endif /* IMF_HAVE_GCC_INLINEASM */ + #endif /* IMF_HAVE_GCC_INLINEASM_X86 */ } @@ -683,7 +669,7 @@ fromHalfZigZag_scalar (unsigned short *src, float *dst) void fromHalfZigZag_f16c (unsigned short *src, float *dst) { - #if defined IMF_HAVE_GCC_INLINEASM_64 + #if defined IMF_HAVE_GCC_INLINEASM_X86_64 __asm__ /* x3 <- 0 @@ -835,7 +821,7 @@ fromHalfZigZag_f16c (unsigned short *src, float *dst) #else fromHalfZigZag_scalar(src, dst); - #endif /* defined IMF_HAVE_GCC_INLINEASM_64 */ + #endif /* defined IMF_HAVE_GCC_INLINEASM_X86_64 */ } @@ -1607,7 +1593,7 @@ template void dctInverse8x8_avx (float *data) { - #if defined IMF_HAVE_GCC_INLINEASM_64 + #if defined IMF_HAVE_GCC_INLINEASM_X86_64 /* The column-major version of M1, followed by the * column-major version of M2: @@ -1735,11 +1721,11 @@ dctInverse8x8_avx (float *data) } else { assert(false); // Invalid template instance parameter } - #else /* IMF_HAVE_GCC_INLINEASM_64 */ + #else /* IMF_HAVE_GCC_INLINEASM_X86_64 */ dctInverse8x8_scalar(data); - #endif /* IMF_HAVE_GCC_INLINEASM_64 */ + #endif /* IMF_HAVE_GCC_INLINEASM_X86_64 */ } diff --git a/OpenEXR/IlmImf/ImfSystemSpecific.cpp b/OpenEXR/IlmImf/ImfSystemSpecific.cpp index ccce8074d6..76fdfe69e6 100644 --- a/OpenEXR/IlmImf/ImfSystemSpecific.cpp +++ b/OpenEXR/IlmImf/ImfSystemSpecific.cpp @@ -63,7 +63,7 @@ namespace { #endif // IMF_HAVE_SSE2 && __GNUC__ -#ifdef OPENEXR_IMF_HAVE_GCC_INLINE_ASM_AVX +#ifdef IMF_HAVE_GCC_INLINEASM_X86 void xgetbv(int n, int &eax, int &edx) { @@ -74,14 +74,14 @@ namespace { : /* Clobber */); } -#else // OPENEXR_IMF_HAVE_GCC_INLINE_ASM_AVX +#else // IMF_HAVE_GCC_INLINEASM_X86 void xgetbv(int n, int &eax, int &edx) { eax = edx = 0; } -#endif // OPENEXR_IMF_HAVE_GCC_INLINE_ASM_AVX +#endif // IMF_HAVE_GCC_INLINEASM_X86 } // namespace diff --git a/OpenEXR/IlmImf/ImfSystemSpecific.h b/OpenEXR/IlmImf/ImfSystemSpecific.h index fa801b2ab0..706c4eb936 100644 --- a/OpenEXR/IlmImf/ImfSystemSpecific.h +++ b/OpenEXR/IlmImf/ImfSystemSpecific.h @@ -39,9 +39,24 @@ #include "ImfSimd.h" #include #include "ImfExport.h" +#include "OpenEXRConfig.h" +#include "OpenEXRConfigInternal.h" OPENEXR_IMF_INTERNAL_NAMESPACE_HEADER_ENTER +// +// Test if we should enable GCC inline asm paths for AVX +// + +#if defined(OPENEXR_IMF_HAVE_GCC_INLINE_ASM_AVX) && (defined(_M_X64) || defined(__x86_64__)) + + #define IMF_HAVE_GCC_INLINEASM_X86 + + #ifdef __LP64__ + #define IMF_HAVE_GCC_INLINEASM_X86_64 + #endif /* __LP64__ */ + +#endif /* OPENEXR_IMF_HAVE_GCC_INLINE_ASM_AVX */ static unsigned long systemEndianCheckValue = 0x12345678; static unsigned long* systemEndianCheckPointer = &systemEndianCheckValue; diff --git a/OpenEXR/config/CMakeLists.txt b/OpenEXR/config/CMakeLists.txt index b52ad0d556..8386e3a0e9 100644 --- a/OpenEXR/config/CMakeLists.txt +++ b/OpenEXR/config/CMakeLists.txt @@ -19,12 +19,13 @@ int main() sysconf(_SC_NPROCESSORS_ONLN); } " OPENEXR_IMF_HAVE_SYSCONF_NPROCESSORS_ONLN -) +) check_cxx_source_compiles( " int main() { +#if defined(_M_X64) || defined(__x86_64__) #if defined(__SSE2__) int n = 0; int eax = 0; @@ -36,6 +37,7 @@ int main() #else # error No SSE support enabled to query AVX support #endif +#endif } " OPENEXR_IMF_HAVE_GCC_INLINE_ASM_AVX ) From 3b495b8df9bd55b8d606ac5a5012c6d22dd8e48c Mon Sep 17 00:00:00 2001 From: Cary Phillips Date: Wed, 13 Jan 2021 13:06:13 -0800 Subject: [PATCH 03/15] Include where required by newer compilers (#893) * Include where required by newer compilers Signed-off-by: Cary Phillips * Removed redundant #include Signed-off-by: Cary Phillips --- OpenEXR/IlmImf/ImfDeepScanLineInputFile.cpp | 5 ++--- OpenEXR/IlmImf/ImfDeepTiledInputFile.cpp | 2 ++ OpenEXR/IlmImfTest/testLargeDataWindowOffsets.cpp | 1 + OpenEXR/IlmImfTest/testOptimizedInterleavePatterns.cpp | 1 + 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/OpenEXR/IlmImf/ImfDeepScanLineInputFile.cpp b/OpenEXR/IlmImf/ImfDeepScanLineInputFile.cpp index c094fa79cc..5f0f439892 100644 --- a/OpenEXR/IlmImf/ImfDeepScanLineInputFile.cpp +++ b/OpenEXR/IlmImf/ImfDeepScanLineInputFile.cpp @@ -65,12 +65,11 @@ #include "Iex.h" +#include +#include #include #include -#include #include -#include - #include "ImfNamespace.h" OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER diff --git a/OpenEXR/IlmImf/ImfDeepTiledInputFile.cpp b/OpenEXR/IlmImf/ImfDeepTiledInputFile.cpp index c81de6cf0b..56161156d5 100644 --- a/OpenEXR/IlmImf/ImfDeepTiledInputFile.cpp +++ b/OpenEXR/IlmImf/ImfDeepTiledInputFile.cpp @@ -64,6 +64,8 @@ #include #include #include +#include +#include #include #include "ImfNamespace.h" diff --git a/OpenEXR/IlmImfTest/testLargeDataWindowOffsets.cpp b/OpenEXR/IlmImfTest/testLargeDataWindowOffsets.cpp index ce799054eb..cc2d7a3d62 100644 --- a/OpenEXR/IlmImfTest/testLargeDataWindowOffsets.cpp +++ b/OpenEXR/IlmImfTest/testLargeDataWindowOffsets.cpp @@ -40,6 +40,7 @@ #include "ImfStandardAttributes.h" #include #include +#include #include #include #include diff --git a/OpenEXR/IlmImfTest/testOptimizedInterleavePatterns.cpp b/OpenEXR/IlmImfTest/testOptimizedInterleavePatterns.cpp index c8ffd3671d..748ea3b10f 100644 --- a/OpenEXR/IlmImfTest/testOptimizedInterleavePatterns.cpp +++ b/OpenEXR/IlmImfTest/testOptimizedInterleavePatterns.cpp @@ -44,6 +44,7 @@ #include "ImfStandardAttributes.h" #include #include +#include #include #include #include From d755bc2d39427a89b73374cb84d05b0399425581 Mon Sep 17 00:00:00 2001 From: Peter Hillman Date: Wed, 13 Jan 2021 14:00:34 +1300 Subject: [PATCH 04/15] add buffer size validation to FastHuf decode Signed-off-by: Peter Hillman --- OpenEXR/IlmImf/ImfHuf.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/OpenEXR/IlmImf/ImfHuf.cpp b/OpenEXR/IlmImf/ImfHuf.cpp index 1cc09d63fa..271849b7bf 100644 --- a/OpenEXR/IlmImf/ImfHuf.cpp +++ b/OpenEXR/IlmImf/ImfHuf.cpp @@ -1119,6 +1119,14 @@ hufUncompress (const char compressed[], if (FastHufDecoder::enabled() && nBits > 128) { FastHufDecoder fhd (ptr, nCompressed - (ptr - compressed), im, iM, iM); + + // must be nBytes remaining in buffer + if( ptr-compressed + nBytes > nCompressed) + { + notEnoughData(); + return; + } + fhd.decode ((unsigned char*)ptr, nBits, raw, nRaw); } else From 5e6f8f55e7974d4aed1dfb4be458c0c1cc5b9534 Mon Sep 17 00:00:00 2001 From: Peter Hillman Date: Wed, 13 Jan 2021 10:59:26 +1300 Subject: [PATCH 05/15] prevent overflow in RgbaFile cachePadding Signed-off-by: Peter Hillman --- OpenEXR/IlmImf/ImfRgbaFile.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/OpenEXR/IlmImf/ImfRgbaFile.cpp b/OpenEXR/IlmImf/ImfRgbaFile.cpp index 68fa51cb52..b020f4e8a2 100644 --- a/OpenEXR/IlmImf/ImfRgbaFile.cpp +++ b/OpenEXR/IlmImf/ImfRgbaFile.cpp @@ -172,16 +172,16 @@ cachePadding (ptrdiff_t size) static int LOG2_CACHE_LINE_SIZE = 8; - int i = LOG2_CACHE_LINE_SIZE + 2; + size_t i = LOG2_CACHE_LINE_SIZE + 2; while ((size >> i) > 1) ++i; - if (size > (1 << (i + 1)) - 64) - return 64 + ((1 << (i + 1)) - size); + if (size > (1ll << (i + 1)) - 64ll) + return 64ll + ((1ll << (i + 1ll)) - size); - if (size < (1 << i) + 64) - return 64 + ((1 << i) - size); + if (size < (1ll << i) + 64ll) + return 64ll + ((1ll << i) - size); return 0; } From 846b470c193f16ebb214de476f87ba78e2143815 Mon Sep 17 00:00:00 2001 From: peterhillman Date: Wed, 20 Jan 2021 14:37:21 +1300 Subject: [PATCH 06/15] Use size_t for DWA buffersize calculation (#901) * Use size_t for DWA buffersize calculation Signed-off-by: Peter Hillman * use Int64 instead of size_t for buffersize calculations Signed-off-by: Peter Hillman Signed-off-by: Cary Phillips --- OpenEXR/IlmImf/ImfDwaCompressor.cpp | 42 ++++++++++++++++------------- OpenEXR/IlmImf/ImfDwaCompressor.h | 20 +++++++------- 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/OpenEXR/IlmImf/ImfDwaCompressor.cpp b/OpenEXR/IlmImf/ImfDwaCompressor.cpp index c84f03074b..c772c0d734 100644 --- a/OpenEXR/IlmImf/ImfDwaCompressor.cpp +++ b/OpenEXR/IlmImf/ImfDwaCompressor.cpp @@ -2932,10 +2932,10 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize) // of channels we have. // - int maxOutBufferSize = 0; + Int64 maxOutBufferSize = 0; int numLossyDctChans = 0; - int unknownBufferSize = 0; - int rleBufferSize = 0; + Int64 unknownBufferSize = 0; + Int64 rleBufferSize = 0; int maxLossyDctAcSize = (int)ceil ((float)numScanLines() / 8.0f) * (int)ceil ((float)(_max[0] - _min[0] + 1) / 8.0f) * @@ -2945,6 +2945,8 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize) (int)ceil ((float)(_max[0] - _min[0] + 1) / 8.0f) * sizeof (unsigned short); + Int64 pixelCount = static_cast(numScanLines()) * static_cast(_max[0] - _min[0] + 1); + for (unsigned int chan = 0; chan < _channelData.size(); ++chan) { switch (_channelData[chan].compression) @@ -2971,8 +2973,7 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize) // of the source data. // - int rleAmount = 2 * numScanLines() * (_max[0] - _min[0] + 1) * - OPENEXR_IMF_NAMESPACE::pixelTypeSize (_channelData[chan].type); + Int64 rleAmount = 2 * pixelCount * OPENEXR_IMF_NAMESPACE::pixelTypeSize (_channelData[chan].type); rleBufferSize += rleAmount; } @@ -2981,8 +2982,7 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize) case UNKNOWN: - unknownBufferSize += numScanLines() * (_max[0] - _min[0] + 1) * - OPENEXR_IMF_NAMESPACE::pixelTypeSize (_channelData[chan].type); + unknownBufferSize += pixelCount * OPENEXR_IMF_NAMESPACE::pixelTypeSize (_channelData[chan].type); break; default: @@ -2999,13 +2999,13 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize) // which could take slightly more space // - maxOutBufferSize += (int)compressBound ((uLongf)rleBufferSize); + maxOutBufferSize += compressBound (rleBufferSize); // // And the same goes for the UNKNOWN data // - maxOutBufferSize += (int)compressBound ((uLongf)unknownBufferSize); + maxOutBufferSize += compressBound (unknownBufferSize); // // Allocate a zip/deflate compressor big enought to hold the DC data @@ -3052,7 +3052,7 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize) // to Huffman encoding // - if (static_cast(maxLossyDctAcSize * numLossyDctChans) > _packedAcBufferSize) + if (static_cast(maxLossyDctAcSize * numLossyDctChans) > _packedAcBufferSize) { _packedAcBufferSize = maxLossyDctAcSize * numLossyDctChans; if (_packedAcBuffer != 0) @@ -3064,7 +3064,7 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize) // _packedDcBuffer holds one quantized DCT coef per 8x8 block // - if (static_cast(maxLossyDctDcSize * numLossyDctChans) > _packedDcBufferSize) + if (static_cast(maxLossyDctDcSize * numLossyDctChans) > _packedDcBufferSize) { _packedDcBufferSize = maxLossyDctDcSize * numLossyDctChans; if (_packedDcBuffer != 0) @@ -3072,7 +3072,7 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize) _packedDcBuffer = new char[_packedDcBufferSize]; } - if (static_cast(rleBufferSize) > _rleBufferSize) + if ( rleBufferSize > _rleBufferSize ) { _rleBufferSize = rleBufferSize; if (_rleBuffer != 0) @@ -3091,7 +3091,7 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize) // all in one swoop (for each compression scheme). // - int planarUncBufferSize[NUM_COMPRESSOR_SCHEMES]; + Int64 planarUncBufferSize[NUM_COMPRESSOR_SCHEMES]; for (int i=0; i 0) { planarUncBufferSize[UNKNOWN] = - compressBound ((uLongf)planarUncBufferSize[UNKNOWN]); + compressBound (planarUncBufferSize[UNKNOWN]); } for (int i = 0; i < NUM_COMPRESSOR_SCHEMES; ++i) { - if (static_cast(planarUncBufferSize[i]) > _planarUncBufferSize[i]) + if ( planarUncBufferSize[i] > _planarUncBufferSize[i]) { _planarUncBufferSize[i] = planarUncBufferSize[i]; if (_planarUncBuffer[i] != 0) delete[] _planarUncBuffer[i]; + + if (planarUncBufferSize[i] > std::numeric_limits::max()) + { + throw IEX_NAMESPACE::ArgExc("DWA buffers too large"); + } + _planarUncBuffer[i] = new char[planarUncBufferSize[i]]; } } diff --git a/OpenEXR/IlmImf/ImfDwaCompressor.h b/OpenEXR/IlmImf/ImfDwaCompressor.h index 04550a1dac..012819c75c 100644 --- a/OpenEXR/IlmImf/ImfDwaCompressor.h +++ b/OpenEXR/IlmImf/ImfDwaCompressor.h @@ -169,16 +169,16 @@ class DwaCompressor: public Compressor std::vector _cscSets; std::vector _channelRules; - char *_packedAcBuffer; - size_t _packedAcBufferSize; - char *_packedDcBuffer; - size_t _packedDcBufferSize; - char *_rleBuffer; - size_t _rleBufferSize; - char *_outBuffer; - size_t _outBufferSize; - char *_planarUncBuffer[NUM_COMPRESSOR_SCHEMES]; - size_t _planarUncBufferSize[NUM_COMPRESSOR_SCHEMES]; + char* _packedAcBuffer; + Int64 _packedAcBufferSize; + char* _packedDcBuffer; + Int64 _packedDcBufferSize; + char* _rleBuffer; + Int64 _rleBufferSize; + char* _outBuffer; + Int64 _outBufferSize; + char* _planarUncBuffer[NUM_COMPRESSOR_SCHEMES]; + Int64 _planarUncBufferSize[NUM_COMPRESSOR_SCHEMES]; Zip *_zip; float _dwaCompressionLevel; From c6682086017123a19f2b83f0282a61ae97549948 Mon Sep 17 00:00:00 2001 From: peterhillman Date: Mon, 25 Jan 2021 21:57:08 +1300 Subject: [PATCH 07/15] prevent overflows by using Int64 for all vars in DWA initialize (#903) Signed-off-by: Peter Hillman Signed-off-by: Cary Phillips --- OpenEXR/IlmImf/ImfDwaCompressor.cpp | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/OpenEXR/IlmImf/ImfDwaCompressor.cpp b/OpenEXR/IlmImf/ImfDwaCompressor.cpp index c772c0d734..16c61889dd 100644 --- a/OpenEXR/IlmImf/ImfDwaCompressor.cpp +++ b/OpenEXR/IlmImf/ImfDwaCompressor.cpp @@ -2933,16 +2933,16 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize) // Int64 maxOutBufferSize = 0; - int numLossyDctChans = 0; + Int64 numLossyDctChans = 0; Int64 unknownBufferSize = 0; Int64 rleBufferSize = 0; - int maxLossyDctAcSize = (int)ceil ((float)numScanLines() / 8.0f) * - (int)ceil ((float)(_max[0] - _min[0] + 1) / 8.0f) * + Int64 maxLossyDctAcSize = static_cast(ceil ((float)numScanLines() / 8.0f)) * + static_cast(ceil ((float)(_max[0] - _min[0] + 1) / 8.0f)) * 63 * sizeof (unsigned short); - int maxLossyDctDcSize = (int)ceil ((float)numScanLines() / 8.0f) * - (int)ceil ((float)(_max[0] - _min[0] + 1) / 8.0f) * + Int64 maxLossyDctDcSize = static_cast(ceil ((float)numScanLines() / 8.0f)) * + static_cast(ceil ((float)(_max[0] - _min[0] + 1) / 8.0f)) * sizeof (unsigned short); Int64 pixelCount = static_cast(numScanLines()) * static_cast(_max[0] - _min[0] + 1); @@ -2961,8 +2961,8 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize) // maxOutBufferSize += std::max( - (int)(2 * maxLossyDctAcSize + 65536), - (int)compressBound (maxLossyDctAcSize) ); + 2lu * maxLossyDctAcSize + 65536lu, + static_cast(compressBound (maxLossyDctAcSize)) ); numLossyDctChans++; break; @@ -2999,13 +2999,13 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize) // which could take slightly more space // - maxOutBufferSize += compressBound (rleBufferSize); + maxOutBufferSize += static_cast(compressBound (rleBufferSize)); // // And the same goes for the UNKNOWN data // - maxOutBufferSize += compressBound (unknownBufferSize); + maxOutBufferSize += static_cast(compressBound (unknownBufferSize)); // // Allocate a zip/deflate compressor big enought to hold the DC data @@ -3052,7 +3052,7 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize) // to Huffman encoding // - if (static_cast(maxLossyDctAcSize * numLossyDctChans) > _packedAcBufferSize) + if (maxLossyDctAcSize * numLossyDctChans > _packedAcBufferSize) { _packedAcBufferSize = maxLossyDctAcSize * numLossyDctChans; if (_packedAcBuffer != 0) @@ -3064,7 +3064,7 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize) // _packedDcBuffer holds one quantized DCT coef per 8x8 block // - if (static_cast(maxLossyDctDcSize * numLossyDctChans) > _packedDcBufferSize) + if (maxLossyDctDcSize * numLossyDctChans > _packedDcBufferSize) { _packedDcBufferSize = maxLossyDctDcSize * numLossyDctChans; if (_packedDcBuffer != 0) @@ -3125,8 +3125,8 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize) if (planarUncBufferSize[UNKNOWN] > 0) { - planarUncBufferSize[UNKNOWN] = - compressBound (planarUncBufferSize[UNKNOWN]); + planarUncBufferSize[UNKNOWN] = + static_cast( compressBound (planarUncBufferSize[UNKNOWN]) ); } for (int i = 0; i < NUM_COMPRESSOR_SCHEMES; ++i) From 10893d0c531e5aad2f79e0341e1876c8a1ae470c Mon Sep 17 00:00:00 2001 From: peterhillman Date: Mon, 1 Feb 2021 10:22:39 +1300 Subject: [PATCH 08/15] update tileoffset sanitycheck to handle ripmaps (#910) * update tileoffset sanitycheck to handle ripmaps Signed-off-by: Peter Hillman * slight reorganization Signed-off-by: Peter Hillman * slight reorganization Signed-off-by: Peter Hillman * remove extra if statement from validateStreamSize Signed-off-by: Peter Hillman Signed-off-by: Cary Phillips --- OpenEXR/IlmImf/ImfTiledInputFile.cpp | 52 ++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/OpenEXR/IlmImf/ImfTiledInputFile.cpp b/OpenEXR/IlmImf/ImfTiledInputFile.cpp index 091ad623a3..ba8c96f1d4 100644 --- a/OpenEXR/IlmImf/ImfTiledInputFile.cpp +++ b/OpenEXR/IlmImf/ImfTiledInputFile.cpp @@ -299,6 +299,58 @@ TiledInputFile::Data::getTileBuffer (int number) } +// +// avoid allocating excessive memory due to large lineOffsets table size. +// If the chunktablesize claims to be large, +// check the file is big enough to contain the table before allocating memory +// in the bytesPerLineTable and the lineOffsets table. +// Attempt to read the last entry in the first level of the table. Either the seekg() or the read() +// call will throw an exception if the file is much too small to contain the table. +// + +// assumes the input stream pointer is at (or before) the beginning of the chunk table + + +void +TiledInputFile::Data::validateStreamSize() +{ + const TileDescription& td = header.tileDescription(); + Int64 chunkCount; + + if (td.mode==RIPMAP_LEVELS) + { + // use slow function to calculate exact size of ripmap + chunkCount = getTiledChunkOffsetTableSize(header); + } + else + { + // for ONE_LEVEL image, calculate exact number of tiles + // MIPMAP_LEVELS images will have roughly 1/3 more tiles than this + // but 'chunkCount' can be less than the real offset table size for a meaningful sanity check + // + const Box2i &dataWindow = header.dataWindow(); + Int64 tileWidth = td.xSize; + Int64 tileHeight = td.ySize; + + Int64 tilesX = (static_cast(dataWindow.max.x+1-dataWindow.min.x) + tileWidth -1) / tileWidth; + Int64 tilesY = (static_cast(dataWindow.max.y+1-dataWindow.min.y) + tileHeight -1) / tileHeight; + + chunkCount = tilesX*tilesY; + } + + if (chunkCount > gLargeChunkTableSize) + { + + Int64 pos = _streamData->is->tellg(); + _streamData->is->seekg(pos + (chunkCount-1)*sizeof(Int64)); + Int64 temp; + OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read (*_streamData->is, temp); + _streamData->is->seekg(pos); + } + +} + +>>>>>>> 52d2dc3a... update tileoffset sanitycheck to handle ripmaps (#910):src/lib/OpenEXR/ImfTiledInputFile.cpp namespace { void From 5ea7546808a526d779200b1fd6e9cfd2d59cfcf0 Mon Sep 17 00:00:00 2001 From: peterhillman Date: Sat, 6 Feb 2021 14:02:59 +1300 Subject: [PATCH 09/15] additional verification of DWA data sizes (#914) Signed-off-by: Peter Hillman --- OpenEXR/IlmImf/ImfDwaCompressor.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/OpenEXR/IlmImf/ImfDwaCompressor.cpp b/OpenEXR/IlmImf/ImfDwaCompressor.cpp index 16c61889dd..59d1d5d1ed 100644 --- a/OpenEXR/IlmImf/ImfDwaCompressor.cpp +++ b/OpenEXR/IlmImf/ImfDwaCompressor.cpp @@ -2606,6 +2606,14 @@ DwaCompressor::uncompress throw IEX_NAMESPACE::BaseExc("DC data corrupt."); } } + else + { + // if the compressed size is 0, then the uncompressed size must also be zero + if (totalDcUncompressedCount!=0) + { + throw IEX_NAMESPACE::BaseExc("DC data corrupt."); + } + } // // Uncompress the RLE data into _rleBuffer, then unRLE the results From 4c8d5f43e3a0ab0aceb38f957a5bbca85f8c11ce Mon Sep 17 00:00:00 2001 From: Cary Phillips Date: Fri, 5 Feb 2021 17:24:59 -0800 Subject: [PATCH 10/15] Release notes for v2.5.5 Signed-off-by: Cary Phillips --- CHANGES.md | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index a147cfcae2..eb06ad8b2d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,6 @@ # OpenEXR Release Notes +* [Version 2.5.5](#version-255-february-7-2021) February 7, 2021 * [Version 2.5.4](#version-254-december-31-2020) December 31, 2020 * [Version 2.5.3](#version-253-august-12-2020) August 12, 2020 * [Version 2.5.2](#version-252-june-15-2020) June 15, 2020 @@ -38,8 +39,34 @@ * [Version 1.0.1](#version-101) * [Version 1.0](#version-10) -## Version 2.5.4 (December 31, 2020) +## Version 2.5.5 (February 7, 2021) + +Patch release with various bug/sanitizer/security fixes, primarily +related to reading corrupted input files, but also a fix for universal +build support on macOS. + +Specific OSS-fuzz issues include: +* OSS-fuzz [#30291](https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=30291) +* OSS-fuzz [#29106](https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=29106) +* OSS-fuzz [#28971](https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=28971) +* OSS-fuzz [#29829](https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=29829) +* OSS-fuzz [#30121](https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=30121) + +### Merged Pull Requests + +* [#914](https://github.com/AcademySoftwareFoundation/openexr/pull/914) additional verification of DWA data sizes +* [#910](https://github.com/AcademySoftwareFoundation/openexr/pull/910) update tileoffset sanitycheck to handle ripmaps +* [#903](https://github.com/AcademySoftwareFoundation/openexr/pull/903) prevent overflows by using Int64 for all vars in DWA initialize +* [#901](https://github.com/AcademySoftwareFoundation/openexr/pull/901) Use size_t for DWA buffersize calculation +* [#897](https://github.com/AcademySoftwareFoundation/openexr/pull/897) prevent overflow in RgbaFile cachePadding +* [#896](https://github.com/AcademySoftwareFoundation/openexr/pull/896) add buffer size validation to FastHuf decode +* [#893](https://github.com/AcademySoftwareFoundation/openexr/pull/893) Include where required by newer compilers +* [#889](https://github.com/AcademySoftwareFoundation/openexr/pull/889) Add explicit #include for numeric_limits +* [#854](https://github.com/AcademySoftwareFoundation/openexr/pull/854) Fix Apple Universal 2 (arm64/x86_64) builds + +## Version 2.5.4 (December 31, 2020) + Patch release with various bug/sanitizer/security fixes, primarily related to reading corrupted input files. From bb6900de2936c96b9161cca0fbc1f469bce16b2a Mon Sep 17 00:00:00 2001 From: Cary Phillips Date: Fri, 5 Feb 2021 17:49:11 -0800 Subject: [PATCH 11/15] fix merge of ImfTiledInputFile.cpp Signed-off-by: Cary Phillips --- OpenEXR/IlmImf/ImfTiledInputFile.cpp | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/OpenEXR/IlmImf/ImfTiledInputFile.cpp b/OpenEXR/IlmImf/ImfTiledInputFile.cpp index ba8c96f1d4..68e0662e30 100644 --- a/OpenEXR/IlmImf/ImfTiledInputFile.cpp +++ b/OpenEXR/IlmImf/ImfTiledInputFile.cpp @@ -253,6 +253,11 @@ struct TiledInputFile::Data: public Mutex Data (Data&& other) = delete; Data& operator = (Data&& other) = delete; + static const int gLargeChunkTableSize = 1024*1024; + void validateStreamSize(); // throw an exception if the file is significantly + // smaller than the data/tile geometry would require + + inline TileBuffer * getTileBuffer (int number); // hash function from tile indices // into our vector of tile buffers @@ -299,6 +304,7 @@ TiledInputFile::Data::getTileBuffer (int number) } + // // avoid allocating excessive memory due to large lineOffsets table size. // If the chunktablesize claims to be large, @@ -350,7 +356,6 @@ TiledInputFile::Data::validateStreamSize() } ->>>>>>> 52d2dc3a... update tileoffset sanitycheck to handle ripmaps (#910):src/lib/OpenEXR/ImfTiledInputFile.cpp namespace { void @@ -661,10 +666,11 @@ TileBufferTask::execute () // The frame buffer contains a slice for this channel. // - char *writePtr = slice.base + + intptr_t base = reinterpret_cast(slice.base); + char *writePtr = reinterpret_cast(base + (y - yOffset) * slice.yStride + (tileRange.min.x - xOffset) * - slice.xStride; + slice.xStride); char *endPtr = writePtr + (numPixelsPerScanLine - 1) * slice.xStride; @@ -783,6 +789,7 @@ TiledInputFile::TiledInputFile (const char fileName[], int numThreads): _data->_streamData = new InputStreamMutex(); _data->_streamData->is = is; _data->header.readFrom (*_data->_streamData->is, _data->version); + initialize(); //read tile offsets - we are not multipart or deep _data->tileOffsets.readFrom (*(_data->_streamData->is), _data->fileIsComplete,false,false); @@ -1025,6 +1032,15 @@ TiledInputFile::initialize () _data->header.sanityCheck (true); + // + // before allocating memory for tile offsets, confirm file is large enough + // to contain tile offset table + // (for multipart files, the chunk offset table has already been read) + // + if (!isMultiPart(_data->version)) + { + _data->validateStreamSize(); + } _data->tileDesc = _data->header.tileDescription(); _data->lineOrder = _data->header.lineOrder(); From 944a7f0e39d54a435075fb827cae74b5a1e92c51 Mon Sep 17 00:00:00 2001 From: Cary Phillips Date: Sat, 6 Feb 2021 11:08:10 -0800 Subject: [PATCH 12/15] Bump version for 2.5.5 Signed-off-by: Cary Phillips --- IlmBase/configure.ac | 6 +++--- OpenEXR/configure.ac | 6 +++--- PyIlmBase/configure.ac | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/IlmBase/configure.ac b/IlmBase/configure.ac index 8519e7351c..44e8e52bec 100644 --- a/IlmBase/configure.ac +++ b/IlmBase/configure.ac @@ -4,11 +4,11 @@ dnl Copyright Contributors to the OpenEXR Project. dnl dnl Process this file with autoconf to produce a configure script. -AC_INIT(IlmBase, 2.5.4) +AC_INIT(IlmBase, 2.5.5) AC_SUBST(ILMBASE_VERSION_MAJOR, 2) AC_SUBST(ILMBASE_VERSION_MINOR, 5) -AC_SUBST(ILMBASE_VERSION_PATCH, 4) +AC_SUBST(ILMBASE_VERSION_PATCH, 5) AC_SUBST(ILMBASE_VERSION, ${ILMBASE_VERSION_MAJOR}.${ILMBASE_VERSION_MINOR}.${ILMBASE_VERSION_PATCH}) AC_SUBST(ILMBASE_VERSION_API, ${ILMBASE_VERSION_MAJOR}_${ILMBASE_VERSION_MINOR}) @@ -22,7 +22,7 @@ AM_MAINTAINER_MODE LIBTOOL_CURRENT=25 -LIBTOOL_REVISION=3 +LIBTOOL_REVISION=4 LIBTOOL_AGE=0 LIBTOOL_VERSION=$LIBTOOL_CURRENT:$LIBTOOL_REVISION:$LIBTOOL_AGE AC_SUBST(LIBTOOL_VERSION) diff --git a/OpenEXR/configure.ac b/OpenEXR/configure.ac index 1ea781872a..1e266c3a44 100644 --- a/OpenEXR/configure.ac +++ b/OpenEXR/configure.ac @@ -5,12 +5,12 @@ dnl dnl Process this file with autoconf to produce a configure script. -AC_INIT(OpenEXR, 2.5.4) +AC_INIT(OpenEXR, 2.5.5) AC_CONFIG_MACRO_DIR([m4]) AC_SUBST(OPENEXR_VERSION_MAJOR, 2) AC_SUBST(OPENEXR_VERSION_MINOR, 5) -AC_SUBST(OPENEXR_VERSION_PATCH, 4) +AC_SUBST(OPENEXR_VERSION_PATCH, 5) AC_SUBST(OPENEXR_VERSION, ${OPENEXR_VERSION_MAJOR}.${OPENEXR_VERSION_MINOR}.${OPENEXR_VERSION_PATCH}) AC_SUBST(OPENEXR_VERSION_API, ${OPENEXR_VERSION_MAJOR}_${OPENEXR_VERSION_MINOR}) @@ -24,7 +24,7 @@ AM_MAINTAINER_MODE LIBTOOL_CURRENT=25 -LIBTOOL_REVISION=3 +LIBTOOL_REVISION=4 LIBTOOL_AGE=0 LIBTOOL_VERSION=$LIBTOOL_CURRENT:$LIBTOOL_REVISION:$LIBTOOL_AGE AC_SUBST(LIBTOOL_VERSION) diff --git a/PyIlmBase/configure.ac b/PyIlmBase/configure.ac index c73fd0c2e1..19519a6d70 100644 --- a/PyIlmBase/configure.ac +++ b/PyIlmBase/configure.ac @@ -4,7 +4,7 @@ dnl Copyright Contributors to the OpenEXR Project. dnl dnl Process this file with autoconf to produce a configure script. -AC_INIT(PyIlmBase, 2.5.4) +AC_INIT(PyIlmBase, 2.5.5) AC_SUBST(PYILMBASE_VERSION, 2.5.4) AC_CANONICAL_HOST AC_CONFIG_SRCDIR(PyIex/iexmodule.cpp) @@ -17,7 +17,7 @@ AC_DISABLE_STATIC LIBTOOL_CURRENT=25 -LIBTOOL_REVISION=3 +LIBTOOL_REVISION=4 LIBTOOL_AGE=0 LIBTOOL_VERSION=$LIBTOOL_CURRENT:$LIBTOOL_REVISION:$LIBTOOL_AGE AC_SUBST(LIBTOOL_VERSION) From 153bf778818117560272e6897e9665e1482e2486 Mon Sep 17 00:00:00 2001 From: Cary Phillips Date: Wed, 10 Feb 2021 13:53:17 -0800 Subject: [PATCH 13/15] Only wait for and join joinable threads (#921) Signed-off-by: Cary Phillips --- IlmBase/IlmThread/IlmThread.cpp | 14 +++++++++++++- IlmBase/IlmThread/IlmThread.h | 6 ++++++ IlmBase/IlmThread/IlmThreadPool.cpp | 11 +++++++++-- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/IlmBase/IlmThread/IlmThread.cpp b/IlmBase/IlmThread/IlmThread.cpp index fe58169a98..1d52588770 100644 --- a/IlmBase/IlmThread/IlmThread.cpp +++ b/IlmBase/IlmThread/IlmThread.cpp @@ -73,6 +73,12 @@ Thread::~Thread () } +bool +Thread::joinable() const +{ + return _thread.joinable(); +} + void Thread::start () { @@ -108,7 +114,13 @@ Thread::start () { throw IEX_NAMESPACE::NoImplExc ("Threads not supported on this platform."); } -# endif + +bool +Thread::joinable () const +{ + throw IEX_NAMESPACE::NoImplExc ("Threads not supported / enabled on this platform."); +} + #endif diff --git a/IlmBase/IlmThread/IlmThread.h b/IlmBase/IlmThread/IlmThread.h index 32e6152cf3..04ce60be3c 100644 --- a/IlmBase/IlmThread/IlmThread.h +++ b/IlmBase/IlmThread/IlmThread.h @@ -129,6 +129,12 @@ class Thread ILMTHREAD_EXPORT void start (); ILMTHREAD_EXPORT virtual void run () = 0; + // + // wait for thread to exit - must be called before deleting thread + // + void join(); + bool joinable() const; + private: #ifdef ILMBASE_FORCE_CXX03 diff --git a/IlmBase/IlmThread/IlmThreadPool.cpp b/IlmBase/IlmThread/IlmThreadPool.cpp index 43edc118d8..7c9a217db8 100644 --- a/IlmBase/IlmThread/IlmThreadPool.cpp +++ b/IlmBase/IlmThread/IlmThreadPool.cpp @@ -423,14 +423,21 @@ DefaultThreadPoolProvider::finish () size_t curT = _data.threads.size(); for (size_t i = 0; i != curT; ++i) { - _data.taskSemaphore.post(); - _data.threadSemaphore.wait(); + if (_data.threads[i]->joinable()) + { + _data.taskSemaphore.post(); + _data.threadSemaphore.wait(); + } } // // Join all the threads // for (size_t i = 0; i != curT; ++i) + { + if (_data.threads[i]->joinable()) + _data.threads[i]->join(); + delete _data.threads[i]; Lock lock1 (_data.taskMutex); From 7a62ffdcb62e46eb7db0d57c385f2be1a5652f67 Mon Sep 17 00:00:00 2001 From: Cary Phillips Date: Wed, 10 Feb 2021 14:35:18 -0800 Subject: [PATCH 14/15] Fixed botched merge or IlmThread.cpp/IlmThreadPool.cpp Signed-off-by: Cary Phillips --- IlmBase/IlmThread/IlmThread.cpp | 13 +++++++++++++ IlmBase/IlmThread/IlmThreadPool.cpp | 3 ++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/IlmBase/IlmThread/IlmThread.cpp b/IlmBase/IlmThread/IlmThread.cpp index 1d52588770..fd2b4f38fd 100644 --- a/IlmBase/IlmThread/IlmThread.cpp +++ b/IlmBase/IlmThread/IlmThread.cpp @@ -72,6 +72,12 @@ Thread::~Thread () _thread.join (); } +void +Thread::join() +{ + if ( _thread.joinable () ) + _thread.join (); +} bool Thread::joinable() const @@ -115,12 +121,19 @@ Thread::start () throw IEX_NAMESPACE::NoImplExc ("Threads not supported on this platform."); } +void +Thread::join () +{ + throw IEX_NAMESPACE::NoImplExc ("Threads not supported / enabled on this platform."); +} + bool Thread::joinable () const { throw IEX_NAMESPACE::NoImplExc ("Threads not supported / enabled on this platform."); } +# endif #endif diff --git a/IlmBase/IlmThread/IlmThreadPool.cpp b/IlmBase/IlmThread/IlmThreadPool.cpp index 7c9a217db8..1ff54ca087 100644 --- a/IlmBase/IlmThread/IlmThreadPool.cpp +++ b/IlmBase/IlmThread/IlmThreadPool.cpp @@ -439,7 +439,8 @@ DefaultThreadPoolProvider::finish () _data.threads[i]->join(); delete _data.threads[i]; - + } + Lock lock1 (_data.taskMutex); #ifdef ILMBASE_FORCE_CXX03 Lock lock2 (_data.stopMutex); From 1ae85cada62fbea95ea726eb79511b8e906c27b4 Mon Sep 17 00:00:00 2001 From: Cary Phillips Date: Fri, 12 Feb 2021 08:49:16 -0800 Subject: [PATCH 15/15] Fix 2.5.5 release date Signed-off-by: Cary Phillips --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index eb06ad8b2d..3964157b80 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,6 @@ # OpenEXR Release Notes -* [Version 2.5.5](#version-255-february-7-2021) February 7, 2021 +* [Version 2.5.5](#version-255-february-12-2021) February 12, 2021 * [Version 2.5.4](#version-254-december-31-2020) December 31, 2020 * [Version 2.5.3](#version-253-august-12-2020) August 12, 2020 * [Version 2.5.2](#version-252-june-15-2020) June 15, 2020 @@ -39,7 +39,7 @@ * [Version 1.0.1](#version-101) * [Version 1.0](#version-10) -## Version 2.5.5 (February 7, 2021) +## Version 2.5.5 (February 12, 2021) Patch release with various bug/sanitizer/security fixes, primarily related to reading corrupted input files, but also a fix for universal