From 1b72a1bca052ef9886725ba9bc6ef6c40cd14707 Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Wed, 23 Oct 2024 16:20:15 -0500 Subject: [PATCH] Add support for native zlib-ng in the library and compression references (#4935) * Correct compile defs for zip_perf program --- CMakeFilters.cmake | 10 +++++- config/cmake/H5pubconf.h.in | 3 ++ config/cmake/HDFLibMacros.cmake | 5 +++ config/cmake/ZLIBNG/CMakeLists.txt | 4 +-- release_docs/RELEASE.txt | 6 ++++ src/CMakeLists.txt | 6 ++-- src/H5Zdeflate.c | 43 +++++++++++++++++++++++--- test/CMakeLists.txt | 13 ++++++++ test/chunk_info.c | 11 ++++++- test/direct_chunk.c | 22 +++++++++++-- tools/test/perform/CMakeLists.txt | 6 ++++ tools/test/perform/direct_write_perf.c | 11 ++++++- tools/test/perform/zip_perf.c | 12 +++++-- 13 files changed, 136 insertions(+), 16 deletions(-) diff --git a/CMakeFilters.cmake b/CMakeFilters.cmake index 9f4955c3d52..3eca6e743c8 100644 --- a/CMakeFilters.cmake +++ b/CMakeFilters.cmake @@ -101,7 +101,11 @@ if (HDF5_ENABLE_Z_LIB_SUPPORT) endif () set(H5_ZLIB_FOUND ${ZLIB_FOUND}) if (H5_ZLIB_FOUND) - set (H5_ZLIB_HEADER "zlib.h") + if (HDF5_USE_ZLIB_NG) + set (H5_ZLIB_HEADER "zlib-ng.h") + else () + set (H5_ZLIB_HEADER "zlib.h") + endif () set (H5_ZLIB_INCLUDE_DIR_GEN ${ZLIB_INCLUDE_DIR}) set (H5_ZLIB_INCLUDE_DIRS ${H5_ZLIB_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIR}) # The FindZLIB.cmake module does not set an OUTPUT_NAME @@ -125,6 +129,9 @@ if (HDF5_ENABLE_Z_LIB_SUPPORT) if (H5_ZLIB_FOUND) set (H5_HAVE_FILTER_DEFLATE 1) set (H5_HAVE_ZLIB_H 1) + if (HDF5_USE_ZLIB_NG) + set (H5_HAVE_ZLIBNG_H 1) + endif () set (H5_HAVE_LIBZ 1) if (H5_HAVE_FILTER_DEFLATE) set (EXTERNAL_FILTERS "${EXTERNAL_FILTERS} DEFLATE") @@ -135,6 +142,7 @@ if (HDF5_ENABLE_Z_LIB_SUPPORT) set (HDF5_ENABLE_Z_LIB_SUPPORT OFF CACHE BOOL "" FORCE) message (WARNING " ZLib support in HDF5 was enabled but not found") endif () + message(STATUS "H5_ZLIB_HEADER=${H5_ZLIB_HEADER}") endif () #----------------------------------------------------------------------------- diff --git a/config/cmake/H5pubconf.h.in b/config/cmake/H5pubconf.h.in index 910bd129227..822875285cf 100644 --- a/config/cmake/H5pubconf.h.in +++ b/config/cmake/H5pubconf.h.in @@ -366,6 +366,9 @@ /* Define to 1 if you have the header file. */ #cmakedefine H5_HAVE_ZLIB_H @H5_HAVE_ZLIB_H@ +/* Define to 1 if you have the header file. */ +#cmakedefine H5_HAVE_ZLIBNG_H @H5_HAVE_ZLIBNG_H@ + /* Define to 1 if you have the `_getvideoconfig' function. */ #cmakedefine H5_HAVE__GETVIDEOCONFIG @H5_HAVE__GETVIDEOCONFIG@ diff --git a/config/cmake/HDFLibMacros.cmake b/config/cmake/HDFLibMacros.cmake index 1907931bb34..f85fb42c51b 100644 --- a/config/cmake/HDFLibMacros.cmake +++ b/config/cmake/HDFLibMacros.cmake @@ -40,6 +40,11 @@ macro (EXTERNAL_ZLIB_LIBRARY compress_type) add_library(${HDF_PACKAGE_NAMESPACE}zlib-static ALIAS zlib-static) set (H5_ZLIB_STATIC_LIBRARY "${HDF_PACKAGE_NAMESPACE}zlib-static") set (H5_ZLIB_LIBRARIES ${H5_ZLIB_STATIC_LIBRARY}) + if (HDF5_USE_ZLIB_NG) + set (H5_ZLIB_HEADER "zlib-ng.h") + else () + set (H5_ZLIB_HEADER "zlib.h") + endif () set (H5_ZLIB_INCLUDE_DIR_GEN "${hdf5_zlib_BINARY_DIR}") set (H5_ZLIB_INCLUDE_DIR "${hdf5_zlib_SOURCE_DIR}") diff --git a/config/cmake/ZLIBNG/CMakeLists.txt b/config/cmake/ZLIBNG/CMakeLists.txt index 5bb5da961e1..a5e26e18bee 100644 --- a/config/cmake/ZLIBNG/CMakeLists.txt +++ b/config/cmake/ZLIBNG/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.18) -project(ZLIB VERSION ${ZLIB_HEADER_VERSION} LANGUAGES C) +project(ZLIB LANGUAGES C) message(STATUS "Using CMake version ${CMAKE_VERSION}") @@ -98,7 +98,7 @@ endif() # Options parsing # option(WITH_GZFILEOP "Compile with support for gzFile related functions" ON) -option(ZLIB_COMPAT "Compile with zlib compatible API" ON) +option(ZLIB_COMPAT "Compile with zlib compatible API" OFF) option(ZLIB_ENABLE_TESTS "Build test binaries" OFF) option(ZLIBNG_ENABLE_TESTS "Test zlib-ng specific API" OFF) option(WITH_GTEST "Build gtest_zlib" OFF) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index e51bc71014d..19e8e59c1fc 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -47,6 +47,12 @@ New Features Configuration: ------------- + - Added support for native zlib-ng compression. + + Changed the zlib-ng CMake logic to prefer the native zlib-ng library. Added + #ifdef around the compression function calls. Added including the correct + header file with the same #ifdef. + - Renamed remaining HDF5 library CMake options except for CMake BUILD* variables DEFAULT_API_VERSION to HDF5_DEFAULT_API_VERSION diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5ce3b7cac18..3db8991df8c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -727,8 +727,10 @@ set (H5Z_SOURCES ${HDF5_SRC_DIR}/H5Ztrans.c ) if (H5_ZLIB_HEADER) - SET_PROPERTY(SOURCE ${HDF5_SRC_DIR}/H5Zdeflate.c PROPERTY - COMPILE_DEFINITIONS H5_ZLIB_HEADER="${H5_ZLIB_HEADER}") + message(STATUS "H5_ZLIB_HEADER=${H5_ZLIB_HEADER}") + set_source_files_properties(${HDF5_SRC_DIR}/H5Zdeflate.c + PROPERTIES COMPILE_DEFINITIONS H5_ZLIB_HEADER="${H5_ZLIB_HEADER}" + ) endif () diff --git a/src/H5Zdeflate.c b/src/H5Zdeflate.c index def170e9f99..7ac4d09b98c 100644 --- a/src/H5Zdeflate.c +++ b/src/H5Zdeflate.c @@ -74,8 +74,12 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[] if (flags & H5Z_FLAG_REVERSE) { /* Input; uncompress */ - z_stream z_strm; /* zlib parameters */ - size_t nalloc = *buf_size; /* Number of bytes for output (compressed) buffer */ +#if defined(H5_HAVE_ZLIBNG_H) + zng_stream z_strm; /* zlib parameters */ +#else + z_stream z_strm; /* zlib parameters */ +#endif + size_t nalloc = *buf_size; /* Number of bytes for output (compressed) buffer */ /* Allocate space for the compressed buffer */ if (NULL == (outbuf = H5MM_malloc(nalloc))) @@ -89,13 +93,22 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[] H5_CHECKED_ASSIGN(z_strm.avail_out, unsigned, nalloc, size_t); /* Initialize the uncompression routines */ +#if defined(H5_HAVE_ZLIBNG_H) + if (Z_OK != zng_inflateInit(&z_strm)) + HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflateInit() failed"); +#else if (Z_OK != inflateInit(&z_strm)) HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflateInit() failed"); +#endif /* Loop to uncompress the buffer */ do { /* Uncompress some data */ +#if defined(H5_HAVE_ZLIBNG_H) + status = zng_inflate(&z_strm, Z_SYNC_FLUSH); +#else status = inflate(&z_strm, Z_SYNC_FLUSH); +#endif /* Check if we are done uncompressing data */ if (Z_STREAM_END == status) @@ -103,7 +116,11 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[] /* Check for error */ if (Z_OK != status) { +#if defined(H5_HAVE_ZLIBNG_H) + (void)zng_inflateEnd(&z_strm); +#else (void)inflateEnd(&z_strm); +#endif HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflate() failed"); } else { @@ -114,7 +131,11 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[] /* Allocate a buffer twice as big */ nalloc *= 2; if (NULL == (new_outbuf = H5MM_realloc(outbuf, nalloc))) { +#if defined(H5_HAVE_ZLIBNG_H) + (void)zng_inflateEnd(&z_strm); +#else (void)inflateEnd(&z_strm); +#endif HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for deflate uncompression"); } /* end if */ @@ -137,7 +158,11 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[] ret_value = z_strm.total_out; /* Finish uncompressing the stream */ +#if defined(H5_HAVE_ZLIBNG_H) + (void)zng_inflateEnd(&z_strm); +#else (void)inflateEnd(&z_strm); +#endif } /* end if */ else { /* @@ -147,9 +172,13 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[] */ const Bytef *z_src = (const Bytef *)(*buf); Bytef *z_dst; /*destination buffer */ - uLongf z_dst_nbytes = (uLongf)compressBound(nbytes); - uLong z_src_nbytes = (uLong)nbytes; - int aggression; /* Compression aggression setting */ +#if defined(H5_HAVE_ZLIBNG_H) + uLongf z_dst_nbytes = (uLongf)zng_compressBound(nbytes); +#else + uLongf z_dst_nbytes = (uLongf)compressBound(nbytes); +#endif + uLong z_src_nbytes = (uLong)nbytes; + int aggression; /* Compression aggression setting */ /* Set the compression aggression level */ H5_CHECKED_ASSIGN(aggression, int, cd_values[0], unsigned); @@ -160,7 +189,11 @@ H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[] z_dst = (Bytef *)outbuf; /* Perform compression from the source to the destination buffer */ +#if defined(H5_HAVE_ZLIBNG_H) + status = zng_compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression); +#else status = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression); +#endif /* Check for various zlib errors */ if (Z_BUF_ERROR == status) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8c9b3073b9d..792b5830003 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -483,6 +483,12 @@ endforeach () ############################################################################## ######### Also special handling of link libs ############# #-- Adding test for chunk_info +if (H5_ZLIB_HEADER) + message(STATUS "H5_ZLIB_HEADER=${H5_ZLIB_HEADER}") + set_source_files_properties(${HDF5_TEST_SOURCE_DIR}/chunk_info.c + PROPERTIES COMPILE_DEFINITIONS H5_ZLIB_HEADER="${H5_ZLIB_HEADER}" + ) +endif () add_executable (chunk_info ${HDF5_TEST_SOURCE_DIR}/chunk_info.c) target_compile_options(chunk_info PRIVATE "${HDF5_CMAKE_C_FLAGS}") target_compile_definitions(chunk_info PRIVATE "${HDF5_TEST_COMPILE_DEFS_PRIVATE}") @@ -504,6 +510,13 @@ if (HDF5_ENABLE_FORMATTERS) endif () #-- Adding test for direct_chunk +if (H5_ZLIB_HEADER) + message(STATUS "H5_ZLIB_HEADER=${H5_ZLIB_HEADER}") + + set_source_files_properties(${HDF5_TEST_SOURCE_DIR}/direct_chunk.c + PROPERTIES COMPILE_DEFINITIONS H5_ZLIB_HEADER="${H5_ZLIB_HEADER}" + ) +endif () add_executable (direct_chunk ${HDF5_TEST_SOURCE_DIR}/direct_chunk.c) target_compile_options(direct_chunk PRIVATE "${HDF5_CMAKE_C_FLAGS}") target_compile_definitions(direct_chunk PRIVATE "${HDF5_TEST_COMPILE_DEFS_PRIVATE}") diff --git a/test/chunk_info.c b/test/chunk_info.c index 1bae1fa2724..3da2d9d258e 100644 --- a/test/chunk_info.c +++ b/test/chunk_info.c @@ -40,7 +40,12 @@ #include "h5test.h" #ifdef H5_HAVE_FILTER_DEFLATE -#include "zlib.h" +#if defined(H5_HAVE_ZLIB_H) && !defined(H5_ZLIB_HEADER) +#define H5_ZLIB_HEADER "zlib.h" +#endif +#if defined(H5_ZLIB_HEADER) +#include H5_ZLIB_HEADER /* "zlib.h" */ +#endif #endif /* Test file names, using H5F_libver_t as indices */ @@ -530,7 +535,11 @@ test_get_chunk_info_highest_v18(hid_t fapl) z_dst = (Bytef *)inbuf; /* Perform compression from the source to the destination buffer */ +#if defined(H5_HAVE_ZLIBNG_H) + ret = zng_compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression); +#else ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression); +#endif /* Set the chunk size to the compressed chunk size */ chunk_size = (hsize_t)z_dst_nbytes; diff --git a/test/direct_chunk.c b/test/direct_chunk.c index ca9825866f1..9afdb47e95b 100644 --- a/test/direct_chunk.c +++ b/test/direct_chunk.c @@ -12,12 +12,14 @@ #include "h5test.h" +#ifdef H5_HAVE_FILTER_DEFLATE #if defined(H5_HAVE_ZLIB_H) && !defined(H5_ZLIB_HEADER) #define H5_ZLIB_HEADER "zlib.h" #endif #if defined(H5_ZLIB_HEADER) #include H5_ZLIB_HEADER /* "zlib.h" */ #endif +#endif #define FILE_NAME "direct_chunk.h5" @@ -205,7 +207,11 @@ test_direct_chunk_write(hid_t file) z_dst = (Bytef *)outbuf; /* Perform compression from the source to the destination buffer */ +#if defined(H5_HAVE_ZLIBNG_H) + ret = zng_compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression); +#else ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression); +#endif /* Check for various zlib errors */ if (Z_BUF_ERROR == ret) { @@ -284,7 +290,11 @@ test_direct_chunk_write(hid_t file) z_dst = (Bytef *)outbuf; /* Perform compression from the source to the destination buffer */ +#if defined(H5_HAVE_ZLIBNG_H) + ret = zng_compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression); +#else ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression); +#endif /* Check for various zlib errors */ if (Z_BUF_ERROR == ret) { @@ -1522,8 +1532,12 @@ test_direct_chunk_read_no_cache(hid_t file) if (filter_mask != 0) goto error; - /* Perform decompression from the source to the destination buffer */ + /* Perform decompression from the source to the destination buffer */ +#if defined(H5_HAVE_ZLIBNG_H) + ret = zng_uncompress(z_dst, &z_dst_nbytes, z_src, z_src_nbytes); +#else ret = uncompress(z_dst, &z_dst_nbytes, z_src, z_src_nbytes); +#endif /* Check for various zlib errors */ if (Z_BUF_ERROR == ret) { @@ -1713,8 +1727,12 @@ test_direct_chunk_read_cache(hid_t file, bool flush) if (filter_mask != 0) goto error; - /* Perform decompression from the source to the destination buffer */ + /* Perform decompression from the source to the destination buffer */ +#if defined(H5_HAVE_ZLIBNG_H) + ret = zng_uncompress(z_dst, &z_dst_nbytes, z_src, z_src_nbytes); +#else ret = uncompress(z_dst, &z_dst_nbytes, z_src, z_src_nbytes); +#endif /* Check for various zlib errors */ if (Z_BUF_ERROR == ret) { diff --git a/tools/test/perform/CMakeLists.txt b/tools/test/perform/CMakeLists.txt index 3a3bc73aa19..646bd1a6de2 100644 --- a/tools/test/perform/CMakeLists.txt +++ b/tools/test/perform/CMakeLists.txt @@ -112,6 +112,12 @@ endif () set (zip_perf_SOURCES ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/zip_perf.c ) +if (H5_ZLIB_HEADER) + message(STATUS "H5_ZLIB_HEADER=${H5_ZLIB_HEADER}") + if (HDF5_USE_ZLIB_NG) + add_compile_definitions(H5_HAVE_ZLIBNG_H=1 H5_ZLIB_HEADER="${H5_ZLIB_HEADER}") + endif () +endif () add_executable (zip_perf ${zip_perf_SOURCES}) target_include_directories (zip_perf PRIVATE "${HDF5_TEST_SRC_DIR};${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") if (NOT BUILD_SHARED_LIBS) diff --git a/tools/test/perform/direct_write_perf.c b/tools/test/perform/direct_write_perf.c index a24710fa3e7..ac206b3993d 100644 --- a/tools/test/perform/direct_write_perf.c +++ b/tools/test/perform/direct_write_perf.c @@ -18,7 +18,12 @@ #include "hdf5.h" #ifdef H5_HAVE_FILTER_DEFLATE -#include +#if defined(H5_HAVE_ZLIB_H) && !defined(H5_ZLIB_HEADER) +#define H5_ZLIB_HEADER "zlib.h" +#endif +#if defined(H5_ZLIB_HEADER) +#include H5_ZLIB_HEADER /* "zlib.h" */ +#endif #if !defined(WIN32) && !defined(__MINGW32__) @@ -242,7 +247,11 @@ create_file(hid_t fapl_id) z_dst = (Bytef *)outbuf[i]; /* Perform compression from the source to the destination buffer */ +#if defined(H5_HAVE_ZLIBNG_H) + ret = zng_compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression); +#else ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression); +#endif data_size[i] = (size_t)z_dst_nbytes; total_size += data_size[i]; diff --git a/tools/test/perform/zip_perf.c b/tools/test/perform/zip_perf.c index 78f62f80c60..f5680589364 100644 --- a/tools/test/perform/zip_perf.c +++ b/tools/test/perform/zip_perf.c @@ -24,8 +24,12 @@ #include "h5tools_utils.h" #ifdef H5_HAVE_FILTER_DEFLATE - -#include +#if defined(H5_HAVE_ZLIB_H) && !defined(H5_ZLIB_HEADER) +#define H5_ZLIB_HEADER "zlib.h" +#endif +#if defined(H5_ZLIB_HEADER) +#include H5_ZLIB_HEADER /* "zlib.h" */ +#endif #define ONE_KB 1024 #define ONE_MB (ONE_KB * ONE_KB) @@ -163,7 +167,11 @@ write_file(Bytef *source, uLongf sourceLen) static void compress_buffer(Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen) { +#if defined(H5_HAVE_ZLIBNG_H) + int rc = zng_compress2(dest, destLen, source, sourceLen, compress_level); +#else int rc = compress2(dest, destLen, source, sourceLen, compress_level); +#endif if (rc != Z_OK) { /* compress2 failed - cleanup and tell why */