Skip to content

Commit

Permalink
Merge pull request #87 from Intel-HLS/pv_update_compression_2.12ISAL
Browse files Browse the repository at this point in the history
Update to ISAL_2.21
  • Loading branch information
pnvaidya authored Feb 6, 2018
2 parents c174e37 + 122eb07 commit 51253aa
Show file tree
Hide file tree
Showing 93 changed files with 5,663 additions and 5,647 deletions.
55 changes: 22 additions & 33 deletions src/main/native/compression/IntelDeflater.cc
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ JNIEXPORT void JNICALL Java_com_intel_gkl_compression_IntelDeflater_resetNative
jint level = env->GetIntField(obj, FID_level);


if(level == 1) {
if(level == 1 || level ==2) {
isal_zstream* lz_stream = (isal_zstream*)env->GetLongField(obj, FID_lz_stream);

if (lz_stream == 0) {
Expand All @@ -95,19 +95,27 @@ JNIEXPORT void JNICALL Java_com_intel_gkl_compression_IntelDeflater_resetNative
}

env->SetLongField(obj, FID_lz_stream, (jlong)lz_stream);

isal_deflate_stateless_init(lz_stream);
lz_stream->hufftables = 0x0;

lz_stream->level = level;
lz_stream->level_buf = (uint8_t*)malloc(ISAL_DEF_LVL2_DEFAULT);
lz_stream->level_buf_size = ISAL_DEF_LVL2_DEFAULT;


}
else {
isal_hufftables *temp_huffman_pointer = lz_stream->hufftables;

DBG("lz_stream = 0x%lx", (long)temp_huffman_pointer);
isal_deflate_stateless_init(lz_stream);
lz_stream->hufftables = temp_huffman_pointer;

}

uint8_t *level_buf = NULL;
lz_stream->level = level;
jint level_size =ISAL_DEF_LVL2_DEFAULT;
level_buf = (uint8_t*)malloc(level_size);
lz_stream->level_buf = level_buf;
lz_stream->level_buf_size = ISAL_DEF_LVL2_DEFAULT;
lz_stream->end_of_stream = 0;

// DBG("lz_stream = 0x%lx", (long)lz_stream);
Expand Down Expand Up @@ -190,7 +198,7 @@ JNIEXPORT jint JNICALL Java_com_intel_gkl_compression_IntelDeflater_deflateNativ
jint level = env->GetIntField(obj, FID_level);


if(level == 1) {
if(level == 1 || level ==2 ) {

isal_zstream* lz_stream = (isal_zstream*)env->GetLongField(obj, FID_lz_stream);

Expand All @@ -211,27 +219,6 @@ JNIEXPORT jint JNICALL Java_com_intel_gkl_compression_IntelDeflater_deflateNativ
struct timeval tv1, tv2;
gettimeofday(&tv1, NULL);
#endif
// compress and update lz_stream state
// Generate the dynamic huff tables using the first buffer of the stream
if(lz_stream->hufftables == 0x0)
{
struct isal_huff_histogram histogram;
struct isal_hufftables *hufftables_custom;

hufftables_custom = (isal_hufftables*) malloc(sizeof(isal_hufftables));

int sixtyfourK = 64*1024;
int usable_buffer= (inputBufferLength < sixtyfourK) ? inputBufferLength : sixtyfourK;
DBG("lz_stream = 0x%lx", (long)hufftables_custom);

memset(&histogram, 0, sizeof(histogram));
isal_update_histogram((unsigned char*)next_in,usable_buffer, &histogram);
isal_create_hufftables(hufftables_custom, &histogram);
isal_deflate_set_hufftables(lz_stream,
hufftables_custom, IGZIP_HUFFTABLE_CUSTOM);
lz_stream->hufftables = hufftables_custom;
DBG("lz_stream = 0x%lx", (long)hufftables_custom);
}

// compress and update lz_stream state
isal_deflate_stateless(lz_stream);
Expand Down Expand Up @@ -313,15 +300,17 @@ JNIEXPORT void JNICALL
Java_com_intel_gkl_compression_IntelDeflater_endNative(JNIEnv *env, jobject obj)
{
jint level = env->GetIntField(obj, FID_level);
if (level == 1 || level == 2 ) {
isal_zstream* lz_stream = (isal_zstream*)env->GetLongField(obj, FID_lz_stream);
free(lz_stream->level_buf);
free(lz_stream);

if (level != 1) {
}
else {
z_stream* lz_stream = (z_stream*)env->GetLongField(obj, FID_lz_stream);
deflateEnd(lz_stream);
free(lz_stream);
}
else {
isal_zstream* lz_stream = (isal_zstream*)env->GetLongField(obj, FID_lz_stream);
free(lz_stream->hufftables);
free(lz_stream);
}


}
90 changes: 81 additions & 9 deletions src/main/native/compression/isa-l-master/.travis.yml
Original file line number Diff line number Diff line change
@@ -1,11 +1,83 @@
language: c
sudo: required
dist: trusty
matrix:
include:
### OS X
- os: osx
env: C_COMPILER=clang

### linux gcc
- dist: trusty
env: C_COMPILER=gcc

### linux clang
- dist: trusty
env: C_COMPILER=clang

### linux newer clang
- dist: trusty
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-trusty-4.0
packages:
- clang-4.0
env: C_COMPILER=clang-4.0

### linux older gcc
- dist: trusty
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- g++-4.7
env: C_COMPILER=gcc-4.7

### linux newer gcc
- dist: trusty
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- g++-6
env: C_COMPILER=gcc-6

### linux with new nasm
# Removed until travis issue fixed pulling from nasm from debian
#- dist: trusty
# addons:
# apt:
# sources:
# - debian-sid
# packages:
# - nasm
# env: C_COMPILER=gcc AS_ASSEMBL=nasm

### linux extended tests
- dist: trusty
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- binutils-mingw-w64-x86-64
- gcc-mingw-w64-x86-64
- wine
env: TEST_TYPE=ext

before_install:
- if [ -n "${C_COMPILER}" ]; then export CC="${C_COMPILER}"; fi
- if [ -n "${AS_ASSEMBL}" ]; then export AS="${AS_ASSEMBL}"; fi

before_script:
- sudo apt-get -q update
- sudo apt-get install -y yasm nasm
- ./autogen.sh
script: ./configure && make && make check
language: c
compiler:
- clang
- gcc
- if [ $TRAVIS_OS_NAME = linux ]; then sudo apt-get -q update; fi
- if [ $TRAVIS_OS_NAME = linux ]; then sudo apt-get install yasm indent; fi
- if [ $TRAVIS_OS_NAME = osx ]; then brew install yasm; fi

script:
- if [ -n "${CC}" ]; then $CC --version; fi
- if [ -n "${AS}" ]; then $AS --version; fi
- ./tools/test_autorun.sh "${TEST_TYPE}"
4 changes: 4 additions & 0 deletions src/main/native/compression/isa-l-master/CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@ the included indent script to format C code.

./tools/iindent your_files.c

And use check format script before submitting.

./tools/check_format.sh

[mailing list]:https://lists.01.org/mailman/listinfo/isal
[license]:LICENSE
[signed-off-by language]:https://01.org/community/signed-process
3 changes: 2 additions & 1 deletion src/main/native/compression/isa-l-master/Makefile.am
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,11 @@ include erasure_code/Makefile.am
include raid/Makefile.am
include crc/Makefile.am
include igzip/Makefile.am
include tests/fuzz/Makefile.am

# LIB version info not necessarily the same as package version
LIBISAL_CURRENT=2
LIBISAL_REVISION=19
LIBISAL_REVISION=21
LIBISAL_AGE=0

lib_LTLIBRARIES = libisal.la
Expand Down
11 changes: 7 additions & 4 deletions src/main/native/compression/isa-l-master/Makefile.nmake
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ objs = \
bin\xor_gen_sse.obj \
bin\crc16_t10dif_01.obj \
bin\crc16_t10dif_by4.obj \
bin\crc16_t10dif_copy_by4.obj \
bin\crc32_gzip.obj \
bin\crc32_ieee_01.obj \
bin\crc32_ieee_by4.obj \
Expand Down Expand Up @@ -120,9 +121,7 @@ objs = \
bin\encode_df.obj \
bin\encode_df_04.obj \
bin\proc_heap.obj \
bin\igzip_icf_body_01.obj \
bin\igzip_icf_body_02.obj \
bin\igzip_icf_body_04.obj \
bin\igzip_icf_body_h1_gr_bt.obj \
bin\igzip_icf_finish.obj \
bin\igzip_icf_base.obj \
bin\igzip_inflate.obj \
Expand All @@ -135,7 +134,10 @@ objs = \
bin\crc32_gzip_refl_by8.obj \
bin\adler32_sse.obj \
bin\adler32_avx2_4.obj \
bin\igzip_deflate_hash.obj
bin\igzip_deflate_hash.obj \
bin\igzip_gen_icf_map_lh1_06.obj \
bin\igzip_set_long_icf_fg_06.obj \
bin\igzip_icf_body.obj

INCLUDES = -I./ -Ierasure_code/ -Iraid/ -Icrc/ -Iigzip/ -Iinclude/
LINKFLAGS = /nologo
Expand Down Expand Up @@ -201,6 +203,7 @@ checks = \
xor_check_test.exe \
pq_check_test.exe \
crc16_t10dif_test.exe \
crc16_t10dif_copy_test.exe \
crc32_ieee_test.exe \
crc32_iscsi_test.exe \
crc64_funcs_test.exe \
Expand Down
6 changes: 5 additions & 1 deletion src/main/native/compression/isa-l-master/Makefile.unx
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,13 @@ default: lib

include $(foreach unit,$(units), $(unit)/Makefile.am)

ifneq (,$(findstring igzip,$(units)))
include tests/fuzz/Makefile.am
endif

# Override individual lib names to make one inclusive library.
lib_name := bin/isa-l.a

include make.inc

VPATH = . $(units) include
VPATH = . $(units) include tests/fuzz
13 changes: 8 additions & 5 deletions src/main/native/compression/isa-l-master/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ applications. ISA-L includes:
Also see:
* [ISA-L for updates](https://github.com/01org/isa-l).
* For crypto functions see [isa-l_crypto on github](https://github.com/01org/isa-l_crypto).
* The [github wiki](https://github.com/01org/isa-l/wiki).
* The [github wiki](https://github.com/01org/isa-l/wiki) including a list of
[distros/ports](https://github.com/01org/isa-l/wiki/Ports--Repos) offering binary packages.
* ISA-L [mailing list](https://lists.01.org/mailman/listinfo/isal).
* [Contributing](CONTRIBUTING.md).

Expand All @@ -27,10 +28,11 @@ Building ISA-L

### Prerequisites

* yasm version 1.2.0 or later or nasm v2.11.01 or later.
* gcc, clang, icc or VC compiler.
* GNU 'make' or 'nmake' (Windows).
* Building with autotools requires autoconf/automake packages.
* Assembler: nasm v2.11.01 or later (nasm v2.13 or better suggested for building in AVX512 support)
or yasm version 1.2.0 or later.
* Compiler: gcc, clang, icc or VC compiler.
* Make: GNU 'make' or 'nmake' (Windows).
* Optional: Building with autotools requires autoconf/automake packages.

### Autotools
To build and install the library with autotools it is usually sufficient to run:
Expand All @@ -57,3 +59,4 @@ Other targets include:
* `make perfs` : create included performance tests
* `make ex` : build examples
* `make other` : build other utilities such as compression file tests
* `make doc` : build API manual
37 changes: 36 additions & 1 deletion src/main/native/compression/isa-l-master/Release_notes.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
v2.19 Intel Intelligent Storage Acceleration Library Release Notes
v2.21 Intel Intelligent Storage Acceleration Library Release Notes
==================================================================

RELEASE NOTE CONTENTS
Expand All @@ -15,6 +15,17 @@ RELEASE NOTE CONTENTS

2. FIXED ISSUES
---------------
v2.20

* Inflate total_out behavior corrected for in-progress decompression.
Previously total_out represented the total bytes decompressed into the output
buffer or temp internal buffer. This is changed to be only the bytes put into
the output buffer.

* Fixed issue with isal_create_hufftables_subset. Affects semi-dynamic
compression use case when explicitly creating hufftables from histogram. The
_hufftables_subset function could fail to generate length symbols for any
length that were never seen.

v2.19

Expand Down Expand Up @@ -55,6 +66,30 @@ v2.10

3. CHANGE LOG & FEATURES ADDED
------------------------------
v2.21

* Igzip improvements
- New compression levels added. ISA-L fast deflate now has more levels to
balance speed vs. target compression level. Level 0, 1 are as in previous
generations. New levels 2 & 3 target higher compression roughly comparable
to zlib levels 2-3. Level 3 is currently only optimized for processors with
AVX512 instructions.

* New T10dif & copy function - crc16_t10dif_copy()
- CRC and copy was added to emulate T10dif operations such as DIF insert and
strip. This function stitches together CRC and memcpy operations
eliminating an extra data read.

* CRC32 iscsi performance improvements
- Fixes issue under some distributions where warm cache performance was
reduced.

v2.20

* Igzip improvements
- Optimized deflate_hash in compression functions.
Improves performance of using preset dictionary.
- Removed alignment restrictions on input structure.

v2.19

Expand Down
8 changes: 4 additions & 4 deletions src/main/native/compression/isa-l-master/configure.ac
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

AC_PREREQ(2.69)
AC_INIT([libisal],
[2.19.0],
[2.21.0],
[[email protected]],
[isa-l],
[http://01.org/storage-acceleration-library])
Expand Down Expand Up @@ -85,8 +85,8 @@ else
with_modern_nasm=yes
AC_MSG_RESULT([yes])
AC_MSG_CHECKING([for optional nasm AVX512 support])
AC_LANG_CONFTEST([AC_LANG_SOURCE([[vpshufb zmm0, zmm1, zmm2;]])])
sed -i -e '/vpshufb/!d' conftest.c
AC_LANG_CONFTEST([AC_LANG_SOURCE([[vinserti32x8 zmm0, ymm1, 1;]])])
sed -i -e '/vinsert/!d' conftest.c
if nasm -f elf64 conftest.c 2> /dev/null; then
nasm_knows_avx512=yes
AC_MSG_RESULT([yes])
Expand All @@ -109,7 +109,7 @@ if test x"$AS" = x""; then
elif test x"$with_modern_nasm" = x"yes"; then
AS=nasm
else
AC_MSG_ERROR([No modern yasm or nasm found as required. Yasm should be 1.2.0 or later, and nasm should be v2.11.01 or later.])
AC_MSG_ERROR([No modern yasm or nasm found as required. Yasm should be 1.2.0 or later, and nasm should be v2.11.01 or later (v2.13 for AVX512).])
fi
fi
echo "Using assembler $AS"
Expand Down
Loading

0 comments on commit 51253aa

Please sign in to comment.