diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 301f985cbfb..bb43c4ed0aa 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -14,7 +14,6 @@ #define ZSTD_LITFREQ_ADD 2 /* scaling factor for litFreq, so that frequencies adapt faster to new stats */ -#define ZSTD_FREQ_DIV 4 /* log factor when using previous stats to init next stats */ #define ZSTD_MAX_PRICE (1<<30) #define ZSTD_PREDEF_THRESHOLD 1024 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */ @@ -24,11 +23,11 @@ * Price functions for optimal parser ***************************************/ -#if 0 /* approximation at bit level */ +#if 0 /* approximation at bit level (for tests) */ # define BITCOST_ACCURACY 0 # define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) -# define WEIGHT(stat) ((void)opt, ZSTD_bitWeight(stat)) -#elif 0 /* fractional bit accuracy */ +# define WEIGHT(stat, opt) ((void)opt, ZSTD_bitWeight(stat)) +#elif 0 /* fractional bit accuracy (for tests) */ # define BITCOST_ACCURACY 8 # define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) # define WEIGHT(stat,opt) ((void)opt, ZSTD_fracWeight(stat)) @@ -79,25 +78,46 @@ static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel) } -/* ZSTD_downscaleStat() : - * reduce all elements in table by a factor 2^(ZSTD_FREQ_DIV+malus) - * return the resulting sum of elements */ -static U32 ZSTD_downscaleStat(unsigned* table, U32 lastEltIndex, int malus) +static U32 sum_u32(const unsigned table[], size_t nbElts) +{ + size_t n; + U32 total = 0; + for (n=0; n 0 && ZSTD_FREQ_DIV+malus < 31); + DEBUGLOG(5, "ZSTD_downscaleStats (nbElts=%u, shift=%u)", (unsigned)lastEltIndex+1, (unsigned)shift); + assert(shift < 30); for (s=0; s> (ZSTD_FREQ_DIV+malus)); + table[s] = 1 + (table[s] >> shift); sum += table[s]; } return sum; } +/* ZSTD_scaleStats() : + * reduce all elements in table is sum too large + * return the resulting sum of elements */ +static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget) +{ + U32 const prevsum = sum_u32(table, lastEltIndex+1); + U32 const factor = prevsum >> logTarget; + DEBUGLOG(5, "ZSTD_scaleStats (nbElts=%u, target=%u)", (unsigned)lastEltIndex+1, (unsigned)logTarget); + assert(logTarget < 30); + if (factor <= 1) return prevsum; + return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor)); +} + /* ZSTD_rescaleFreqs() : * if first block (detected by optPtr->litLengthSum == 0) : init statistics * take hints from dictionary if there is one - * or init from zero, using src for literals stats, or flat 1 for match symbols + * and init from zero if there is none, + * using src for literals stats, and baseline stats for sequence symbols * otherwise downscale existing stats, to be used as seed for next block. */ static void @@ -174,14 +194,18 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, if (compressedLiterals) { unsigned lit = MaxLit; HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */ - optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1); + optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8); } - { unsigned ll; - for (ll=0; ll<=MaxLL; ll++) - optPtr->litLengthFreq[ll] = 1; + { unsigned const baseLLfreqs[MaxLL+1] = { + 4, 2, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1 + }; + ZSTD_memcpy(optPtr->litLengthFreq, baseLLfreqs, sizeof(baseLLfreqs)); optPtr->litLengthSum = sum_u32(baseLLfreqs, MaxLL+1); } - optPtr->litLengthSum = MaxLL+1; { unsigned ml; for (ml=0; ml<=MaxML; ml++) @@ -189,21 +213,25 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, } optPtr->matchLengthSum = MaxML+1; - { unsigned of; - for (of=0; of<=MaxOff; of++) - optPtr->offCodeFreq[of] = 1; + { unsigned const baseOFCfreqs[MaxOff+1] = { + 6, 2, 1, 1, 2, 3, 4, 4, + 4, 3, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1 + }; + ZSTD_memcpy(optPtr->offCodeFreq, baseOFCfreqs, sizeof(baseOFCfreqs)); optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1); } - optPtr->offCodeSum = MaxOff+1; + } } else { /* new block : re-use previous statistics, scaled down */ if (compressedLiterals) - optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1); - optPtr->litLengthSum = ZSTD_downscaleStat(optPtr->litLengthFreq, MaxLL, 0); - optPtr->matchLengthSum = ZSTD_downscaleStat(optPtr->matchLengthFreq, MaxML, 0); - optPtr->offCodeSum = ZSTD_downscaleStat(optPtr->offCodeFreq, MaxOff, 0); + optPtr->litSum = ZSTD_scaleStats(optPtr->litFreq, MaxLit, 12); + optPtr->litLengthSum = ZSTD_scaleStats(optPtr->litLengthFreq, MaxLL, 11); + optPtr->matchLengthSum = ZSTD_scaleStats(optPtr->matchLengthFreq, MaxML, 11); + optPtr->offCodeSum = ZSTD_scaleStats(optPtr->offCodeFreq, MaxOff, 11); } ZSTD_setBasePrices(optPtr, optLevel); @@ -901,11 +929,11 @@ static void ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm, ZSTD_match_ ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock); } + /*-******************************* * Optimal parser *********************************/ - static U32 ZSTD_totalLen(ZSTD_optimal_t sol) { return sol.litlen + sol.mlen; @@ -987,7 +1015,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, * in every price. We include the literal length to avoid negative * prices when we subtract the previous literal length. */ - opt[0].price = ZSTD_litLengthPrice(litlen, optStatePtr, optLevel); + opt[0].price = (int)ZSTD_litLengthPrice(litlen, optStatePtr, optLevel); /* large match -> immediate encoding */ { U32 const maxML = matches[nbMatches-1].len; @@ -1007,7 +1035,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, } } /* set prices for first matches starting position == 0 */ - { U32 const literalsPrice = opt[0].price + ZSTD_litLengthPrice(0, optStatePtr, optLevel); + assert(opt[0].price >= 0); + { U32 const literalsPrice = (U32)opt[0].price + ZSTD_litLengthPrice(0, optStatePtr, optLevel); U32 pos; U32 matchNb; for (pos = 1; pos < minMatch; pos++) { @@ -1024,7 +1053,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, opt[pos].mlen = pos; opt[pos].off = offset; opt[pos].litlen = litlen; - opt[pos].price = sequencePrice; + opt[pos].price = (int)sequencePrice; } } last_pos = pos-1; } @@ -1039,9 +1068,9 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, /* Fix current position with one literal if cheaper */ { U32 const litlen = (opt[cur-1].mlen == 0) ? opt[cur-1].litlen + 1 : 1; int const price = opt[cur-1].price - + ZSTD_rawLiteralsCost(ip+cur-1, 1, optStatePtr, optLevel) - + ZSTD_litLengthPrice(litlen, optStatePtr, optLevel) - - ZSTD_litLengthPrice(litlen-1, optStatePtr, optLevel); + + (int)ZSTD_rawLiteralsCost(ip+cur-1, 1, optStatePtr, optLevel) + + (int)ZSTD_litLengthPrice(litlen, optStatePtr, optLevel) + - (int)ZSTD_litLengthPrice(litlen-1, optStatePtr, optLevel); assert(price < 1000000000); /* overflow check */ if (price <= opt[cur].price) { DEBUGLOG(7, "cPos:%zi==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)", @@ -1084,9 +1113,10 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, continue; /* skip unpromising positions; about ~+6% speed, -0.01 ratio */ } + assert(opt[cur].price >= 0); { U32 const ll0 = (opt[cur].mlen != 0); U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0; - U32 const previousPrice = opt[cur].price; + U32 const previousPrice = (U32)opt[cur].price; U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel); U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch); U32 matchNb; @@ -1126,7 +1156,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, for (mlen = lastML; mlen >= startML; mlen--) { /* scan downward */ U32 const pos = cur + mlen; - int const price = basePrice + ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel); + int const price = (int)basePrice + (int)ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel); if ((pos > last_pos) || (price < opt[pos].price)) { DEBUGLOG(7, "rPos:%u (ml=%2u) => new better price (%.2f<%.2f)", @@ -1222,28 +1252,7 @@ size_t ZSTD_compressBlock_btopt( } -/* used in 2-pass strategy */ -static U32 ZSTD_upscaleStat(unsigned* table, U32 lastEltIndex, int bonus) -{ - U32 s, sum=0; - assert(ZSTD_FREQ_DIV+bonus >= 0); - for (s=0; slitSum = ZSTD_upscaleStat(optPtr->litFreq, MaxLit, 0); - optPtr->litLengthSum = ZSTD_upscaleStat(optPtr->litLengthFreq, MaxLL, 0); - optPtr->matchLengthSum = ZSTD_upscaleStat(optPtr->matchLengthFreq, MaxML, 0); - optPtr->offCodeSum = ZSTD_upscaleStat(optPtr->offCodeFreq, MaxOff, 0); -} /* ZSTD_initStats_ultra(): * make a first compression pass, just to seed stats with more accurate starting values. @@ -1274,8 +1283,6 @@ ZSTD_initStats_ultra(ZSTD_matchState_t* ms, ms->window.lowLimit = ms->window.dictLimit; ms->nextToUpdate = ms->window.dictLimit; - /* re-inforce weight of collected statistics */ - ZSTD_upscaleStats(&ms->opt); } size_t ZSTD_compressBlock_btultra( diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index 8364444d159..ec5a169c0f2 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -47,7 +47,7 @@ * Console display ***************************************/ #ifndef LOCALDISPLAYLEVEL -static int g_displayLevel = 2; +static int g_displayLevel = 0; #endif #undef DISPLAY #define DISPLAY(...) \ @@ -735,7 +735,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( COVER_map_t activeDmers; parameters.splitPoint = 1.0; /* Initialize global data */ - g_displayLevel = parameters.zParams.notificationLevel; + g_displayLevel = (int)parameters.zParams.notificationLevel; /* Checks */ if (!COVER_checkParameters(parameters, dictBufferCapacity)) { DISPLAYLEVEL(1, "Cover parameters incorrect\n"); diff --git a/lib/dictBuilder/fastcover.c b/lib/dictBuilder/fastcover.c index ed789f92f48..f4a2eed99a4 100644 --- a/lib/dictBuilder/fastcover.c +++ b/lib/dictBuilder/fastcover.c @@ -44,7 +44,7 @@ * Console display ***************************************/ #ifndef LOCALDISPLAYLEVEL -static int g_displayLevel = 2; +static int g_displayLevel = 0; #endif #undef DISPLAY #define DISPLAY(...) \ @@ -549,7 +549,7 @@ ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity, ZDICT_cover_params_t coverParams; FASTCOVER_accel_t accelParams; /* Initialize global data */ - g_displayLevel = parameters.zParams.notificationLevel; + g_displayLevel = (int)parameters.zParams.notificationLevel; /* Assign splitPoint and f if not provided */ parameters.splitPoint = 1.0; parameters.f = parameters.f == 0 ? DEFAULT_F : parameters.f; @@ -632,7 +632,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover( const unsigned accel = parameters->accel == 0 ? DEFAULT_ACCEL : parameters->accel; const unsigned shrinkDict = 0; /* Local variables */ - const int displayLevel = parameters->zParams.notificationLevel; + const int displayLevel = (int)parameters->zParams.notificationLevel; unsigned iteration = 1; unsigned d; unsigned k; @@ -716,7 +716,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover( data->parameters.splitPoint = splitPoint; data->parameters.steps = kSteps; data->parameters.shrinkDict = shrinkDict; - data->parameters.zParams.notificationLevel = g_displayLevel; + data->parameters.zParams.notificationLevel = (unsigned)g_displayLevel; /* Check the parameters */ if (!FASTCOVER_checkParameters(data->parameters, dictBufferCapacity, data->ctx->f, accel)) { diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 459cbe4d19b..9edc77fe800 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -139,7 +139,7 @@ static unsigned ZDICT_NbCommonBytes (size_t val) _BitScanForward64( &r, (U64)val ); return (unsigned)(r>>3); # elif defined(__GNUC__) && (__GNUC__ >= 3) - return (__builtin_ctzll((U64)val) >> 3); + return (unsigned)(__builtin_ctzll((U64)val) >> 3); # else static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; @@ -150,7 +150,7 @@ static unsigned ZDICT_NbCommonBytes (size_t val) _BitScanForward( &r, (U32)val ); return (unsigned)(r>>3); # elif defined(__GNUC__) && (__GNUC__ >= 3) - return (__builtin_ctz((U32)val) >> 3); + return (unsigned)(__builtin_ctz((U32)val) >> 3); # else static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; @@ -163,7 +163,7 @@ static unsigned ZDICT_NbCommonBytes (size_t val) _BitScanReverse64( &r, val ); return (unsigned)(r>>3); # elif defined(__GNUC__) && (__GNUC__ >= 3) - return (__builtin_clzll(val) >> 3); + return (unsigned)(__builtin_clzll(val) >> 3); # else unsigned r; const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */ @@ -178,7 +178,7 @@ static unsigned ZDICT_NbCommonBytes (size_t val) _BitScanReverse( &r, (unsigned long)val ); return (unsigned)(r>>3); # elif defined(__GNUC__) && (__GNUC__ >= 3) - return (__builtin_clz((U32)val) >> 3); + return (unsigned)(__builtin_clz((U32)val) >> 3); # else unsigned r; if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } @@ -235,7 +235,7 @@ static dictItem ZDICT_analyzePos( U32 savings[LLIMIT] = {0}; const BYTE* b = (const BYTE*)buffer; size_t maxLength = LLIMIT; - size_t pos = suffix[start]; + size_t pos = (size_t)suffix[start]; U32 end = start; dictItem solution; @@ -369,7 +369,7 @@ static dictItem ZDICT_analyzePos( savings[i] = savings[i-1] + (lengthList[i] * (i-3)); DISPLAYLEVEL(4, "Selected dict at position %u, of length %u : saves %u (ratio: %.2f) \n", - (unsigned)pos, (unsigned)maxLength, (unsigned)savings[maxLength], (double)savings[maxLength] / maxLength); + (unsigned)pos, (unsigned)maxLength, (unsigned)savings[maxLength], (double)savings[maxLength] / (double)maxLength); solution.pos = (U32)pos; solution.length = (U32)maxLength; @@ -379,7 +379,7 @@ static dictItem ZDICT_analyzePos( { U32 id; for (id=start; id= elt.pos) && (table[u].pos < elt.pos)) { /* overlap, existing < new */ /* append */ - int const addedLength = (int)eltEnd - (table[u].pos + table[u].length); + int const addedLength = (int)eltEnd - (int)(table[u].pos + table[u].length); table[u].savings += elt.length / 8; /* rough approx bonus */ if (addedLength > 0) { /* otherwise, elt fully included into existing */ table[u].length += addedLength; @@ -766,6 +766,13 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, pos += fileSizes[u]; } + if (notificationLevel >= 4) { + /* writeStats */ + DISPLAYLEVEL(4, "Offset Code Frequencies : \n"); + for (u=0; u<=offcodeMax; u++) { + DISPLAYLEVEL(4, "%2u :%7u \n", u, offcodeCount[u]); + } } + /* analyze, build stats, starting with literals */ { size_t maxNbBits = HUF_buildCTable (hufTable, countLit, 255, huffLog); if (HUF_isError(maxNbBits)) { @@ -872,7 +879,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, MEM_writeLE32(dstPtr+8, bestRepOffset[2].offset); #else /* at this stage, we don't use the result of "most common first offset", - as the impact of statistics is not properly evaluated */ + * as the impact of statistics is not properly evaluated */ MEM_writeLE32(dstPtr+0, repStartValue[0]); MEM_writeLE32(dstPtr+4, repStartValue[1]); MEM_writeLE32(dstPtr+8, repStartValue[2]); diff --git a/programs/benchzstd.c b/programs/benchzstd.c index d77959aac94..5cfe624b072 100644 --- a/programs/benchzstd.c +++ b/programs/benchzstd.c @@ -70,6 +70,8 @@ static const size_t maxMemory = (sizeof(size_t)==4) ? #define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush(NULL); } #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ +#define OUTPUT(...) { fprintf(stdout, __VA_ARGS__); fflush(NULL); } +#define OUTPUTLEVEL(l, ...) if (displayLevel>=l) { OUTPUT(__VA_ARGS__); } /* ************************************* @@ -181,7 +183,7 @@ BMK_initCCtx(ZSTD_CCtx* ctx, CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_minMatch, (int)comprParams->minMatch)); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_targetLength, (int)comprParams->targetLength)); CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_literalCompressionMode, (int)adv->literalCompressionMode)); - CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_strategy, comprParams->strategy)); + CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_strategy, (int)comprParams->strategy)); CHECK_Z(ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize)); } @@ -371,10 +373,7 @@ BMK_benchMemAdvancedNoAlloc( if (adv->mode == BMK_decodeOnly) { cSizes[nbBlocks] = thisBlockSize; benchResult.cSize = thisBlockSize; - } - } - } - } + } } } } /* warming up `compressedBuffer` */ if (adv->mode == BMK_decodeOnly) { @@ -393,8 +392,6 @@ BMK_benchMemAdvancedNoAlloc( BMK_benchParams_t cbp, dbp; BMK_initCCtxArgs cctxprep; BMK_initDCtxArgs dctxprep; - UTIL_HumanReadableSize_t hr_isize; - UTIL_HumanReadableSize_t hr_osize; cbp.benchFn = local_defaultCompress; /* ZSTD_compress2 */ cbp.benchPayload = cctx; @@ -431,10 +428,9 @@ BMK_benchMemAdvancedNoAlloc( dctxprep.dictBuffer = dictBuffer; dctxprep.dictBufferSize = dictBufferSize; - hr_isize = UTIL_makeHumanReadableSize((U64) srcSize); - - DISPLAYLEVEL(2, "\r%70s\r", ""); /* blank line */ - DISPLAYLEVEL(2, "%2s-%-17.17s : %.*f%s -> \r", marks[markNb], displayName, hr_isize.precision, hr_isize.value, hr_isize.suffix); + OUTPUTLEVEL(2, "\r%70s\r", ""); /* blank line */ + assert(srcSize < UINT_MAX); + OUTPUTLEVEL(2, "%2s-%-17.17s :%10u -> \r", marks[markNb], displayName, (unsigned)srcSize); while (!(compressionCompleted && decompressionCompleted)) { if (!compressionCompleted) { @@ -446,7 +442,7 @@ BMK_benchMemAdvancedNoAlloc( { BMK_runTime_t const cResult = BMK_extract_runTime(cOutcome); cSize = cResult.sumOfReturn; - ratio = (double)srcSize / cSize; + ratio = (double)srcSize / (double)cSize; { BMK_benchResult_t newResult; newResult.cSpeed = (U64)((double)srcSize * TIMELOOP_NANOSEC / cResult.nanoSecPerRun); benchResult.cSize = cSize; @@ -455,11 +451,10 @@ BMK_benchMemAdvancedNoAlloc( } } { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - hr_osize = UTIL_makeHumanReadableSize((U64) cSize); - DISPLAYLEVEL(2, "%2s-%-17.17s : %.*f%s -> %.*f%s (%5.*f), %6.*f MB/s\r", + assert(cSize < UINT_MAX); + OUTPUTLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f), %6.*f MB/s\r", marks[markNb], displayName, - hr_isize.precision, hr_isize.value, hr_isize.suffix, - hr_osize.precision, hr_osize.value, hr_osize.suffix, + (unsigned)srcSize, (unsigned)cSize, ratioAccuracy, ratio, benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT); } @@ -480,11 +475,9 @@ BMK_benchMemAdvancedNoAlloc( } { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - hr_osize = UTIL_makeHumanReadableSize((U64) cSize); - DISPLAYLEVEL(2, "%2s-%-17.17s : %.*f%s -> %.*f%s (%5.*f), %6.*f MB/s, %6.1f MB/s \r", + OUTPUTLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f), %6.*f MB/s, %6.1f MB/s \r", marks[markNb], displayName, - hr_isize.precision, hr_isize.value, hr_isize.suffix, - hr_osize.precision, hr_osize.value, hr_osize.suffix, + (unsigned)srcSize, (unsigned)cSize, ratioAccuracy, ratio, benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT, (double)benchResult.dSpeed / MB_UNIT); @@ -543,13 +536,13 @@ BMK_benchMemAdvancedNoAlloc( double const cSpeed = (double)benchResult.cSpeed / MB_UNIT; double const dSpeed = (double)benchResult.dSpeed / MB_UNIT; if (adv->additionalParam) { - DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, adv->additionalParam); + OUTPUT("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, adv->additionalParam); } else { - DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName); + OUTPUT("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName); } } - DISPLAYLEVEL(2, "%2i#\n", cLevel); + OUTPUTLEVEL(2, "%2i#\n", cLevel); } /* Bench */ benchResult.cMem = (1ULL << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(cctx); @@ -678,7 +671,7 @@ static BMK_benchOutcome_t BMK_benchCLevel(const void* srcBuffer, size_t benchedS } if (displayLevel == 1 && !adv->additionalParam) /* --quiet mode */ - DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", + OUTPUT("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (unsigned)benchedSize, adv->nbSeconds, (unsigned)(adv->blockSize>>10)); @@ -768,7 +761,7 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize, } { FILE* const f = fopen(fileNamesTable[n], "rb"); if (f==NULL) RETURN_ERROR_INT(10, "impossible to open file %s", fileNamesTable[n]); - DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[n]); + OUTPUTLEVEL(2, "Loading %s... \r", fileNamesTable[n]); if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); if (readSize != (size_t)fileSize) RETURN_ERROR_INT(11, "could not read %s", fileNamesTable[n]); diff --git a/programs/util.c b/programs/util.c index b29177818b8..61cde727e01 100644 --- a/programs/util.c +++ b/programs/util.c @@ -308,7 +308,8 @@ U64 UTIL_getFileSizeStat(const stat_t* statbuf) return (U64)statbuf->st_size; } -UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size) { +UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size) +{ UTIL_HumanReadableSize_t hrs; if (g_utilDisplayLevel > 3) { @@ -1121,9 +1122,9 @@ DWORD CountSetBits(ULONG_PTR bitMask) { DWORD LSHIFT = sizeof(ULONG_PTR)*8 - 1; DWORD bitSetCount = 0; - ULONG_PTR bitTest = (ULONG_PTR)1 << LSHIFT; + ULONG_PTR bitTest = (ULONG_PTR)1 << LSHIFT; DWORD i; - + for (i = 0; i <= LSHIFT; ++i) { bitSetCount += ((bitMask & bitTest)?1:0); diff --git a/programs/util.h b/programs/util.h index 7632b9a0a66..5e529b7ebae 100644 --- a/programs/util.h +++ b/programs/util.h @@ -180,9 +180,13 @@ U64 UTIL_getFileSize(const char* infilename); U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles); /** - * Take a size in bytes and prepare the components to pretty-print it in a - * scaled way. The components in the returned struct should be passed in + * Take @size in bytes, + * prepare the components to pretty-print it in a scaled way. + * The components in the returned struct should be passed in * precision, value, suffix order to a "%.*f%s" format string. + * Output policy is sensible to @g_utilDisplayLevel, + * for verbose mode (@g_utilDisplayLevel >= 4), + * does not scale down. */ typedef struct { double value; diff --git a/tests/automated_benchmarking.py b/tests/automated_benchmarking.py index 458bda4517e..e6da9024418 100644 --- a/tests/automated_benchmarking.py +++ b/tests/automated_benchmarking.py @@ -87,7 +87,7 @@ def clone_and_build(build): git clone {github_url} zstd-{user}-{sha} && cd zstd-{user}-{sha} && {checkout_command} - make && + make -j && cd ../ """.format( user=build["user"], @@ -100,7 +100,7 @@ def clone_and_build(build): ) return "zstd-{user}-{sha}/zstd".format(user=build["user"], sha=build["hash"]) else: - os.system("cd ../ && make && cd tests") + os.system("cd ../ && make -j && cd tests") return "../zstd" @@ -112,9 +112,9 @@ def parse_benchmark_output(output): def benchmark_single(executable, level, filename): return parse_benchmark_output(( subprocess.run( - [executable, "-qb{}".format(level), filename], stderr=subprocess.PIPE + [executable, "-qb{}".format(level), filename], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) - .stderr.decode("utf-8") + .stdout.decode("utf-8") .split(" ") )) @@ -145,7 +145,7 @@ def benchmark(build, filenames, levels, iterations): def benchmark_dictionary_single(executable, filenames_directory, dictionary_filename, level, iterations): cspeeds, dspeeds = [], [] for _ in range(iterations): - output = subprocess.run([executable, "-qb{}".format(level), "-D", dictionary_filename, "-r", filenames_directory], stderr=subprocess.PIPE).stderr.decode("utf-8").split(" ") + output = subprocess.run([executable, "-qb{}".format(level), "-D", dictionary_filename, "-r", filenames_directory], stdout=subprocess.PIPE).stdout.decode("utf-8").split(" ") cspeed, dspeed = parse_benchmark_output(output) cspeeds.append(cspeed) dspeeds.append(dspeed) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index fff963176fd..ce79f53acf3 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -42,7 +42,7 @@ #include "util.h" #include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */ /* must be included after util.h, due to ERROR macro redefinition issue on Visual Studio */ -#include "zstd_internal.h" /* ZSTD_WORKSPACETOOLARGE_MAXDURATION, ZSTD_WORKSPACETOOLARGE_FACTOR, KB, MB */ +#include "zstd_internal.h" /* ZSTD_WORKSPACETOOLARGE_MAXDURATION, ZSTD_WORKSPACETOOLARGE_FACTOR, KB, MB */ #include "threading.h" /* ZSTD_pthread_create, ZSTD_pthread_join */ @@ -128,7 +128,7 @@ static U32 FUZ_highbit32(U32 v32) #define CHECK_VAR(var, fn) var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); goto _output_error; } #define CHECK_NEWV(var, fn) size_t const CHECK_VAR(var, fn) -#define CHECK(fn) { CHECK_NEWV(err, fn); } +#define CHECK(fn) { CHECK_NEWV(__err, fn); } #define CHECKPLUS(var, fn, more) { CHECK_NEWV(var, fn); more; } #define CHECK_OP(op, lhs, rhs) { \ @@ -1832,8 +1832,7 @@ static int basicUnitTests(U32 const seed, double compressibility) /* Simple API skippable frame test */ DISPLAYLEVEL(3, "test%3i : read/write a skippable frame : ", testNb++); - { - U32 i; + { U32 i; unsigned readMagic; unsigned long long receivedSize; size_t skippableSize; @@ -1841,9 +1840,10 @@ static int basicUnitTests(U32 const seed, double compressibility) char* const skipBuff = (char*)malloc(skipLen); assert(skipBuff != NULL); for (i = 0; i < skipLen; i++) - skipBuff[i] = (BYTE) ((seed + i) % 256); - skippableSize = ZSTD_writeSkippableFrame((BYTE*)compressedBuffer, compressedBufferSize, - skipBuff, skipLen, seed % 15); + skipBuff[i] = (char) ((seed + i) % 256); + skippableSize = ZSTD_writeSkippableFrame( + compressedBuffer, compressedBufferSize, + skipBuff, skipLen, seed % 15); CHECK_Z(skippableSize); CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize)); receivedSize = ZSTD_readSkippableFrame(decodedBuffer, CNBuffSize, &readMagic, compressedBuffer, skippableSize); @@ -1860,8 +1860,9 @@ static int basicUnitTests(U32 const seed, double compressibility) unsigned readMagic; unsigned long long receivedSize; size_t skippableSize; - skippableSize = ZSTD_writeSkippableFrame((BYTE*)compressedBuffer, compressedBufferSize, - CNBuffer, 0, seed % 15); + skippableSize = ZSTD_writeSkippableFrame( + compressedBuffer, compressedBufferSize, + CNBuffer, 0, seed % 15); CHECK_EQ(ZSTD_SKIPPABLEHEADERSIZE, skippableSize); CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize)); receivedSize = ZSTD_readSkippableFrame(NULL, 0, &readMagic, compressedBuffer, skippableSize); @@ -1955,6 +1956,9 @@ static int basicUnitTests(U32 const seed, double compressibility) } } DISPLAYLEVEL(3, "OK \n"); + /* Note : these tests should be replaced by proper regression tests, + * but existing ones do not focus on small data + dictionary + all levels. + */ if ((int)(compressibility * 100 + 0.1) == FUZ_compressibility_default) { /* test only valid with known input */ size_t const flatdictSize = 22 KB; size_t const contentSize = 9 KB; @@ -1963,14 +1967,14 @@ static int basicUnitTests(U32 const seed, double compressibility) /* These upper bounds are generally within a few bytes of the compressed size */ size_t target_nodict_cSize[22+1] = { 3840, 3770, 3870, 3830, 3770, 3770, 3770, 3770, 3750, 3750, - 3742, 3670, 3670, 3660, 3660, - 3660, 3660, 3660, 3660, 3660, + 3742, 3675, 3674, 3665, 3664, + 3663, 3662, 3661, 3660, 3660, 3660, 3660, 3660 }; size_t const target_wdict_cSize[22+1] = { 2830, 2896, 2890, 2820, 2940, 2950, 2950, 2925, 2900, 2891, - 2910, 2910, 2910, 2770, 2760, - 2750, 2750, 2750, 2750, 2750, - 2750, 2750, 2750 }; + 2910, 2910, 2910, 2780, 2775, + 2765, 2760, 2755, 2754, 2753, + 2753, 2753, 2753 }; int l = 1; int const maxLevel = ZSTD_maxCLevel(); /* clevels with strategies that support rowhash on small inputs */ @@ -3471,11 +3475,7 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "error! l: %d dict: %zu srcSize: %zu cctx size cpar: %zu, cctx size level: %zu\n", level, dictSize, srcSize, cctxSizeUsingCParams, cctxSizeUsingLevel); goto _output_error; - } - } - } - } - } + } } } } } DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "test%3i : thread pool API tests : \n", testNb++) @@ -3591,8 +3591,7 @@ static int longUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "longtest%3i : testing ldm no regressions in size for opt parser : ", testNb++); - { - size_t cSizeLdm; + { size_t cSizeLdm; size_t cSizeNoLdm; ZSTD_CCtx* const cctx = ZSTD_createCCtx(); @@ -3670,7 +3669,7 @@ static int longUnitTests(U32 const seed, double compressibility) CHECK(cdict != NULL); CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); - CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, attachPref)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, (int)attachPref)); cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); CHECK_Z(cSize); diff --git a/tests/regression/results.csv b/tests/regression/results.csv index e3ed129bc1e..9909a15a014 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -11,10 +11,10 @@ silesia.tar, level 6, compress silesia.tar, level 7, compress simple, 4576829 silesia.tar, level 9, compress simple, 4552584 silesia.tar, level 13, compress simple, 4502956 -silesia.tar, level 16, compress simple, 4356834 -silesia.tar, level 19, compress simple, 4264388 +silesia.tar, level 16, compress simple, 4360527 +silesia.tar, level 19, compress simple, 4267266 silesia.tar, uncompressed literals, compress simple, 4861424 -silesia.tar, uncompressed literals optimal, compress simple, 4264388 +silesia.tar, uncompressed literals optimal, compress simple, 4267266 silesia.tar, huffman literals, compress simple, 6182241 github.tar, level -5, compress simple, 66914 github.tar, level -3, compress simple, 52127 @@ -28,10 +28,10 @@ github.tar, level 6, compress github.tar, level 7, compress simple, 38073 github.tar, level 9, compress simple, 36767 github.tar, level 13, compress simple, 35501 -github.tar, level 16, compress simple, 40255 -github.tar, level 19, compress simple, 32837 +github.tar, level 16, compress simple, 40471 +github.tar, level 19, compress simple, 32134 github.tar, uncompressed literals, compress simple, 38441 -github.tar, uncompressed literals optimal, compress simple, 32837 +github.tar, uncompressed literals optimal, compress simple, 32134 github.tar, huffman literals, compress simple, 42560 silesia, level -5, compress cctx, 7354675 silesia, level -3, compress cctx, 6902374 @@ -45,8 +45,8 @@ silesia, level 6, compress silesia, level 7, compress cctx, 4567204 silesia, level 9, compress cctx, 4543310 silesia, level 13, compress cctx, 4493990 -silesia, level 16, compress cctx, 4360251 -silesia, level 19, compress cctx, 4283237 +silesia, level 16, compress cctx, 4359864 +silesia, level 19, compress cctx, 4296880 silesia, long distance mode, compress cctx, 4849553 silesia, multithreaded, compress cctx, 4849553 silesia, multithreaded long distance mode, compress cctx, 4849553 @@ -55,7 +55,7 @@ silesia, small hash log, compress silesia, small chain log, compress cctx, 4912197 silesia, explicit params, compress cctx, 4794481 silesia, uncompressed literals, compress cctx, 4849553 -silesia, uncompressed literals optimal, compress cctx, 4283237 +silesia, uncompressed literals optimal, compress cctx, 4296880 silesia, huffman literals, compress cctx, 6177565 silesia, multithreaded with advanced params, compress cctx, 4849553 github, level -5, compress cctx, 232315 @@ -109,8 +109,8 @@ silesia, level 6, zstdcli, silesia, level 7, zstdcli, 4567252 silesia, level 9, zstdcli, 4543358 silesia, level 13, zstdcli, 4494038 -silesia, level 16, zstdcli, 4360299 -silesia, level 19, zstdcli, 4283285 +silesia, level 16, zstdcli, 4359912 +silesia, level 19, zstdcli, 4296928 silesia, long distance mode, zstdcli, 4840807 silesia, multithreaded, zstdcli, 4849601 silesia, multithreaded long distance mode, zstdcli, 4840807 @@ -119,7 +119,7 @@ silesia, small hash log, zstdcli, silesia, small chain log, zstdcli, 4912245 silesia, explicit params, zstdcli, 4795856 silesia, uncompressed literals, zstdcli, 5128030 -silesia, uncompressed literals optimal, zstdcli, 4317944 +silesia, uncompressed literals optimal, zstdcli, 4319566 silesia, huffman literals, zstdcli, 5326394 silesia, multithreaded with advanced params, zstdcli, 5128030 silesia.tar, level -5, zstdcli, 7363866 @@ -134,8 +134,8 @@ silesia.tar, level 6, zstdcli, silesia.tar, level 7, zstdcli, 4578884 silesia.tar, level 9, zstdcli, 4553499 silesia.tar, level 13, zstdcli, 4502960 -silesia.tar, level 16, zstdcli, 4356838 -silesia.tar, level 19, zstdcli, 4264392 +silesia.tar, level 16, zstdcli, 4360531 +silesia.tar, level 19, zstdcli, 4267270 silesia.tar, no source size, zstdcli, 4861508 silesia.tar, long distance mode, zstdcli, 4853225 silesia.tar, multithreaded, zstdcli, 4861512 @@ -145,7 +145,7 @@ silesia.tar, small hash log, zstdcli, silesia.tar, small chain log, zstdcli, 4917022 silesia.tar, explicit params, zstdcli, 4821274 silesia.tar, uncompressed literals, zstdcli, 5129559 -silesia.tar, uncompressed literals optimal, zstdcli, 4307404 +silesia.tar, uncompressed literals optimal, zstdcli, 4310145 silesia.tar, huffman literals, zstdcli, 5344915 silesia.tar, multithreaded with advanced params, zstdcli, 5129559 github, level -5, zstdcli, 234315 @@ -210,11 +210,11 @@ github.tar, level 7 with dict, zstdcli, github.tar, level 9, zstdcli, 36771 github.tar, level 9 with dict, zstdcli, 36623 github.tar, level 13, zstdcli, 35505 -github.tar, level 13 with dict, zstdcli, 38730 -github.tar, level 16, zstdcli, 40259 -github.tar, level 16 with dict, zstdcli, 33643 -github.tar, level 19, zstdcli, 32841 -github.tar, level 19 with dict, zstdcli, 32899 +github.tar, level 13 with dict, zstdcli, 37134 +github.tar, level 16, zstdcli, 40475 +github.tar, level 16 with dict, zstdcli, 33382 +github.tar, level 19, zstdcli, 32138 +github.tar, level 19 with dict, zstdcli, 32713 github.tar, no source size, zstdcli, 38442 github.tar, no source size with dict, zstdcli, 38004 github.tar, long distance mode, zstdcli, 39730 @@ -225,7 +225,7 @@ github.tar, small hash log, zstdcli, github.tar, small chain log, zstdcli, 41673 github.tar, explicit params, zstdcli, 41227 github.tar, uncompressed literals, zstdcli, 41126 -github.tar, uncompressed literals optimal, zstdcli, 35392 +github.tar, uncompressed literals optimal, zstdcli, 35401 github.tar, huffman literals, zstdcli, 38857 github.tar, multithreaded with advanced params, zstdcli, 41126 silesia, level -5, advanced one pass, 7354675 @@ -248,8 +248,8 @@ silesia, level 11 row 2, advanced silesia, level 12 row 1, advanced one pass, 4503116 silesia, level 12 row 2, advanced one pass, 4505153 silesia, level 13, advanced one pass, 4493990 -silesia, level 16, advanced one pass, 4360251 -silesia, level 19, advanced one pass, 4283237 +silesia, level 16, advanced one pass, 4359864 +silesia, level 19, advanced one pass, 4296880 silesia, no source size, advanced one pass, 4849553 silesia, long distance mode, advanced one pass, 4840737 silesia, multithreaded, advanced one pass, 4849553 @@ -259,7 +259,7 @@ silesia, small hash log, advanced silesia, small chain log, advanced one pass, 4912197 silesia, explicit params, advanced one pass, 4795856 silesia, uncompressed literals, advanced one pass, 5127982 -silesia, uncompressed literals optimal, advanced one pass, 4317896 +silesia, uncompressed literals optimal, advanced one pass, 4319518 silesia, huffman literals, advanced one pass, 5326346 silesia, multithreaded with advanced params, advanced one pass, 5127982 silesia.tar, level -5, advanced one pass, 7359401 @@ -282,8 +282,8 @@ silesia.tar, level 11 row 2, advanced silesia.tar, level 12 row 1, advanced one pass, 4513604 silesia.tar, level 12 row 2, advanced one pass, 4514568 silesia.tar, level 13, advanced one pass, 4502956 -silesia.tar, level 16, advanced one pass, 4356834 -silesia.tar, level 19, advanced one pass, 4264388 +silesia.tar, level 16, advanced one pass, 4360527 +silesia.tar, level 19, advanced one pass, 4267266 silesia.tar, no source size, advanced one pass, 4861424 silesia.tar, long distance mode, advanced one pass, 4847752 silesia.tar, multithreaded, advanced one pass, 4861508 @@ -293,7 +293,7 @@ silesia.tar, small hash log, advanced silesia.tar, small chain log, advanced one pass, 4917041 silesia.tar, explicit params, advanced one pass, 4807381 silesia.tar, uncompressed literals, advanced one pass, 5129458 -silesia.tar, uncompressed literals optimal, advanced one pass, 4307400 +silesia.tar, uncompressed literals optimal, advanced one pass, 4310141 silesia.tar, huffman literals, advanced one pass, 5344545 silesia.tar, multithreaded with advanced params, advanced one pass, 5129555 github, level -5, advanced one pass, 232315 @@ -516,23 +516,23 @@ github.tar, level 12 row 2 with dict dds, advanced github.tar, level 12 row 2 with dict copy, advanced one pass, 36609 github.tar, level 12 row 2 with dict load, advanced one pass, 36460 github.tar, level 13, advanced one pass, 35501 -github.tar, level 13 with dict, advanced one pass, 38726 -github.tar, level 13 with dict dms, advanced one pass, 38903 -github.tar, level 13 with dict dds, advanced one pass, 38903 -github.tar, level 13 with dict copy, advanced one pass, 38726 +github.tar, level 13 with dict, advanced one pass, 37130 +github.tar, level 13 with dict dms, advanced one pass, 37267 +github.tar, level 13 with dict dds, advanced one pass, 37267 +github.tar, level 13 with dict copy, advanced one pass, 37130 github.tar, level 13 with dict load, advanced one pass, 36010 -github.tar, level 16, advanced one pass, 40255 -github.tar, level 16 with dict, advanced one pass, 33639 -github.tar, level 16 with dict dms, advanced one pass, 33544 -github.tar, level 16 with dict dds, advanced one pass, 33544 -github.tar, level 16 with dict copy, advanced one pass, 33639 -github.tar, level 16 with dict load, advanced one pass, 39353 -github.tar, level 19, advanced one pass, 32837 -github.tar, level 19 with dict, advanced one pass, 32895 -github.tar, level 19 with dict dms, advanced one pass, 32672 -github.tar, level 19 with dict dds, advanced one pass, 32672 -github.tar, level 19 with dict copy, advanced one pass, 32895 -github.tar, level 19 with dict load, advanced one pass, 32676 +github.tar, level 16, advanced one pass, 40471 +github.tar, level 16 with dict, advanced one pass, 33378 +github.tar, level 16 with dict dms, advanced one pass, 33213 +github.tar, level 16 with dict dds, advanced one pass, 33213 +github.tar, level 16 with dict copy, advanced one pass, 33378 +github.tar, level 16 with dict load, advanced one pass, 39081 +github.tar, level 19, advanced one pass, 32134 +github.tar, level 19 with dict, advanced one pass, 32709 +github.tar, level 19 with dict dms, advanced one pass, 32553 +github.tar, level 19 with dict dds, advanced one pass, 32553 +github.tar, level 19 with dict copy, advanced one pass, 32709 +github.tar, level 19 with dict load, advanced one pass, 32474 github.tar, no source size, advanced one pass, 38441 github.tar, no source size with dict, advanced one pass, 37995 github.tar, long distance mode, advanced one pass, 39757 @@ -543,7 +543,7 @@ github.tar, small hash log, advanced github.tar, small chain log, advanced one pass, 41669 github.tar, explicit params, advanced one pass, 41227 github.tar, uncompressed literals, advanced one pass, 41122 -github.tar, uncompressed literals optimal, advanced one pass, 35388 +github.tar, uncompressed literals optimal, advanced one pass, 35397 github.tar, huffman literals, advanced one pass, 38853 github.tar, multithreaded with advanced params, advanced one pass, 41122 silesia, level -5, advanced one pass small out, 7354675 @@ -566,8 +566,8 @@ silesia, level 11 row 2, advanced silesia, level 12 row 1, advanced one pass small out, 4503116 silesia, level 12 row 2, advanced one pass small out, 4505153 silesia, level 13, advanced one pass small out, 4493990 -silesia, level 16, advanced one pass small out, 4360251 -silesia, level 19, advanced one pass small out, 4283237 +silesia, level 16, advanced one pass small out, 4359864 +silesia, level 19, advanced one pass small out, 4296880 silesia, no source size, advanced one pass small out, 4849553 silesia, long distance mode, advanced one pass small out, 4840737 silesia, multithreaded, advanced one pass small out, 4849553 @@ -577,7 +577,7 @@ silesia, small hash log, advanced silesia, small chain log, advanced one pass small out, 4912197 silesia, explicit params, advanced one pass small out, 4795856 silesia, uncompressed literals, advanced one pass small out, 5127982 -silesia, uncompressed literals optimal, advanced one pass small out, 4317896 +silesia, uncompressed literals optimal, advanced one pass small out, 4319518 silesia, huffman literals, advanced one pass small out, 5326346 silesia, multithreaded with advanced params, advanced one pass small out, 5127982 silesia.tar, level -5, advanced one pass small out, 7359401 @@ -600,8 +600,8 @@ silesia.tar, level 11 row 2, advanced silesia.tar, level 12 row 1, advanced one pass small out, 4513604 silesia.tar, level 12 row 2, advanced one pass small out, 4514568 silesia.tar, level 13, advanced one pass small out, 4502956 -silesia.tar, level 16, advanced one pass small out, 4356834 -silesia.tar, level 19, advanced one pass small out, 4264388 +silesia.tar, level 16, advanced one pass small out, 4360527 +silesia.tar, level 19, advanced one pass small out, 4267266 silesia.tar, no source size, advanced one pass small out, 4861424 silesia.tar, long distance mode, advanced one pass small out, 4847752 silesia.tar, multithreaded, advanced one pass small out, 4861508 @@ -611,7 +611,7 @@ silesia.tar, small hash log, advanced silesia.tar, small chain log, advanced one pass small out, 4917041 silesia.tar, explicit params, advanced one pass small out, 4807381 silesia.tar, uncompressed literals, advanced one pass small out, 5129458 -silesia.tar, uncompressed literals optimal, advanced one pass small out, 4307400 +silesia.tar, uncompressed literals optimal, advanced one pass small out, 4310141 silesia.tar, huffman literals, advanced one pass small out, 5344545 silesia.tar, multithreaded with advanced params, advanced one pass small out, 5129555 github, level -5, advanced one pass small out, 232315 @@ -834,23 +834,23 @@ github.tar, level 12 row 2 with dict dds, advanced github.tar, level 12 row 2 with dict copy, advanced one pass small out, 36609 github.tar, level 12 row 2 with dict load, advanced one pass small out, 36460 github.tar, level 13, advanced one pass small out, 35501 -github.tar, level 13 with dict, advanced one pass small out, 38726 -github.tar, level 13 with dict dms, advanced one pass small out, 38903 -github.tar, level 13 with dict dds, advanced one pass small out, 38903 -github.tar, level 13 with dict copy, advanced one pass small out, 38726 +github.tar, level 13 with dict, advanced one pass small out, 37130 +github.tar, level 13 with dict dms, advanced one pass small out, 37267 +github.tar, level 13 with dict dds, advanced one pass small out, 37267 +github.tar, level 13 with dict copy, advanced one pass small out, 37130 github.tar, level 13 with dict load, advanced one pass small out, 36010 -github.tar, level 16, advanced one pass small out, 40255 -github.tar, level 16 with dict, advanced one pass small out, 33639 -github.tar, level 16 with dict dms, advanced one pass small out, 33544 -github.tar, level 16 with dict dds, advanced one pass small out, 33544 -github.tar, level 16 with dict copy, advanced one pass small out, 33639 -github.tar, level 16 with dict load, advanced one pass small out, 39353 -github.tar, level 19, advanced one pass small out, 32837 -github.tar, level 19 with dict, advanced one pass small out, 32895 -github.tar, level 19 with dict dms, advanced one pass small out, 32672 -github.tar, level 19 with dict dds, advanced one pass small out, 32672 -github.tar, level 19 with dict copy, advanced one pass small out, 32895 -github.tar, level 19 with dict load, advanced one pass small out, 32676 +github.tar, level 16, advanced one pass small out, 40471 +github.tar, level 16 with dict, advanced one pass small out, 33378 +github.tar, level 16 with dict dms, advanced one pass small out, 33213 +github.tar, level 16 with dict dds, advanced one pass small out, 33213 +github.tar, level 16 with dict copy, advanced one pass small out, 33378 +github.tar, level 16 with dict load, advanced one pass small out, 39081 +github.tar, level 19, advanced one pass small out, 32134 +github.tar, level 19 with dict, advanced one pass small out, 32709 +github.tar, level 19 with dict dms, advanced one pass small out, 32553 +github.tar, level 19 with dict dds, advanced one pass small out, 32553 +github.tar, level 19 with dict copy, advanced one pass small out, 32709 +github.tar, level 19 with dict load, advanced one pass small out, 32474 github.tar, no source size, advanced one pass small out, 38441 github.tar, no source size with dict, advanced one pass small out, 37995 github.tar, long distance mode, advanced one pass small out, 39757 @@ -861,7 +861,7 @@ github.tar, small hash log, advanced github.tar, small chain log, advanced one pass small out, 41669 github.tar, explicit params, advanced one pass small out, 41227 github.tar, uncompressed literals, advanced one pass small out, 41122 -github.tar, uncompressed literals optimal, advanced one pass small out, 35388 +github.tar, uncompressed literals optimal, advanced one pass small out, 35397 github.tar, huffman literals, advanced one pass small out, 38853 github.tar, multithreaded with advanced params, advanced one pass small out, 41122 silesia, level -5, advanced streaming, 7292053 @@ -884,8 +884,8 @@ silesia, level 11 row 2, advanced silesia, level 12 row 1, advanced streaming, 4503116 silesia, level 12 row 2, advanced streaming, 4505153 silesia, level 13, advanced streaming, 4493990 -silesia, level 16, advanced streaming, 4360251 -silesia, level 19, advanced streaming, 4283237 +silesia, level 16, advanced streaming, 4359864 +silesia, level 19, advanced streaming, 4296880 silesia, no source size, advanced streaming, 4849517 silesia, long distance mode, advanced streaming, 4840737 silesia, multithreaded, advanced streaming, 4849553 @@ -895,7 +895,7 @@ silesia, small hash log, advanced silesia, small chain log, advanced streaming, 4912197 silesia, explicit params, advanced streaming, 4795883 silesia, uncompressed literals, advanced streaming, 5127982 -silesia, uncompressed literals optimal, advanced streaming, 4317896 +silesia, uncompressed literals optimal, advanced streaming, 4319518 silesia, huffman literals, advanced streaming, 5332234 silesia, multithreaded with advanced params, advanced streaming, 5127982 silesia.tar, level -5, advanced streaming, 7260007 @@ -918,8 +918,8 @@ silesia.tar, level 11 row 2, advanced silesia.tar, level 12 row 1, advanced streaming, 4513604 silesia.tar, level 12 row 2, advanced streaming, 4514569 silesia.tar, level 13, advanced streaming, 4502956 -silesia.tar, level 16, advanced streaming, 4356834 -silesia.tar, level 19, advanced streaming, 4264388 +silesia.tar, level 16, advanced streaming, 4360527 +silesia.tar, level 19, advanced streaming, 4267266 silesia.tar, no source size, advanced streaming, 4861422 silesia.tar, long distance mode, advanced streaming, 4847752 silesia.tar, multithreaded, advanced streaming, 4861508 @@ -929,7 +929,7 @@ silesia.tar, small hash log, advanced silesia.tar, small chain log, advanced streaming, 4917021 silesia.tar, explicit params, advanced streaming, 4807401 silesia.tar, uncompressed literals, advanced streaming, 5129461 -silesia.tar, uncompressed literals optimal, advanced streaming, 4307400 +silesia.tar, uncompressed literals optimal, advanced streaming, 4310141 silesia.tar, huffman literals, advanced streaming, 5350519 silesia.tar, multithreaded with advanced params, advanced streaming, 5129555 github, level -5, advanced streaming, 232315 @@ -1152,23 +1152,23 @@ github.tar, level 12 row 2 with dict dds, advanced github.tar, level 12 row 2 with dict copy, advanced streaming, 36609 github.tar, level 12 row 2 with dict load, advanced streaming, 36460 github.tar, level 13, advanced streaming, 35501 -github.tar, level 13 with dict, advanced streaming, 38726 -github.tar, level 13 with dict dms, advanced streaming, 38903 -github.tar, level 13 with dict dds, advanced streaming, 38903 -github.tar, level 13 with dict copy, advanced streaming, 38726 +github.tar, level 13 with dict, advanced streaming, 37130 +github.tar, level 13 with dict dms, advanced streaming, 37267 +github.tar, level 13 with dict dds, advanced streaming, 37267 +github.tar, level 13 with dict copy, advanced streaming, 37130 github.tar, level 13 with dict load, advanced streaming, 36010 -github.tar, level 16, advanced streaming, 40255 -github.tar, level 16 with dict, advanced streaming, 33639 -github.tar, level 16 with dict dms, advanced streaming, 33544 -github.tar, level 16 with dict dds, advanced streaming, 33544 -github.tar, level 16 with dict copy, advanced streaming, 33639 -github.tar, level 16 with dict load, advanced streaming, 39353 -github.tar, level 19, advanced streaming, 32837 -github.tar, level 19 with dict, advanced streaming, 32895 -github.tar, level 19 with dict dms, advanced streaming, 32672 -github.tar, level 19 with dict dds, advanced streaming, 32672 -github.tar, level 19 with dict copy, advanced streaming, 32895 -github.tar, level 19 with dict load, advanced streaming, 32676 +github.tar, level 16, advanced streaming, 40471 +github.tar, level 16 with dict, advanced streaming, 33378 +github.tar, level 16 with dict dms, advanced streaming, 33213 +github.tar, level 16 with dict dds, advanced streaming, 33213 +github.tar, level 16 with dict copy, advanced streaming, 33378 +github.tar, level 16 with dict load, advanced streaming, 39081 +github.tar, level 19, advanced streaming, 32134 +github.tar, level 19 with dict, advanced streaming, 32709 +github.tar, level 19 with dict dms, advanced streaming, 32553 +github.tar, level 19 with dict dds, advanced streaming, 32553 +github.tar, level 19 with dict copy, advanced streaming, 32709 +github.tar, level 19 with dict load, advanced streaming, 32474 github.tar, no source size, advanced streaming, 38438 github.tar, no source size with dict, advanced streaming, 38000 github.tar, long distance mode, advanced streaming, 39757 @@ -1179,7 +1179,7 @@ github.tar, small hash log, advanced github.tar, small chain log, advanced streaming, 41669 github.tar, explicit params, advanced streaming, 41227 github.tar, uncompressed literals, advanced streaming, 41122 -github.tar, uncompressed literals optimal, advanced streaming, 35388 +github.tar, uncompressed literals optimal, advanced streaming, 35397 github.tar, huffman literals, advanced streaming, 38874 github.tar, multithreaded with advanced params, advanced streaming, 41122 silesia, level -5, old streaming, 7292053 @@ -1194,11 +1194,11 @@ silesia, level 6, old stre silesia, level 7, old streaming, 4567204 silesia, level 9, old streaming, 4543310 silesia, level 13, old streaming, 4493990 -silesia, level 16, old streaming, 4360251 -silesia, level 19, old streaming, 4283237 +silesia, level 16, old streaming, 4359864 +silesia, level 19, old streaming, 4296880 silesia, no source size, old streaming, 4849517 silesia, uncompressed literals, old streaming, 4849553 -silesia, uncompressed literals optimal, old streaming, 4283237 +silesia, uncompressed literals optimal, old streaming, 4296880 silesia, huffman literals, old streaming, 6183923 silesia.tar, level -5, old streaming, 7260007 silesia.tar, level -3, old streaming, 6845151 @@ -1212,11 +1212,11 @@ silesia.tar, level 6, old stre silesia.tar, level 7, old streaming, 4576831 silesia.tar, level 9, old streaming, 4552590 silesia.tar, level 13, old streaming, 4502956 -silesia.tar, level 16, old streaming, 4356834 -silesia.tar, level 19, old streaming, 4264388 +silesia.tar, level 16, old streaming, 4360527 +silesia.tar, level 19, old streaming, 4267266 silesia.tar, no source size, old streaming, 4861422 silesia.tar, uncompressed literals, old streaming, 4861426 -silesia.tar, uncompressed literals optimal, old streaming, 4264388 +silesia.tar, uncompressed literals optimal, old streaming, 4267266 silesia.tar, huffman literals, old streaming, 6187938 github, level -5, old streaming, 232315 github, level -5 with dict, old streaming, 46718 @@ -1274,15 +1274,15 @@ github.tar, level 7 with dict, old stre github.tar, level 9, old streaming, 36767 github.tar, level 9 with dict, old streaming, 36457 github.tar, level 13, old streaming, 35501 -github.tar, level 13 with dict, old streaming, 38726 -github.tar, level 16, old streaming, 40255 -github.tar, level 16 with dict, old streaming, 33639 -github.tar, level 19, old streaming, 32837 -github.tar, level 19 with dict, old streaming, 32895 +github.tar, level 13 with dict, old streaming, 37130 +github.tar, level 16, old streaming, 40471 +github.tar, level 16 with dict, old streaming, 33378 +github.tar, level 19, old streaming, 32134 +github.tar, level 19 with dict, old streaming, 32709 github.tar, no source size, old streaming, 38438 github.tar, no source size with dict, old streaming, 38000 github.tar, uncompressed literals, old streaming, 38441 -github.tar, uncompressed literals optimal, old streaming, 32837 +github.tar, uncompressed literals optimal, old streaming, 32134 github.tar, huffman literals, old streaming, 42536 silesia, level -5, old streaming advanced, 7292053 silesia, level -3, old streaming advanced, 6867875 @@ -1296,8 +1296,8 @@ silesia, level 6, old stre silesia, level 7, old streaming advanced, 4567204 silesia, level 9, old streaming advanced, 4543310 silesia, level 13, old streaming advanced, 4493990 -silesia, level 16, old streaming advanced, 4360251 -silesia, level 19, old streaming advanced, 4283237 +silesia, level 16, old streaming advanced, 4359864 +silesia, level 19, old streaming advanced, 4296880 silesia, no source size, old streaming advanced, 4849517 silesia, long distance mode, old streaming advanced, 4849553 silesia, multithreaded, old streaming advanced, 4849553 @@ -1307,7 +1307,7 @@ silesia, small hash log, old stre silesia, small chain log, old streaming advanced, 4912197 silesia, explicit params, old streaming advanced, 4795883 silesia, uncompressed literals, old streaming advanced, 4849553 -silesia, uncompressed literals optimal, old streaming advanced, 4283237 +silesia, uncompressed literals optimal, old streaming advanced, 4296880 silesia, huffman literals, old streaming advanced, 6183923 silesia, multithreaded with advanced params, old streaming advanced, 4849553 silesia.tar, level -5, old streaming advanced, 7260007 @@ -1322,8 +1322,8 @@ silesia.tar, level 6, old stre silesia.tar, level 7, old streaming advanced, 4576831 silesia.tar, level 9, old streaming advanced, 4552590 silesia.tar, level 13, old streaming advanced, 4502956 -silesia.tar, level 16, old streaming advanced, 4356834 -silesia.tar, level 19, old streaming advanced, 4264388 +silesia.tar, level 16, old streaming advanced, 4360527 +silesia.tar, level 19, old streaming advanced, 4267266 silesia.tar, no source size, old streaming advanced, 4861422 silesia.tar, long distance mode, old streaming advanced, 4861426 silesia.tar, multithreaded, old streaming advanced, 4861426 @@ -1333,7 +1333,7 @@ silesia.tar, small hash log, old stre silesia.tar, small chain log, old streaming advanced, 4917021 silesia.tar, explicit params, old streaming advanced, 4807401 silesia.tar, uncompressed literals, old streaming advanced, 4861426 -silesia.tar, uncompressed literals optimal, old streaming advanced, 4264388 +silesia.tar, uncompressed literals optimal, old streaming advanced, 4267266 silesia.tar, huffman literals, old streaming advanced, 6187938 silesia.tar, multithreaded with advanced params, old streaming advanced, 4861426 github, level -5, old streaming advanced, 241214 @@ -1401,10 +1401,10 @@ github.tar, level 9, old stre github.tar, level 9 with dict, old streaming advanced, 36233 github.tar, level 13, old streaming advanced, 35501 github.tar, level 13 with dict, old streaming advanced, 35807 -github.tar, level 16, old streaming advanced, 40255 -github.tar, level 16 with dict, old streaming advanced, 38736 -github.tar, level 19, old streaming advanced, 32837 -github.tar, level 19 with dict, old streaming advanced, 32876 +github.tar, level 16, old streaming advanced, 40471 +github.tar, level 16 with dict, old streaming advanced, 38578 +github.tar, level 19, old streaming advanced, 32134 +github.tar, level 19 with dict, old streaming advanced, 32702 github.tar, no source size, old streaming advanced, 38438 github.tar, no source size with dict, old streaming advanced, 38015 github.tar, long distance mode, old streaming advanced, 38441 @@ -1415,7 +1415,7 @@ github.tar, small hash log, old stre github.tar, small chain log, old streaming advanced, 41669 github.tar, explicit params, old streaming advanced, 41227 github.tar, uncompressed literals, old streaming advanced, 38441 -github.tar, uncompressed literals optimal, old streaming advanced, 32837 +github.tar, uncompressed literals optimal, old streaming advanced, 32134 github.tar, huffman literals, old streaming advanced, 42536 github.tar, multithreaded with advanced params, old streaming advanced, 38441 github, level -5 with dict, old streaming cdict, 46718 @@ -1445,8 +1445,8 @@ github.tar, level 6 with dict, old stre github.tar, level 7 with dict, old streaming cdict, 37371 github.tar, level 9 with dict, old streaming cdict, 36352 github.tar, level 13 with dict, old streaming cdict, 36010 -github.tar, level 16 with dict, old streaming cdict, 39353 -github.tar, level 19 with dict, old streaming cdict, 32676 +github.tar, level 16 with dict, old streaming cdict, 39081 +github.tar, level 19 with dict, old streaming cdict, 32474 github.tar, no source size with dict, old streaming cdict, 38000 github, level -5 with dict, old streaming advanced cdict, 49562 github, level -3 with dict, old streaming advanced cdict, 44956 @@ -1475,6 +1475,6 @@ github.tar, level 6 with dict, old stre github.tar, level 7 with dict, old streaming advanced cdict, 37322 github.tar, level 9 with dict, old streaming advanced cdict, 36233 github.tar, level 13 with dict, old streaming advanced cdict, 35807 -github.tar, level 16 with dict, old streaming advanced cdict, 38736 -github.tar, level 19 with dict, old streaming advanced cdict, 32876 +github.tar, level 16 with dict, old streaming advanced cdict, 38578 +github.tar, level 19 with dict, old streaming advanced cdict, 32702 github.tar, no source size with dict, old streaming advanced cdict, 38015