Skip to content

Commit

Permalink
fix(infalte): use smaller types in inflate struct, split up huffman t…
Browse files Browse the repository at this point in the history 
…able arrays to make struct smaller, make zlib level 0 if using rle, other minor tweaks
  • Loading branch information
@oyvindln
oyvindln committed Dec 1, 2024
1 parent 02a8857 commit c5f8f76
Show file tree
Hide file tree
Showing 2 changed files with 73 additions and 54 deletions.
8 changes: 5 additions & 3 deletions miniz_oxide/src/deflate/core.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -306,15 +306,15 @@ pub(crate) const MAX_MATCH_LEN: usize = 258;
const DEFAULT_FLAGS: u32 = NUM_PROBES[4] | TDEFL_WRITE_ZLIB_HEADER;

mod zlib {
use super::TDEFL_RLE_MATCHES;
use super::{TDEFL_FORCE_ALL_RAW_BLOCKS, TDEFL_RLE_MATCHES};

const DEFAULT_CM: u8 = 8;
const DEFAULT_CINFO: u8 = 7 << 4;
const _DEFAULT_FDICT: u8 = 0;
const DEFAULT_CMF: u8 = DEFAULT_CM | DEFAULT_CINFO;
// CMF used for RLE (technically it uses a window size of 0 but the lowest that can
// be specified in the header corresponds to a window size of 1 << (0 + 8) aka 256.
const MIN_CMF: u8 = DEFAULT_CM | 0;
const MIN_CMF: u8 = DEFAULT_CM; // | 0
/// The 16-bit value consisting of CMF and FLG must be divisible by this to be valid.
const FCHECK_DIVISOR: u8 = 31;

Expand Down Expand Up @@ -352,8 +352,10 @@ mod zlib {
}

const fn cmf_from_flags(flags: u32) -> u8 {
if flags & TDEFL_RLE_MATCHES == 0 {
if (flags & TDEFL_RLE_MATCHES == 0) && (flags & TDEFL_FORCE_ALL_RAW_BLOCKS == 0) {
DEFAULT_CMF
// If we are using RLE encoding or no compression the window bits can be set as the
// minimum.
} else {
MIN_CMF
}
Expand Down
119 changes: 68 additions & 51 deletions miniz_oxide/src/inflate/core.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@ pub const TINFL_LZ_DICT_SIZE: usize = 32_768;
/// A struct containing huffman code lengths and the huffman code tree used by the decompressor.
#[derive(Clone)]
struct HuffmanTable {
/// Length of the code at each index.
pub code_size: [u8; MAX_HUFF_SYMBOLS_0],
/// Fast lookup table for shorter huffman codes.
///
/// See `HuffmanTable::fast_lookup`.
Expand All @@ -30,7 +28,6 @@ struct HuffmanTable {
impl HuffmanTable {
const fn new() -> HuffmanTable {
HuffmanTable {
code_size: [0; MAX_HUFF_SYMBOLS_0],
look_up: [0; FAST_LOOKUP_SIZE as usize],
tree: [0; MAX_HUFF_TREE_SIZE],
}
Expand Down Expand Up @@ -81,11 +78,10 @@ impl HuffmanTable {
fn lookup(&self, bit_buf: BitBuffer) -> Option<(i32, u32)> {
let symbol = self.fast_lookup(bit_buf).into();
if symbol >= 0 {
if (symbol >> 9) as u32 != 0 {
Some((symbol, (symbol >> 9) as u32))
} else {
// Zero-length code.
None
let length = (symbol >> 9) as u32;
match length {
0 => None,
_ => Some((symbol, length)),
}
} else {
// We didn't get a symbol from the fast lookup table, so check the tree instead.
Expand All @@ -101,7 +97,7 @@ const MAX_HUFF_SYMBOLS_0: usize = 288;
/// The length of the second (distance) huffman table.
const MAX_HUFF_SYMBOLS_1: usize = 32;
/// The length of the last (huffman code length) huffman table.
// const _MAX_HUFF_SYMBOLS_2: usize = 19;
const MAX_HUFF_SYMBOLS_2: usize = 19;
/// The maximum length of a code that can be looked up in the fast lookup table.
const FAST_LOOKUP_BITS: u8 = 10;
/// The size of the fast lookup table.
Expand Down Expand Up @@ -167,6 +163,13 @@ type BitBuffer = u64;
#[cfg(not(target_pointer_width = "64"))]
type BitBuffer = u32;

/*
enum HuffmanTableType {
LiteralLength = 0,
Dist = 1,
Huffman = 2,
}*/

/// Main decompression struct.
///
#[derive(Clone)]
Expand All @@ -182,23 +185,26 @@ pub struct DecompressorOxide {
/// Adler32 checksum from the zlib header.
z_adler32: u32,
/// 1 if the current block is the last block, 0 otherwise.
finish: u32,
finish: u8,
/// The type of the current block.
block_type: u32,
block_type: u8,
/// 1 if the adler32 value should be checked.
check_adler32: u32,
/// Last match distance.
dist: u32,
/// Variable used for match length, symbols, and a number of other things.
counter: u32,
/// Number of extra bits for the last length or distance code.
num_extra: u32,
num_extra: u8,
/// Number of entries in each huffman table.
table_sizes: [u32; MAX_HUFF_TABLES],
table_sizes: [u16; MAX_HUFF_TABLES],
/// Buffer of input data.
bit_buf: BitBuffer,
/// Huffman tables.
tables: [HuffmanTable; MAX_HUFF_TABLES],
code_size_literal: [u8; MAX_HUFF_SYMBOLS_0],
code_size_dist: [u8; MAX_HUFF_SYMBOLS_1],
code_size_huffman: [u8; MAX_HUFF_SYMBOLS_2],
/// Raw block header.
raw_header: [u8; 4],
/// Huffman length codes.
Expand Down Expand Up @@ -245,6 +251,14 @@ impl DecompressorOxide {
pub(crate) const fn zlib_header(&self) -> (u32, u32) {
(self.z_header0, self.z_header1)
}

/*fn code_size_table(&mut self, table_num: u8) -> &mut [u8] {
match table_num {
0 => &mut self.code_size_literal,
1 => &mut self.code_size_dist,
_ => &mut self.code_size_huffman,
}
}*/
}

impl Default for DecompressorOxide {
Expand All @@ -271,6 +285,9 @@ impl Default for DecompressorOxide {
HuffmanTable::new(),
HuffmanTable::new(),
],
code_size_literal: [0; MAX_HUFF_SYMBOLS_0],
code_size_dist: [0; MAX_HUFF_SYMBOLS_1],
code_size_huffman: [0; MAX_HUFF_SYMBOLS_2],
raw_header: [0; 4],
len_codes: [0; MAX_HUFF_SYMBOLS_0 + MAX_HUFF_SYMBOLS_1 + 137],
}
Expand Down Expand Up @@ -670,11 +687,11 @@ fn undo_bytes(l: &mut LocalVars, max: u32) -> u32 {
fn start_static_table(r: &mut DecompressorOxide) {
r.table_sizes[LITLEN_TABLE] = 288;
r.table_sizes[DIST_TABLE] = 32;
memset(&mut r.tables[LITLEN_TABLE].code_size[0..144], 8);
memset(&mut r.tables[LITLEN_TABLE].code_size[144..256], 9);
memset(&mut r.tables[LITLEN_TABLE].code_size[256..280], 7);
memset(&mut r.tables[LITLEN_TABLE].code_size[280..288], 8);
memset(&mut r.tables[DIST_TABLE].code_size[0..32], 5);
memset(&mut r.code_size_literal[0..144], 8);
memset(&mut r.code_size_literal[144..256], 9);
memset(&mut r.code_size_literal[256..280], 7);
memset(&mut r.code_size_literal[280..288], 8);
memset(&mut r.code_size_dist[0..32], 5);
}

#[cfg(feature = "rustc-dep-of-std")]
Expand Down Expand Up @@ -706,20 +723,25 @@ fn reverse_bits(n: u32) -> u32 {
fn init_tree(r: &mut DecompressorOxide, l: &mut LocalVars) -> Option<Action> {
loop {
let bt = r.block_type as usize;
if bt >= r.tables.len() {
return None;
}

let code_sizes = match bt {
0 => &mut r.code_size_literal[..],
1 => &mut r.code_size_dist,
2 => &mut r.code_size_huffman,
_ => return None,
};
let table = &mut r.tables[bt];
let table_size = r.table_sizes[bt] as usize;
if table_size > table.code_size.len() {
return None;
}
let mut total_symbols = [0u32; 16];

let mut total_symbols = [0u16; 16];
let mut next_code = [0u32; 17];
memset(&mut table.look_up[..], 0);
memset(&mut table.tree[..], 0);

for &code_size in &table.code_size[..table_size] {
let table_size = r.table_sizes[bt] as usize;
if table_size > code_sizes.len() {
return None;
}
for &code_size in &code_sizes[..table_size] {
let cs = code_size as usize;
if cs >= total_symbols.len() {
return None;
Expand All @@ -728,15 +750,10 @@ fn init_tree(r: &mut DecompressorOxide, l: &mut LocalVars) -> Option<Action> {
}

let mut used_symbols = 0;
let mut total = 0;
for (ts, next) in total_symbols
.iter()
.copied()
.zip(next_code.iter_mut().skip(1))
.skip(1)
{
let mut total = 0u32;
for (&ts, next) in total_symbols.iter().zip(next_code[1..].iter_mut()).skip(1) {
used_symbols += ts;
total += ts;
total += u32::from(ts);
total <<= 1;
*next = total;
}
Expand All @@ -747,7 +764,7 @@ fn init_tree(r: &mut DecompressorOxide, l: &mut LocalVars) -> Option<Action> {

let mut tree_next = -1;
for symbol_index in 0..table_size {
let code_size = table.code_size[symbol_index];
let code_size = code_sizes[symbol_index];
if code_size == 0 || usize::from(code_size) >= next_code.len() {
continue;
}
Expand Down Expand Up @@ -822,6 +839,7 @@ fn init_tree(r: &mut DecompressorOxide, l: &mut LocalVars) -> Option<Action> {
}

l.counter = 0;

Some(Action::Jump(DecodeLitlen))
}

Expand Down Expand Up @@ -1203,7 +1221,7 @@ pub fn decompress(
num_bits: r.num_bits,
dist: r.dist,
counter: r.counter,
num_extra: r.num_extra as u8,
num_extra: r.num_extra,
};

let mut status = 'state_machine: loop {
Expand Down Expand Up @@ -1242,8 +1260,8 @@ pub fn decompress(
// Read the block header and jump to the relevant section depending on the block type.
ReadBlockHeader => generate_state!(state, 'state_machine, {
read_bits(&mut l, 3, &mut in_iter, flags, |l, bits| {
r.finish = (bits & 1) as u32;
r.block_type = (bits >> 1) as u32 & 3;
r.finish = (bits & 1) as u8;
r.block_type = ((bits >> 1) & 3) as u8;
match r.block_type {
0 => Action::Jump(BlockTypeNoCompression),
1 => {
Expand Down Expand Up @@ -1375,12 +1393,12 @@ pub fn decompress(
let num_bits = [5, 5, 4][l.counter as usize];
read_bits(&mut l, num_bits, &mut in_iter, flags, |l, bits| {
r.table_sizes[l.counter as usize] =
bits as u32 + u32::from(MIN_TABLE_SIZES[l.counter as usize]);
bits as u16 + MIN_TABLE_SIZES[l.counter as usize];
l.counter += 1;
Action::None
})
} else {
memset(&mut r.tables[HUFFLEN_TABLE].code_size[..], 0);
memset(&mut r.code_size_huffman[..], 0);
l.counter = 0;
// Check that the litlen and distance are within spec.
// litlen table should be <=286 acc to the RFC and
Expand All @@ -1400,25 +1418,24 @@ pub fn decompress(
// Read the 3-bit lengths of the huffman codes describing the huffman code lengths used
// to decode the lengths of the main tables.
ReadHufflenTableCodeSize => generate_state!(state, 'state_machine, {
if l.counter < r.table_sizes[HUFFLEN_TABLE] {
if l.counter < r.table_sizes[HUFFLEN_TABLE].into() {
read_bits(&mut l, 3, &mut in_iter, flags, |l, bits| {
// These lengths are not stored in a normal ascending order, but rather one
// specified by the deflate specification intended to put the most used
// values at the front as trailing zero lengths do not have to be stored.
r.tables[HUFFLEN_TABLE]
.code_size[HUFFMAN_LENGTH_ORDER[l.counter as usize] as usize] =
r.code_size_huffman[HUFFMAN_LENGTH_ORDER[l.counter as usize] as usize] =
bits as u8;
l.counter += 1;
Action::None
})
} else {
r.table_sizes[HUFFLEN_TABLE] = 19;
r.table_sizes[HUFFLEN_TABLE] = MAX_HUFF_SYMBOLS_2 as u16;
init_tree(r, &mut l).unwrap_or(Action::End(TINFLStatus::Failed))
}
}),

ReadLitlenDistTablesCodeSize => generate_state!(state, 'state_machine, {
if l.counter < r.table_sizes[LITLEN_TABLE] + r.table_sizes[DIST_TABLE] {
if l.counter < u32::from(r.table_sizes[LITLEN_TABLE]) + u32::from(r.table_sizes[DIST_TABLE]) {
decode_huffman_code(
r, &mut l, HUFFLEN_TABLE,
flags, &mut in_iter, |r, l, symbol| {
Expand All @@ -1435,16 +1452,16 @@ pub fn decompress(
}
}
)
} else if l.counter != r.table_sizes[LITLEN_TABLE] + r.table_sizes[DIST_TABLE] {
} else if l.counter != u32::from(r.table_sizes[LITLEN_TABLE]) + u32::from(r.table_sizes[DIST_TABLE]) {
Action::Jump(BadCodeSizeSum)
} else {
r.tables[LITLEN_TABLE].code_size[..r.table_sizes[LITLEN_TABLE] as usize]
r.code_size_literal[..r.table_sizes[LITLEN_TABLE] as usize]
.copy_from_slice(&r.len_codes[..r.table_sizes[LITLEN_TABLE] as usize]);

let dist_table_start = r.table_sizes[LITLEN_TABLE] as usize;
let dist_table_end = (r.table_sizes[LITLEN_TABLE] +
r.table_sizes[DIST_TABLE]) as usize;
r.tables[DIST_TABLE].code_size[..r.table_sizes[DIST_TABLE] as usize]
r.code_size_dist[..r.table_sizes[DIST_TABLE] as usize]
.copy_from_slice(&r.len_codes[dist_table_start..dist_table_end]);

r.block_type -= 1;
Expand Down Expand Up @@ -1782,7 +1799,7 @@ pub fn decompress(
r.num_bits = l.num_bits;
r.dist = l.dist;
r.counter = l.counter;
r.num_extra = l.num_extra.into();
r.num_extra = l.num_extra;

r.bit_buf &= ((1 as BitBuffer) << r.num_bits) - 1;

Expand Down Expand Up @@ -1908,7 +1925,7 @@ mod test {
num_bits: d.num_bits,
dist: d.dist,
counter: d.counter,
num_extra: d.num_extra as u8,
num_extra: d.num_extra,
};
init_tree(&mut d, &mut l).unwrap();
let llt = &d.tables[LITLEN_TABLE];
Expand Down

0 comments on commit c5f8f76

Please sign in to comment.