From e20b4215e800b32004dab5c42d13db0c0c603bbc Mon Sep 17 00:00:00 2001 From: Mark Harmstone Date: Tue, 19 Nov 2024 15:26:49 +0000 Subject: [PATCH] btrfs-progs: mkfs: add lzo to --compress option Allow --compress to work with lzo. Signed-off-by: Mark Harmstone --- Documentation/mkfs.btrfs.rst | 2 +- mkfs/main.c | 2 +- mkfs/rootdir.c | 178 ++++++++++++++++++++++++++++++++++- 3 files changed, 178 insertions(+), 4 deletions(-) diff --git a/Documentation/mkfs.btrfs.rst b/Documentation/mkfs.btrfs.rst index 175970e93..b830f8c5b 100644 --- a/Documentation/mkfs.btrfs.rst +++ b/Documentation/mkfs.btrfs.rst @@ -214,7 +214,7 @@ OPTIONS --compress [:] Try to compress files when using *--rootdir*. Supported values for *algo* are - *no* (the default), *zlib*, and *zstd*. The optional value *level* is a + *no* (the default), *zlib*, *lzo*, and *zstd*. The optional value *level* is a compression level, from 1 to 9 for ZLIB and from 1 to 15 for ZSTD. As with the kernel, :command:`mkfs.btrfs` won't write compressed extents when diff --git a/mkfs/main.c b/mkfs/main.c index 89a78db88..cdb9862a9 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -443,7 +443,7 @@ static const char * const mkfs_usage[] = { OPTLINE("-u|--subvol TYPE:SUBDIR", "create SUBDIR as subvolume rather than normal directory, can be specified multiple times"), OPTLINE("--shrink", "(with --rootdir) shrink the filled filesystem to minimal size"), OPTLINE("-K|--nodiscard", "do not perform whole device TRIM"), - OPTLINE("--compress ALGO[:LEVEL]", "compression algorithm and level to use; ALGO can be no (default), zlib, zstd"), + OPTLINE("--compress ALGO[:LEVEL]", "compression algorithm and level to use; ALGO can be no (default), zlib, lzo, zstd"), OPTLINE("-f|--force", "force overwrite of existing filesystem"), "General:", OPTLINE("-q|--quiet", "no messages except errors"), diff --git a/mkfs/rootdir.c b/mkfs/rootdir.c index e057fe88e..29164880e 100644 --- a/mkfs/rootdir.c +++ b/mkfs/rootdir.c @@ -32,6 +32,7 @@ #include #endif #include +#include #include "kernel-lib/sizes.h" #include "kernel-shared/accessors.h" #include "kernel-shared/uapi/btrfs_tree.h" @@ -57,6 +58,8 @@ #define ZSTD_BTRFS_DEFAULT_LEVEL 3 #define ZSTD_BTRFS_MAX_LEVEL 15 +#define LZO_LEN 4 + static u32 fs_block_size; /* @@ -525,6 +528,61 @@ static ssize_t zlib_compress_extent(struct btrfs_inode_item *btrfs_inode, return 0; } +static ssize_t lzo_compress_extent(struct btrfs_inode_item *btrfs_inode, + u32 sectorsize, const void *in_buf, + size_t in_size, void *out_buf, char *wrkmem) +{ + int ret; + unsigned int sectors; + u32 total_size, out_pos; + + out_pos = LZO_LEN; + total_size = LZO_LEN; + sectors = DIV_ROUND_UP(in_size, sectorsize); + + for (unsigned int i = 0; i < sectors; i++) { + size_t in_len, out_len, new_pos; + u32 padding; + + in_len = min((size_t)sectorsize, in_size - (i * sectorsize)); + + ret = lzo1x_1_compress(in_buf + (i * sectorsize), in_len, + out_buf + out_pos + LZO_LEN, &out_len, + wrkmem); + if (ret) { + error("lzo1x_1_compress returned %i", ret); + return -EINVAL; + } + + put_unaligned_le32(out_len, out_buf + out_pos); + + new_pos = out_pos + LZO_LEN + out_len; + + /* Make sure that our header doesn't cross a sector boundary. */ + if (new_pos / sectorsize != (new_pos + LZO_LEN - 1) / sectorsize) + padding = round_up(new_pos, LZO_LEN) - new_pos; + else + padding = 0; + + out_pos += out_len + LZO_LEN + padding; + total_size += out_len + LZO_LEN + padding; + + /* + * Follow kernel in trying to compress the first three sectors, + * then giving up if the output isn't any smaller. + */ + if (i >= 3 && total_size > i * sectorsize) + return -E2BIG; + } + + if (total_size > in_size) + return -E2BIG; + + put_unaligned_le32(total_size, out_buf); + + return total_size; +} + #if COMPRESSION_ZSTD static ssize_t zstd_compress_extent(struct btrfs_inode_item *btrfs_inode, u32 sectorsize, const void *in_buf, @@ -629,6 +687,7 @@ struct source_descriptor { u64 size; const char *path_name; char *comp_buf; + char *wrkmem; }; static int add_file_item_extent(struct btrfs_trans_handle *trans, @@ -681,6 +740,12 @@ static int add_file_item_extent(struct btrfs_trans_handle *trans, source->buf, bytes_read, source->comp_buf); break; + case BTRFS_COMPRESS_LZO: + comp_ret = lzo_compress_extent(btrfs_inode, sectorsize, + source->buf, bytes_read, + source->comp_buf, + source->wrkmem); + break; #if COMPRESSION_ZSTD case BTRFS_COMPRESS_ZSTD: comp_ret = zstd_compress_extent(btrfs_inode, sectorsize, @@ -750,6 +815,11 @@ static int add_file_item_extent(struct btrfs_trans_handle *trans, features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD; btrfs_set_super_incompat_flags(trans->fs_info->super_copy, features); + } else if (g_compression == BTRFS_COMPRESS_LZO) { + features = btrfs_super_incompat_flags(trans->fs_info->super_copy); + features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO; + btrfs_set_super_incompat_flags(trans->fs_info->super_copy, + features); } } else { to_write = round_up(to_read, sectorsize); @@ -848,6 +918,58 @@ static int zlib_compress_inline_extent(char *buf, u64 size, char **comp_buf, return ret; } +static u32 lzo_max_outlen(u32 inlen) { + /* + * Return the worst-case output length for LZO. Formula comes from + * LZO.FAQ. + */ + return inlen + (inlen / 16) + 64 + 3; +} + +static int lzo_compress_inline_extent(void *buf, u64 size, char **comp_buf, + u64 *comp_size, char *wrkmem) +{ + int ret; + size_t out_len, out_size; + void *out = NULL; + + out_size = LZO_LEN + LZO_LEN + lzo_max_outlen(size); + + out = malloc(out_size); + if (!out) { + error_msg(ERROR_MSG_MEMORY, NULL); + ret = -ENOMEM; + goto out; + } + + ret = lzo1x_1_compress(buf, size, out + LZO_LEN + LZO_LEN, &out_len, + wrkmem); + if (ret) { + error("lzo1x_1_compress returned %i", ret); + ret = -EINVAL; + goto out; + } + + if (out_len + LZO_LEN + LZO_LEN >= size) { + ret = 0; + goto out; + } + + put_unaligned_le32(out_len + LZO_LEN + LZO_LEN, out); + put_unaligned_le32(out_len, out + LZO_LEN); + + *comp_buf = out; + *comp_size = out_len + LZO_LEN + LZO_LEN; + + ret = 1; + +out: + if (ret != 1) + free(out); + + return ret; +} + #if COMPRESSION_ZSTD static int zstd_compress_inline_extent(char *buf, u64 size, char **comp_buf, u64 *comp_size) @@ -934,7 +1056,7 @@ static int add_file_items(struct btrfs_trans_handle *trans, ssize_t ret_read; u32 sectorsize = fs_info->sectorsize; u64 file_pos = 0; - char *buf = NULL, *comp_buf = NULL; + char *buf = NULL, *comp_buf = NULL, *wrkmem = NULL; struct source_descriptor source; int fd; @@ -947,6 +1069,14 @@ static int add_file_items(struct btrfs_trans_handle *trans, return ret; } + if (g_compression == BTRFS_COMPRESS_LZO) { + wrkmem = malloc(LZO1X_1_MEM_COMPRESS); + if (!wrkmem) { + ret = -ENOMEM; + goto end; + } + } + if (st->st_size <= BTRFS_MAX_INLINE_DATA_SIZE(fs_info) && st->st_size < sectorsize) { char *buffer = malloc(st->st_size); @@ -972,6 +1102,13 @@ static int add_file_items(struct btrfs_trans_handle *trans, if (ret < 0) goto end; break; + case BTRFS_COMPRESS_LZO: + ret = lzo_compress_inline_extent(buffer, st->st_size, + &comp_buf, &comp_size, + wrkmem); + if (ret < 0) + goto end; + break; #if COMPRESSION_ZSTD case BTRFS_COMPRESS_ZSTD: ret = zstd_compress_inline_extent(buffer, st->st_size, @@ -1008,7 +1145,41 @@ static int add_file_items(struct btrfs_trans_handle *trans, goto end; } - if (g_compression != BTRFS_COMPRESS_NONE) { + if (g_compression == BTRFS_COMPRESS_LZO) { + unsigned int sectors; + size_t comp_buf_len; + + /* + * LZO helpfully doesn't provide a way to specify the output + * buffer size, so we need to allocate for the worst-case + * scenario to avoid buffer overruns. + * + * 4 bytes for the total size + * And for each sector: + * - 4 bytes for the compressed sector size + * - the worst-case output size + * - 3 bytes for possible padding + */ + + sectors = BTRFS_MAX_COMPRESSED / sectorsize; + + comp_buf_len = LZO_LEN; + comp_buf_len += (LZO_LEN + lzo_max_outlen(sectorsize) + + LZO_LEN - 1) * sectors; + + comp_buf = malloc(comp_buf_len); + if (!comp_buf) { + ret = -ENOMEM; + goto end; + } + + ret = lzo_init(); + if (ret) { + error("lzo_init returned %i", ret); + ret = -EINVAL; + goto end; + } + } else if (g_compression != BTRFS_COMPRESS_NONE) { comp_buf = malloc(BTRFS_MAX_COMPRESSED); if (!comp_buf) { ret = -ENOMEM; @@ -1021,6 +1192,7 @@ static int add_file_items(struct btrfs_trans_handle *trans, source.size = st->st_size; source.path_name = path_name; source.comp_buf = comp_buf; + source.wrkmem = wrkmem; while (file_pos < st->st_size) { ret = add_file_item_extent(trans, root, btrfs_inode, objectid, @@ -1032,6 +1204,7 @@ static int add_file_items(struct btrfs_trans_handle *trans, } end: + free(wrkmem); free(comp_buf); free(buf); close(fd); @@ -1452,6 +1625,7 @@ int btrfs_mkfs_fill_dir(struct btrfs_trans_handle *trans, const char *source_dir switch (compression) { case BTRFS_COMPRESS_NONE: + case BTRFS_COMPRESS_LZO: break; case BTRFS_COMPRESS_ZLIB: if (compression_level > ZLIB_BTRFS_MAX_LEVEL)