Merge pull request #296 from Gottox/fix/big-files

file: use 64bit indexing for block size
Gottox · Sep 5, 2024 · a2c1f41 · a2c1f41
2 parents 058077b + 829b56b
commit a2c1f41
Show file tree

Hide file tree

Showing 12 changed files with 79 additions and 35 deletions.
diff --git a/common/include/sqsh_data_private.h b/common/include/sqsh_data_private.h
@@ -358,7 +358,7 @@ sqsh__data_inode_file_block_offset(const struct SqshDataInodeFile *file);
 SQSH_NO_EXPORT uint32_t
 sqsh__data_inode_file_size(const struct SqshDataInodeFile *file);
 SQSH_NO_EXPORT uint32_t sqsh__data_inode_file_block_size_info(
-		const struct SqshDataInodeFile *file, sqsh_index_t index);
+		const struct SqshDataInodeFile *file, uint64_t index);
 
 SQSH_NO_EXPORT uint64_t sqsh__data_inode_file_ext_blocks_start(
 		const struct SqshDataInodeFileExt *file_ext);
@@ -375,7 +375,7 @@ SQSH_NO_EXPORT uint32_t sqsh__data_inode_file_ext_block_offset(
 SQSH_NO_EXPORT uint32_t sqsh__data_inode_file_ext_xattr_idx(
 		const struct SqshDataInodeFileExt *file_ext);
 SQSH_NO_EXPORT uint32_t sqsh__data_inode_file_ext_block_size_info(
-		const struct SqshDataInodeFileExt *file_ext, sqsh_index_t index);
+		const struct SqshDataInodeFileExt *file_ext, uint64_t index);
 
 SQSH_NO_EXPORT uint32_t sqsh__data_inode_symlink_hard_link_count(
 		const struct SqshDataInodeSymlink *directory);

diff --git a/common/src/data/inode_data.c b/common/src/data/inode_data.c
@@ -78,7 +78,7 @@ sqsh__data_inode_file_size(const struct SqshDataInodeFile *file) {
 }
 uint32_t
 sqsh__data_inode_file_block_size_info(
-		const struct SqshDataInodeFile *file, sqsh_index_t index) {
+		const struct SqshDataInodeFile *file, uint64_t index) {
 	const struct {
 		uint32_t b;
 	} SQSH_UNALIGNED *block_sizes = (const void *)&file[1];
@@ -120,7 +120,7 @@ sqsh__data_inode_file_ext_xattr_idx(
 }
 uint32_t
 sqsh__data_inode_file_ext_block_size_info(
-		const struct SqshDataInodeFileExt *file_ext, sqsh_index_t index) {
+		const struct SqshDataInodeFileExt *file_ext, uint64_t index) {
 	const struct {
 		uint32_t b;
 	} SQSH_UNALIGNED *block_sizes = (const void *)&file_ext[1];

diff --git a/include/sqsh_file.h b/include/sqsh_file.h
@@ -472,9 +472,25 @@ uint64_t sqsh_file_block_count2(const struct SqshFile *context);
  *
  * @return the size of the block with the index.
  */
-uint32_t sqsh_file_block_size(const struct SqshFile *context, uint32_t index);
+uint32_t sqsh_file_block_size2(const struct SqshFile *context, uint64_t index);
 
 /**
+ * @deprecated Since 1.6.0. Use sqsh_file_block_size2() instead.
+ * @memberof SqshFile
+ * @brief Getter the size of a block of the file content. This is only
+ * internally used and will be used while retrieving the file content.
+ *
+ * @param[in] context The file context.
+ * @param index The index of the block.
+ *
+ * @return the size of the block with the index.
+ */
+__attribute__((deprecated("Since 1.6.0. Use sqsh_file_block_size2() instead.")))
+uint32_t
+sqsh_file_block_size(const struct SqshFile *context, uint32_t index);
+
+/**
+ * @deprecated Since 1.6.0. Use sqsh_file_block_is_compressed2() instead.
  * @memberof SqshFile
  * @brief Checks whether a certain block is compressed.
  *
@@ -483,9 +499,22 @@ uint32_t sqsh_file_block_size(const struct SqshFile *context, uint32_t index);
  *
  * @return true if the block is compressed, false otherwise.
  */
-bool
+__attribute__((deprecated(
+		"Since 1.6.0. Use sqsh_file_block_is_compressed2() instead."))) bool
 sqsh_file_block_is_compressed(const struct SqshFile *context, uint32_t index);
 
+/**
+ * @memberof SqshFile
+ * @brief Checks whether a certain block is compressed.
+ *
+ * @param[in] context The file context.
+ * @param index The index of the block.
+ *
+ * @return true if the block is compressed, false otherwise.
+ */
+bool
+sqsh_file_block_is_compressed2(const struct SqshFile *context, uint64_t index);
+
 /**
  * @memberof SqshFile
  * @brief retrieve the fragment block index. This is only internally used

diff --git a/libsqsh/include/sqsh_file_private.h b/libsqsh/include/sqsh_file_private.h
@@ -147,7 +147,7 @@ struct SqshFileIterator {
 	struct SqshFragmentView fragment_view;
 	size_t sparse_size;
 	size_t block_size;
-	uint32_t block_index;
+	uint64_t block_index;
 	const uint8_t *data;
 	size_t size;
 };
@@ -249,7 +249,7 @@ struct SqshInodeImpl {
 
 	uint64_t (*blocks_start)(const struct SqshDataInode *inode);
 	uint32_t (*block_size_info)(
-			const struct SqshDataInode *inode, sqsh_index_t index);
+			const struct SqshDataInode *inode, uint64_t index);
 	uint32_t (*fragment_block_index)(const struct SqshDataInode *inode);
 	uint32_t (*fragment_block_offset)(const struct SqshDataInode *inode);
 
@@ -384,7 +384,7 @@ SQSH_NO_EXPORT uint64_t
 sqsh__file_inode_null_blocks_start(const struct SqshDataInode *inode);
 
 SQSH_NO_EXPORT uint32_t sqsh__file_inode_null_block_size_info(
-		const struct SqshDataInode *inode, sqsh_index_t index);
+		const struct SqshDataInode *inode, uint64_t index);
 
 SQSH_NO_EXPORT uint32_t
 sqsh__file_inode_null_fragment_block_index(const struct SqshDataInode *inode);

diff --git a/libsqsh/src/extract/lzma.c b/libsqsh/src/extract/lzma.c
@@ -109,7 +109,7 @@ sqsh_lzma_finish(void *context, uint8_t *target, size_t *target_size) {
 
 	lzma_ret ret = lzma_code(stream, LZMA_FINISH);
 
-	*target_size = stream->total_out;
+	*target_size = (size_t)stream->total_out;
 	lzma_end(stream);
 
 	if (ret == LZMA_STREAM_END) {

diff --git a/libsqsh/src/file/file.c b/libsqsh/src/file/file.c
@@ -309,6 +309,14 @@ sqsh_file_block_count(const struct SqshFile *context) {
 	return (uint32_t)block_count;
 }
 
+uint32_t
+sqsh_file_block_size2(const struct SqshFile *context, uint64_t index) {
+	const uint32_t size_info =
+			context->impl->block_size_info(get_inode(context), index);
+
+	return sqsh_datablock_size(size_info);
+}
+
 uint32_t
 sqsh_file_block_size(const struct SqshFile *context, uint32_t index) {
 	const uint32_t size_info =
@@ -317,6 +325,14 @@ sqsh_file_block_size(const struct SqshFile *context, uint32_t index) {
 	return sqsh_datablock_size(size_info);
 }
 
+bool
+sqsh_file_block_is_compressed2(const struct SqshFile *context, uint64_t index) {
+	const uint32_t size_info =
+			context->impl->block_size_info(get_inode(context), index);
+
+	return sqsh_datablock_is_compressed(size_info);
+}
+
 bool
 sqsh_file_block_is_compressed(const struct SqshFile *context, uint32_t index) {
 	const uint32_t size_info =

diff --git a/libsqsh/src/file/file_iterator.c b/libsqsh/src/file/file_iterator.c
@@ -45,7 +45,7 @@ static bool
 is_last_block(const struct SqshFileIterator *iterator) {
 	const struct SqshFile *file = iterator->file;
 	const bool has_fragment = sqsh_file_has_fragment(file);
-	const sqsh_index_t block_index = iterator->block_index;
+	const uint64_t block_index = iterator->block_index;
 	const uint64_t block_count = sqsh_file_block_count2(file);
 
 	if (has_fragment) {
@@ -135,9 +135,8 @@ map_block_compressed(
 			iterator->compression_manager;
 	const struct SqshFile *file = iterator->file;
 	struct SqshExtractView *extract_view = &iterator->extract_view;
-	const uint32_t block_index = iterator->block_index;
-	const sqsh_index_t data_block_size =
-			sqsh_file_block_size(file, block_index);
+	const uint64_t block_index = iterator->block_index;
+	const uint32_t data_block_size = sqsh_file_block_size2(file, block_index);
 
 	rv = sqsh__map_reader_advance(
 			&iterator->map_reader, next_offset, data_block_size);
@@ -168,22 +167,22 @@ map_block_uncompressed(
 		size_t desired_size) {
 	int rv = 0;
 	const struct SqshFile *file = iterator->file;
-	uint32_t block_index = iterator->block_index;
+	uint64_t block_index = iterator->block_index;
 	struct SqshMapReader *reader = &iterator->map_reader;
 	const uint64_t block_count = sqsh_file_block_count2(file);
 	size_t outer_size = 0;
 	const size_t remaining_direct = sqsh__map_reader_remaining_direct(reader);
 
 	for (; iterator->sparse_size == 0 && block_index < block_count;
 		 block_index++) {
-		if (sqsh_file_block_is_compressed(file, block_index)) {
+		if (sqsh_file_block_is_compressed2(file, block_index)) {
 			break;
 		}
 		if (outer_size >= desired_size) {
 			break;
 		}
 		const uint32_t data_block_size =
-				sqsh_file_block_size(file, block_index);
+				sqsh_file_block_size2(file, block_index);
 		/* Set the sparse size only if we are not at the last block. */
 		if (block_index + 1 != block_count) {
 			iterator->sparse_size = iterator->block_size - data_block_size;
@@ -241,11 +240,12 @@ map_block(struct SqshFileIterator *iterator, size_t desired_size) {
 	int rv = 0;
 	const struct SqshFile *file = iterator->file;
 
-	const uint32_t block_index = iterator->block_index;
+	const uint64_t block_index = iterator->block_index;
 	const size_t block_size = iterator->block_size;
-	const bool is_compressed = sqsh_file_block_is_compressed(file, block_index);
+	const bool is_compressed =
+			sqsh_file_block_is_compressed2(file, block_index);
 	const uint64_t file_size = sqsh_file_size(file);
-	const size_t data_block_size = sqsh_file_block_size(file, block_index);
+	const size_t data_block_size = sqsh_file_block_size2(file, block_index);
 	const sqsh_index_t next_offset =
 			sqsh__map_reader_size(&iterator->map_reader);
 
@@ -374,10 +374,10 @@ sqsh_file_iterator_skip2(
 	}
 
 	sqsh_index_t reader_forward = 0;
-	uint32_t block_index = iterator->block_index;
+	uint64_t block_index = iterator->block_index;
 	const uint64_t block_count = sqsh_file_block_count2(iterator->file);
 	for (sqsh_index_t i = 0; i < skip_index && block_index < block_count; i++) {
-		reader_forward += sqsh_file_block_size(iterator->file, block_index);
+		reader_forward += sqsh_file_block_size2(iterator->file, block_index);
 		block_index += 1;
 	}
 	rv = sqsh__map_reader_advance(&iterator->map_reader, reader_forward, 0);

diff --git a/libsqsh/src/file/inode_file.c b/libsqsh/src/file/inode_file.c
@@ -117,15 +117,14 @@ inode_file_ext_blocks_start(const struct SqshDataInode *inode) {
 }
 
 static uint32_t
-inode_file_block_size_info(
-		const struct SqshDataInode *inode, sqsh_index_t index) {
+inode_file_block_size_info(const struct SqshDataInode *inode, uint64_t index) {
 	return sqsh__data_inode_file_block_size_info(
 			sqsh__data_inode_file(inode), index);
 }
 
 static uint32_t
 inode_file_ext_block_size_info(
-		const struct SqshDataInode *inode, sqsh_index_t index) {
+		const struct SqshDataInode *inode, uint64_t index) {
 	return sqsh__data_inode_file_ext_block_size_info(
 			sqsh__data_inode_file_ext(inode), index);
 }

diff --git a/libsqsh/src/file/inode_null.c b/libsqsh/src/file/inode_null.c
@@ -70,7 +70,7 @@ sqsh__file_inode_null_blocks_start(const struct SqshDataInode *inode) {
 
 uint32_t
 sqsh__file_inode_null_block_size_info(
-		const struct SqshDataInode *inode, sqsh_index_t index) {
+		const struct SqshDataInode *inode, uint64_t index) {
 	(void)inode;
 	(void)index;
 	return UINT32_MAX;

diff --git a/tools/src/fs2.c b/tools/src/fs2.c
@@ -165,7 +165,7 @@ fs_open(const char *path, struct fuse_file_info *fi) {
 		goto out;
 	}
 
-	fi->fh = (uint64_t)file;
+	fi->fh = (uintptr_t)file;
 
 out:
 	if (rv < 0) {
@@ -179,7 +179,7 @@ fs_read(const char *path, char *buf, size_t size, off_t offset,
 		struct fuse_file_info *fi) {
 	(void)path;
 	int rv = 0;
-	struct SqshFile *file = (struct SqshFile *)fi->fh;
+	struct SqshFile *file = (struct SqshFile *)(uintptr_t)fi->fh;
 	struct SqshFileReader *reader = NULL;
 
 	rv = fs_common_read(&reader, file, offset, size);
@@ -201,7 +201,7 @@ fs_read(const char *path, char *buf, size_t size, off_t offset,
 static int
 fs_release(const char *path, struct fuse_file_info *fi) {
 	(void)path;
-	struct SqshFile *file = (struct SqshFile *)fi->fh;
+	struct SqshFile *file = (struct SqshFile *)(uintptr_t)fi->fh;
 
 	sqsh_close(file);
 	return 0;

diff --git a/tools/src/fs3.c b/tools/src/fs3.c
@@ -325,7 +325,7 @@ fs_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) {
 static void
 fs_release(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) {
 	(void)ino;
-	struct SqshFile *file = (void *)fi->fh;
+	struct SqshFile *file = (struct SqshFile *)(uintptr_t)fi->fh;
 
 	sqsh_close(file);
 	fuse_reply_err(req, 0);
@@ -335,7 +335,7 @@ static void
 fs_read(fuse_req_t req, fuse_ino_t ino, size_t size, off_t offset,
 		struct fuse_file_info *fi) {
 	(void)ino;
-	struct SqshFile *file = (void *)fi->fh;
+	struct SqshFile *file = (struct SqshFile *)(uintptr_t)fi->fh;
 	int rv = 0;
 	struct SqshFileReader *reader = NULL;
 

diff --git a/tools/src/stat.c b/tools/src/stat.c
@@ -294,11 +294,11 @@ stat_file(struct SqshArchive *archive, const char *path) {
 		}
 		printf("   number of blocks: %" PRIu64 "\n",
 			   sqsh_file_block_count2(file));
-		for (uint32_t i = 0; i < sqsh_file_block_count2(file); i++) {
-			bool is_compressed = sqsh_file_block_is_compressed(file, i);
-			uint32_t size = sqsh_file_block_size(file, i);
+		for (uint64_t i = 0; i < sqsh_file_block_count2(file); i++) {
+			bool is_compressed = sqsh_file_block_is_compressed2(file, i);
+			uint32_t size = sqsh_file_block_size2(file, i);
 
-			printf("          % 9i - %i (compressed: %s)\n", i, size,
+			printf("          % 9" PRIi64 " - %i (compressed: %s)\n", i, size,
 				   is_compressed ? "yes" : "no");
 		}
 		break;