-
Notifications
You must be signed in to change notification settings - Fork 118
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add support for Zstd for on-disk data compression
In measurements, zstd was as fast or even faster than gzip to decompress, and it provides a small but pretty nice reduction in tarball size for the larger icon tarballs. We can't make the switch to zstd for a while, but this change lies the groundwork to make this possible in future.
- Loading branch information
Showing
11 changed files
with
262 additions
and
25 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,126 @@ | ||
| /* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- | ||
| * | ||
| * Copyright (C) 2022 Richard Hughes <[email protected]> | ||
| * | ||
| * Licensed under the GNU Lesser General Public License Version 2.1 | ||
| * | ||
| * This library is free software: you can redistribute it and/or modify | ||
| * it under the terms of the GNU Lesser General Public License as published by | ||
| * the Free Software Foundation, either version 2.1 of the license, or | ||
| * (at your option) any later version. | ||
| * | ||
| * This library is distributed in the hope that it will be useful, | ||
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| * GNU Lesser General Public License for more details. | ||
| * | ||
| * You should have received a copy of the GNU Lesser General Public License | ||
| * along with this library. If not, see <http://www.gnu.org/licenses/>. | ||
| */ | ||
|
|
||
| #include "as-zstd-decompressor.h" | ||
|
|
||
| #include "config.h" | ||
|
|
||
| #include <gio/gio.h> | ||
| #include <zstd.h> | ||
|
|
||
| static void as_zstd_decompressor_iface_init (GConverterIface *iface); | ||
|
|
||
| struct _AsZstdDecompressor { | ||
| GObject parent_instance; | ||
| ZSTD_DStream *zstdstream; | ||
| }; | ||
|
|
||
| G_DEFINE_TYPE_WITH_CODE (AsZstdDecompressor, | ||
| as_zstd_decompressor, | ||
| G_TYPE_OBJECT, | ||
| G_IMPLEMENT_INTERFACE (G_TYPE_CONVERTER, as_zstd_decompressor_iface_init)) | ||
|
|
||
| static void | ||
| as_zstd_decompressor_finalize (GObject *object) | ||
| { | ||
| AsZstdDecompressor *self = AS_ZSTD_DECOMPRESSOR (object); | ||
| ZSTD_freeDStream (self->zstdstream); | ||
| G_OBJECT_CLASS (as_zstd_decompressor_parent_class)->finalize (object); | ||
| } | ||
|
|
||
| static void | ||
| as_zstd_decompressor_init (AsZstdDecompressor *self) | ||
| { | ||
| } | ||
|
|
||
| static void | ||
| as_zstd_decompressor_constructed (GObject *object) | ||
| { | ||
| AsZstdDecompressor *self = AS_ZSTD_DECOMPRESSOR (object); | ||
| self->zstdstream = ZSTD_createDStream (); | ||
| } | ||
|
|
||
| static void | ||
| as_zstd_decompressor_class_init (AsZstdDecompressorClass *klass) | ||
| { | ||
| GObjectClass *object_class = G_OBJECT_CLASS (klass); | ||
| object_class->finalize = as_zstd_decompressor_finalize; | ||
| object_class->constructed = as_zstd_decompressor_constructed; | ||
| } | ||
|
|
||
| AsZstdDecompressor * | ||
| as_zstd_decompressor_new (void) | ||
| { | ||
| return g_object_new (AS_TYPE_ZSTD_DECOMPRESSOR, NULL); | ||
| } | ||
|
|
||
| static void | ||
| as_zstd_decompressor_reset (GConverter *converter) | ||
| { | ||
| AsZstdDecompressor *self = AS_ZSTD_DECOMPRESSOR (converter); | ||
| ZSTD_initDStream (self->zstdstream); | ||
| } | ||
|
|
||
| static GConverterResult | ||
| as_zstd_decompressor_convert (GConverter *converter, | ||
| const void *inbuf, | ||
| gsize inbuf_size, | ||
| void *outbuf, | ||
| gsize outbuf_size, | ||
| GConverterFlags flags, | ||
| gsize *bytes_read, | ||
| gsize *bytes_written, | ||
| GError **error) | ||
| { | ||
| AsZstdDecompressor *self = AS_ZSTD_DECOMPRESSOR (converter); | ||
| ZSTD_outBuffer output = { | ||
| .dst = outbuf, | ||
| .size = outbuf_size, | ||
| .pos = 0, | ||
| }; | ||
| ZSTD_inBuffer input = { | ||
| .src = inbuf, | ||
| .size = inbuf_size, | ||
| .pos = 0, | ||
| }; | ||
| size_t res; | ||
|
|
||
| res = ZSTD_decompressStream (self->zstdstream, &output, &input); | ||
| if (res == 0) | ||
| return G_CONVERTER_FINISHED; | ||
| if (ZSTD_isError (res)) { | ||
| g_set_error (error, | ||
| G_IO_ERROR, | ||
| G_IO_ERROR_INVALID_DATA, | ||
| "cannot decompress data: %s", | ||
| ZSTD_getErrorName (res)); | ||
| return G_CONVERTER_ERROR; | ||
| } | ||
| *bytes_read = input.pos; | ||
| *bytes_written = output.pos; | ||
| return G_CONVERTER_CONVERTED; | ||
| } | ||
|
|
||
| static void | ||
| as_zstd_decompressor_iface_init (GConverterIface *iface) | ||
| { | ||
| iface->convert = as_zstd_decompressor_convert; | ||
| iface->reset = as_zstd_decompressor_reset; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| /* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- | ||
| * | ||
| * Copyright (C) 2022 Richard Hughes <[email protected]> | ||
| * | ||
| * Licensed under the GNU Lesser General Public License Version 2.1 | ||
| * | ||
| * This library is free software: you can redistribute it and/or modify | ||
| * it under the terms of the GNU Lesser General Public License as published by | ||
| * the Free Software Foundation, either version 2.1 of the license, or | ||
| * (at your option) any later version. | ||
| * | ||
| * This library is distributed in the hope that it will be useful, | ||
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| * GNU Lesser General Public License for more details. | ||
| * | ||
| * You should have received a copy of the GNU Lesser General Public License | ||
| * along with this library. If not, see <http://www.gnu.org/licenses/>. | ||
| */ | ||
|
|
||
| #pragma once | ||
|
|
||
| #include <glib-object.h> | ||
|
|
||
| #include "as-macros-private.h" | ||
|
|
||
| AS_BEGIN_PRIVATE_DECLS | ||
|
|
||
| #define AS_TYPE_ZSTD_DECOMPRESSOR (as_zstd_decompressor_get_type ()) | ||
| G_DECLARE_FINAL_TYPE (AsZstdDecompressor, as_zstd_decompressor, AS, ZSTD_DECOMPRESSOR, GObject) | ||
|
|
||
| AsZstdDecompressor *as_zstd_decompressor_new (void); | ||
|
|
||
| AS_END_PRIVATE_DECLS |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file not shown.
Oops, something went wrong.