From 66041f37467e9cc837ec32f92b7c5dbbf4116234 Mon Sep 17 00:00:00 2001 From: Ib Green Date: Tue, 23 May 2023 10:43:26 -0400 Subject: [PATCH] fix --- modules/3d-tiles/test/lib/utils/load-utils.ts | 9 +- modules/compression/src/index.ts | 14 ++- .../src/lib/brotli-compression-zlib.ts | 37 ++++--- .../compression/src/lib/brotli-compression.ts | 45 ++++----- modules/compression/src/lib/compression.ts | 31 +++++- ...ssion.ts => deflate-compression-fflate.ts} | 16 ++- .../src/lib/deflate-compression-pako.ts | 24 +++-- .../src/lib/deflate-compression-zlib.ts | 29 ++++-- ...pression.ts => gzip-compression-fflate.ts} | 40 +++++--- .../src/lib/gzip-compression-pako.ts | 52 +++++----- .../src/lib/gzip-compression-zlib.ts | 35 ++++--- .../compression/src/lib/zstd-compression.ts | 4 +- modules/compression/src/workers/worker.ts | 29 +++--- modules/compression/test/compression.bench.ts | 97 ++++++++++--------- yarn.lock | 10 +- 15 files changed, 271 insertions(+), 201 deletions(-) rename modules/compression/src/lib/{deflate-compression.ts => deflate-compression-fflate.ts} (86%) rename modules/compression/src/lib/{gzip-compression.ts => gzip-compression-fflate.ts} (75%) diff --git a/modules/3d-tiles/test/lib/utils/load-utils.ts b/modules/3d-tiles/test/lib/utils/load-utils.ts index 21d810854e..6618c3c310 100644 --- a/modules/3d-tiles/test/lib/utils/load-utils.ts +++ b/modules/3d-tiles/test/lib/utils/load-utils.ts @@ -3,22 +3,17 @@ import {fetchFile, load} from '@loaders.gl/core'; import {Tiles3DLoader} from '@loaders.gl/3d-tiles'; -import {Tileset3D} from '@loaders.gl/tiles'; - -/** @typedef {import('@loaders.gl/tiles').Tile3D} Tile3D */ +import {Tileset3D, Tile3D} from '@loaders.gl/tiles'; /** - * @returns {Promise} */ -export async function loadRootTile(t, tilesetUrl) { +export async function loadRootTile(t, tilesetUrl): Promise { try { // Load tileset const tilesetJson = await load(tilesetUrl, Tiles3DLoader); const tileset = new Tileset3D(tilesetJson, tilesetUrl); // Load root tile - /** @type {Tile3D} */ - // @ts-ignore const sourceRootTile = tileset.root as Tile3D; await tileset._loadTile(sourceRootTile); return sourceRootTile; diff --git a/modules/compression/src/index.ts b/modules/compression/src/index.ts index efe7d7a0ca..29b9a12cb7 100644 --- a/modules/compression/src/index.ts +++ b/modules/compression/src/index.ts @@ -4,10 +4,10 @@ export {Compression} from './lib/compression'; export {NoCompression} from './lib/no-compression'; -export {DeflateCompression} from './lib/deflate-compression'; -export {DeflateCompression as _DeflateCompressionZlib} from './lib/deflate-compression-zlib'; -export {GZipCompression} from './lib/gzip-compression'; -export {GZipCompression as _GZipCompressionZlib} from './lib/gzip-compression-zlib'; +export {DeflateCompression} from './lib/deflate-compression-pako'; +export {DeflateCompressionZlib} from './lib/deflate-compression-zlib'; +export {GZipCompression} from './lib/gzip-compression-pako'; +export {GZipCompressionZlib} from './lib/gzip-compression-zlib'; export {BrotliCompression} from './lib/brotli-compression'; export {BrotliCompressionZlib} from './lib/brotli-compression-zlib'; @@ -24,10 +24,8 @@ export type {CompressionWorkerOptions} from './compression-worker'; export {CompressionWorker, compressOnWorker} from './compression-worker'; // Versions -export {DeflateCompression as _DeflateCompressionFflate} from './lib/deflate-compression'; -export {GZipCompression as _GZipCompressionFflate} from './lib/gzip-compression'; +export {DeflateCompression as _DeflateCompressionFflate} from './lib/deflate-compression-fflate'; +export {GZipCompression as _GZipCompressionFflate} from './lib/gzip-compression-fflate'; export {DeflateCompression as _DeflateCompressionPako} from './lib/deflate-compression-pako'; export {GZipCompression as _GZipCompressionPako} from './lib/gzip-compression-pako'; - - diff --git a/modules/compression/src/lib/brotli-compression-zlib.ts b/modules/compression/src/lib/brotli-compression-zlib.ts index 7bc429cc36..32fde79d02 100644 --- a/modules/compression/src/lib/brotli-compression-zlib.ts +++ b/modules/compression/src/lib/brotli-compression-zlib.ts @@ -2,29 +2,26 @@ import type {CompressionOptions} from './compression'; import {Compression} from './compression'; import {isBrowser, toArrayBuffer} from '@loaders.gl/loader-utils'; -import zlib from 'zlib'; +import zlib, {BrotliOptions} from 'zlib'; import {promisify1, promisify2} from '@loaders.gl/loader-utils'; -export type BrotliCompressionOptions = CompressionOptions & { - brotli?: { - mode?: number; - quality?: number; - lgwin?: number; - useZlib?: boolean; - }; +export type BrotliCompressionZlibOptions = CompressionOptions & { + brotliZlib?: BrotliOptions; }; /** - * brotli compression / decompression using zlib + * brotli compression / decompression + * zlib implementation + * @note Node uses compression level 11 by default which is 100x slower!! */ export class BrotliCompressionZlib extends Compression { readonly name: string = 'brotli'; readonly extensions = ['br']; readonly contentEncodings = ['br']; readonly isSupported = true; - readonly options: BrotliCompressionOptions; + readonly options: BrotliCompressionZlibOptions; - constructor(options: BrotliCompressionOptions = {}) { + constructor(options: BrotliCompressionZlibOptions = {}) { super(options); this.options = options; if (isBrowser) { @@ -33,13 +30,15 @@ export class BrotliCompressionZlib extends Compression { } async compress(input: ArrayBuffer): Promise { - // @ts-expect-error overloads - Node uses compression level 11 by default which is 100x slower - const buffer = await promisify2(zlib.brotliCompress)(input, {params: {[zlib.constants.BROTLI_PARAM_QUALITY]: 4}}); + const options = this._getBrotliZlibOptions(); + // @ts-expect-error promisify type failure on overload + const buffer = await promisify2(zlib.brotliCompress)(input, options); return toArrayBuffer(buffer); } compressSync(input: ArrayBuffer): ArrayBuffer { - const buffer = zlib.brotliCompressSync(input, {params: {[zlib.constants.BROTLI_PARAM_QUALITY]: 4}}); + const options = this._getBrotliZlibOptions(); + const buffer = zlib.brotliCompressSync(input, options); return toArrayBuffer(buffer); } @@ -52,4 +51,14 @@ export class BrotliCompressionZlib extends Compression { const buffer = zlib.brotliDecompressSync(input); return toArrayBuffer(buffer); } + + private _getBrotliZlibOptions(): BrotliOptions { + // {params: {[zlib.constants.BROTLI_PARAM_QUALITY]: 4}} + return { + params: { + [zlib.constants.BROTLI_PARAM_QUALITY]: Compression.DEFAULT_COMPRESSION_LEVEL, + ...this.options?.brotliZlib + } + }; + } } diff --git a/modules/compression/src/lib/brotli-compression.ts b/modules/compression/src/lib/brotli-compression.ts index 417c351887..9e0688139d 100644 --- a/modules/compression/src/lib/brotli-compression.ts +++ b/modules/compression/src/lib/brotli-compression.ts @@ -1,33 +1,22 @@ // BROTLI import {isBrowser} from '@loaders.gl/loader-utils'; -import type {CompressionOptions} from './compression'; import {Compression} from './compression'; -import {BrotliCompressionZlib} from './brotli-compression-zlib'; +import {BrotliCompressionZlib, BrotliCompressionZlibOptions} from './brotli-compression-zlib'; import type brotliNamespace from 'brotli'; -// import brotli from 'brotli'; // https://bundlephobia.com/package/brotli +import type {BrotliOptions} from 'brotli'; +// import brotli from 'brotli'; // import {BrotliDecode} from '../brotli/decode'; -export type BrotliCompressionOptions = CompressionOptions & { - brotli?: { - mode?: number; - quality?: number; - lgwin?: number; - useZlib?: boolean; - }; -}; - -const DEFAULT_BROTLI_OPTIONS = { - brotli: { - mode: 0, - quality: 8, - lgwin: 22 - } +export type BrotliCompressionOptions = BrotliCompressionZlibOptions & { + brotli?: {}; }; let brotli: typeof brotliNamespace; /** * brotli compression / decompression + * Implemented with brotli package + * @see https://bundlephobia.com/package/brotli */ export class BrotliCompression extends Compression { readonly name: string = 'brotli'; @@ -50,7 +39,8 @@ export class BrotliCompression extends Compression { // dependency injection brotli = brotli || this.options?.modules?.brotli || Compression.modules.brotli; - if (!isBrowser && this.options.brotli?.useZlib) { + if (!isBrowser && this.options.useZlib) { + // @ts-ignore public API is equivalent return new BrotliCompressionZlib(options); } } @@ -71,11 +61,9 @@ export class BrotliCompression extends Compression { if (!brotli) { throw new Error('brotli compression: brotli module not installed'); } - - const brotliOptions = {...DEFAULT_BROTLI_OPTIONS.brotli, ...this.options?.brotli}; + const options = this._getBrotliOptions(); const inputArray = new Uint8Array(input); - - const outputArray = brotli.compress(inputArray, {quality: 5, brotliOptions}); + const outputArray = brotli.compress(inputArray, options); return outputArray.buffer; } @@ -84,13 +72,20 @@ export class BrotliCompression extends Compression { throw new Error('brotli compression: brotli module not installed'); } - const brotliOptions = {...DEFAULT_BROTLI_OPTIONS.brotli, ...this.options?.brotli}; + const options = this._getBrotliOptions(); const inputArray = new Uint8Array(input); // @ts-ignore brotli types state that only Buffers are accepted... - const outputArray = brotli.decompress(inputArray, brotliOptions); + const outputArray = brotli.decompress(inputArray, options); return outputArray.buffer; // const outputArray = BrotliDecode(inputArray, undefined); // return outputArray.buffer; } + + private _getBrotliOptions(): BrotliOptions { + return { + level: this.options.quality || Compression.DEFAULT_COMPRESSION_LEVEL, + ...this.options?.brotli + }; + } } diff --git a/modules/compression/src/lib/compression.ts b/modules/compression/src/lib/compression.ts index 520f02c08f..d602c830b7 100644 --- a/modules/compression/src/lib/compression.ts +++ b/modules/compression/src/lib/compression.ts @@ -3,22 +3,41 @@ import {concatenateArrayBuffersAsync} from '@loaders.gl/loader-utils'; /** Compression options */ export type CompressionOptions = { - /** Compression quality typically goes from 1-11 (higher values better but slower) */ + /** + * Compression quality (higher values better compression but exponentially slower) + * brotli goes from 1-11 + * zlib goes from 1-9 + * 5 or 6 is usually a good compromise + */ quality?: number; - /** Injection of npm modules for large libraries */ + + /** + * Whether to use built-in Zlib on node.js for max performance (doesn't handle incremental compression) + * Currently only deflate, gzip and brotli are supported. + */ + useZlib?: boolean; + + /** + * Injection of npm modules - keeps large compression libraries out of standard bundle + */ modules?: CompressionModules; }; +/** + * Injection of npm modules - keeps large compression libraries out of standard bundle + */ export type CompressionModules = { brotli?: any; lz4js?: any; lzo?: any; 'zstd-codec'?: any; -} - +}; /** Compression */ export abstract class Compression { + /** Default compression level for gzip, brotli etc */ + static DEFAULT_COMPRESSION_LEVEL = 5; + /** Name of the compression */ abstract readonly name: string; /** File extensions used for this */ @@ -28,7 +47,9 @@ export abstract class Compression { /** Whether decompression is supported */ abstract readonly isSupported: boolean; /** Whether compression is supported */ - get isCompressionSupported(): boolean { return this.isSupported; }; + get isCompressionSupported(): boolean { + return this.isSupported; + } static modules: CompressionModules = {}; diff --git a/modules/compression/src/lib/deflate-compression.ts b/modules/compression/src/lib/deflate-compression-fflate.ts similarity index 86% rename from modules/compression/src/lib/deflate-compression.ts rename to modules/compression/src/lib/deflate-compression-fflate.ts index 4a07f57964..85152fc300 100644 --- a/modules/compression/src/lib/deflate-compression.ts +++ b/modules/compression/src/lib/deflate-compression-fflate.ts @@ -1,28 +1,34 @@ // loaders.gl, MIT license -import type {CompressionOptions} from './compression'; +import {isBrowser} from '@loaders.gl/loader-utils'; +import {DeflateCompressionZlib, DeflateCompressionZlibOptions} from './deflate-compression-zlib'; import {Compression} from './compression'; - +import type {DeflateOptions} from 'fflate'; import {deflateSync, inflateSync} from 'fflate'; -import type {DeflateOptions} from 'fflate'; // https://bundlephobia.com/package/pako -export type DeflateCompressionOptions = CompressionOptions & { +export type DeflateCompressionOptions = DeflateCompressionZlibOptions & { deflate?: DeflateOptions; }; /** * DEFLATE compression / decompression + * Implementation using fflate + * @see https://bundlephobia.com/package/fflate */ export class DeflateCompression extends Compression { readonly name: string = 'fflate'; readonly extensions: string[] = []; readonly contentEncodings = ['fflate', 'gzip, zlib']; - readonly isSupported = true; + readonly isSupported: boolean = true; readonly options: DeflateCompressionOptions; constructor(options: DeflateCompressionOptions = {}) { super(options); this.options = options; + if (!isBrowser && this.options.useZlib) { + // @ts-ignore public API is equivalent + return new DeflateCompressionZlib(options); + } } // Async fflate uses Workers which interferes with loaders.gl diff --git a/modules/compression/src/lib/deflate-compression-pako.ts b/modules/compression/src/lib/deflate-compression-pako.ts index bc8464bf54..c079ab6393 100644 --- a/modules/compression/src/lib/deflate-compression-pako.ts +++ b/modules/compression/src/lib/deflate-compression-pako.ts @@ -1,16 +1,18 @@ // loaders.gl, MIT license -import type {CompressionOptions} from './compression'; +import {isBrowser} from '@loaders.gl/loader-utils'; +import {DeflateCompressionZlib, DeflateCompressionZlibOptions} from './deflate-compression-zlib'; import {Compression} from './compression'; import {getPakoError} from './utils/pako-utils'; -import pako from 'pako'; // https://bundlephobia.com/package/pako +import pako from 'pako'; -export type DeflateCompressionOptions = CompressionOptions & { - deflate?: pako.InflateOptions & pako.DeflateOptions & {useZlib?: boolean}; +export type DeflateCompressionOptions = DeflateCompressionZlibOptions & { + deflate?: pako.InflateOptions & pako.DeflateOptions; }; /** * DEFLATE compression / decompression - * Using PAKO library + * Implementation using pako + * @see https://bundlephobia.com/package/pako */ export class DeflateCompression extends Compression { readonly name: string = 'deflate'; @@ -25,6 +27,10 @@ export class DeflateCompression extends Compression { constructor(options: DeflateCompressionOptions = {}) { super(options); this.options = options; + if (!isBrowser && this.options.useZlib) { + // @ts-ignore public API is equivalent + return new DeflateCompressionZlib(options); + } } async compress(input: ArrayBuffer): Promise { @@ -63,7 +69,7 @@ export class DeflateCompression extends Compression { yield* this.transformBatches(pakoProcessor, asyncIterator); } - async *transformBatches( + private async *transformBatches( pakoProcessor: pako.Inflate | pako.Deflate, asyncIterator: AsyncIterable | Iterable ): AsyncIterable { @@ -90,17 +96,17 @@ export class DeflateCompression extends Compression { yield* chunks; } - _onData(chunk) { + private _onData(chunk) { this._chunks.push(chunk); } - _onEnd(status) { + private _onEnd(status) { if (status !== 0) { throw new Error(getPakoError(status) + this._chunks.length); } } - _getChunks(): ArrayBuffer[] { + private _getChunks(): ArrayBuffer[] { const chunks = this._chunks; this._chunks = []; return chunks; diff --git a/modules/compression/src/lib/deflate-compression-zlib.ts b/modules/compression/src/lib/deflate-compression-zlib.ts index 900a760174..57cd9a9111 100644 --- a/modules/compression/src/lib/deflate-compression-zlib.ts +++ b/modules/compression/src/lib/deflate-compression-zlib.ts @@ -6,23 +6,23 @@ import {Compression} from './compression'; import * as zlib from 'zlib'; import type {ZlibOptions} from 'zlib'; -export type DeflateCompressionOptions = CompressionOptions & { - deflate?: ZlibOptions; +export type DeflateCompressionZlibOptions = CompressionOptions & { + deflateZlib?: ZlibOptions; }; /** * DEFLATE compression / decompression * Using Node.js zlib library (works under Node only) */ -export class DeflateCompression extends Compression { +export class DeflateCompressionZlib extends Compression { readonly name: string = 'deflate'; readonly extensions: string[] = []; readonly contentEncodings = ['deflate']; readonly isSupported = isBrowser; - readonly options: DeflateCompressionOptions; + readonly options: DeflateCompressionZlibOptions; - constructor(options: DeflateCompressionOptions = {}) { + constructor(options: DeflateCompressionZlibOptions = {}) { super(options); this.options = options; if (!isBrowser) { @@ -31,22 +31,33 @@ export class DeflateCompression extends Compression { } async compress(input: ArrayBuffer): Promise { - const buffer = await promisify2(zlib.deflate)(input, this.options.deflate || {}); + const options = this._getZlibOptions(); + const buffer = await promisify2(zlib.deflate)(input, options); return toArrayBuffer(buffer); } async decompress(input: ArrayBuffer): Promise { - const buffer = await promisify2(zlib.inflate)(input, this.options.deflate || {}); + const options = this._getZlibOptions(); + const buffer = await promisify2(zlib.inflate)(input, options); return toArrayBuffer(buffer); } compressSync(input: ArrayBuffer): ArrayBuffer { - const buffer = zlib.deflateSync(input, this.options.deflate || {}); + const options = this._getZlibOptions(); + const buffer = zlib.deflateSync(input, options); return toArrayBuffer(buffer); } decompressSync(input: ArrayBuffer): ArrayBuffer { - const buffer = zlib.inflateSync(input, this.options.deflate || {}); + const options = this._getZlibOptions(); + const buffer = zlib.inflateSync(input, options); return toArrayBuffer(buffer); } + + protected _getZlibOptions(): ZlibOptions { + return { + level: this.options.quality || Compression.DEFAULT_COMPRESSION_LEVEL, + ...this.options?.deflateZlib + }; + } } diff --git a/modules/compression/src/lib/gzip-compression.ts b/modules/compression/src/lib/gzip-compression-fflate.ts similarity index 75% rename from modules/compression/src/lib/gzip-compression.ts rename to modules/compression/src/lib/gzip-compression-fflate.ts index 743af74e79..f14f041a08 100644 --- a/modules/compression/src/lib/gzip-compression.ts +++ b/modules/compression/src/lib/gzip-compression-fflate.ts @@ -1,21 +1,24 @@ // loaders.gl, MIT license -import type {CompressionOptions} from './compression'; +import {isBrowser} from '@loaders.gl/loader-utils'; +import {GZipCompressionZlib, GZipCompressionZlibOptions} from './gzip-compression-zlib'; import {Compression} from './compression'; import type {GzipOptions, AsyncGzipOptions} from 'fflate'; import {gzipSync, gunzipSync, Gzip, Gunzip} from 'fflate'; // https://bundlephobia.com/package/pako -export type GZipCompressionOptions = CompressionOptions & { +export type GZipCompressionOptions = GZipCompressionZlibOptions & { gzip?: GzipOptions | AsyncGzipOptions; }; /** * GZIP compression / decompression + * Implementation using fflate + * @see https://bundlephobia.com/package/fflate */ export class GZipCompression extends Compression { readonly name: string = 'gzip'; readonly extensions = ['gz', 'gzip']; readonly contentEncodings = ['gzip', 'x-gzip']; - readonly isSupported = true; + readonly isSupported = true; readonly options: GZipCompressionOptions; private _chunks: ArrayBuffer[] = []; @@ -23,10 +26,13 @@ export class GZipCompression extends Compression { constructor(options: GZipCompressionOptions = {}) { super(options); this.options = options; + if (!isBrowser && this.options.useZlib) { + // @ts-ignore public API is equivalent + return new GZipCompressionZlib(options); + } } // Async fflate uses Workers which interferes with loaders.gl - // async compress(input: ArrayBuffer): Promise { // // const options = this.options?.gzip || {}; // const inputArray = new Uint8Array(input); @@ -34,6 +40,7 @@ export class GZipCompression extends Compression { // return outputArray.buffer; // } + // Async fflate uses Workers which interferes with loaders.gl // async decompress(input: ArrayBuffer): Promise { // // const options = this.options?.gzip || {}; // const inputArray = new Uint8Array(input); @@ -42,16 +49,12 @@ export class GZipCompression extends Compression { // } compressSync(input: ArrayBuffer): ArrayBuffer { - // @ts-expect-error level - const options: GzipOptions = this.options?.gzip || { - level: this.options.quality || 6 - }; + const options = this._getFflateOptions(); const inputArray = new Uint8Array(input); return gzipSync(inputArray, options).buffer; } decompressSync(input: ArrayBuffer): ArrayBuffer { - // const options = this.options?.gzip || {}; const inputArray = new Uint8Array(input); return gunzipSync(inputArray).buffer; } @@ -59,10 +62,7 @@ export class GZipCompression extends Compression { async *compressBatches( asyncIterator: AsyncIterable | Iterable ): AsyncIterable { - // @ts-expect-error level - const options: GzipOptions = this.options?.gzip || { - level: this.options.quality || 6 - }; + const options = this._getFflateOptions(); const streamProcessor = new Gzip(options); streamProcessor.ondata = this._onData.bind(this); yield* this.transformBatches(streamProcessor, asyncIterator); @@ -76,7 +76,7 @@ export class GZipCompression extends Compression { yield* this.transformBatches(streamProcessor, asyncIterator); } - protected async *transformBatches( + private async *transformBatches( streamProcessor: Gzip | Gunzip, asyncIterator: AsyncIterable | Iterable ): AsyncIterable { @@ -94,13 +94,21 @@ export class GZipCompression extends Compression { yield* chunks; } - _onData(data: Uint8Array, final: boolean): void { + private _onData(data: Uint8Array, final: boolean): void { this._chunks.push(data); } - _getChunks(): ArrayBuffer[] { + private _getChunks(): ArrayBuffer[] { const chunks = this._chunks; this._chunks = []; return chunks; } + + private _getFflateOptions(): GzipOptions { + return { + // @ts-ignore-error + level: this.options.quality || Compression.DEFAULT_COMPRESSION_LEVEL, + ...this.options?.gzip + }; + } } diff --git a/modules/compression/src/lib/gzip-compression-pako.ts b/modules/compression/src/lib/gzip-compression-pako.ts index ea095a88da..386e47b60d 100644 --- a/modules/compression/src/lib/gzip-compression-pako.ts +++ b/modules/compression/src/lib/gzip-compression-pako.ts @@ -1,20 +1,19 @@ // loaders.gl, MIT license -import type {CompressionOptions} from './compression'; +import {isBrowser} from '@loaders.gl/loader-utils'; +import {GZipCompressionZlib, GZipCompressionZlibOptions} from './gzip-compression-zlib'; + import {Compression} from './compression'; import {getPakoError} from './utils/pako-utils'; -import pako from 'pako'; // https://bundlephobia.com/package/pako - -export type DeflateCompressionOptions = CompressionOptions & { - deflate?: pako.InflateOptions & pako.DeflateOptions & {useZlib?: boolean}; -}; +import pako from 'pako'; -export type GZipCompressionOptions = CompressionOptions & { +export type GZipCompressionOptions = GZipCompressionZlibOptions & { gzip?: pako.InflateOptions & pako.DeflateOptions; }; /** * GZIP compression / decompression - * Using PAKO library. + * Implementation using pako + * @see https://bundlephobia.com/package/pako */ export class GZipCompression extends Compression { readonly name: string = 'deflate'; @@ -22,13 +21,17 @@ export class GZipCompression extends Compression { readonly contentEncodings = ['deflate']; readonly isSupported = true; - readonly options: DeflateCompressionOptions; + readonly options: GZipCompressionOptions; private _chunks: ArrayBuffer[] = []; - constructor(options: DeflateCompressionOptions = {}) { + constructor(options: GZipCompressionOptions = {}) { super(options); this.options = options; + if (!isBrowser && this.options.useZlib) { + // @ts-ignore public API is equivalent + return new GZipCompressionZlib(options); + } } async compress(input: ArrayBuffer): Promise { @@ -40,13 +43,13 @@ export class GZipCompression extends Compression { } compressSync(input: ArrayBuffer): ArrayBuffer { - const pakoOptions = this._getDeflateOptions(); + const pakoOptions = this._getPakoOptions(); const inputArray = new Uint8Array(input); return pako.gzip(inputArray, pakoOptions).buffer; } decompressSync(input: ArrayBuffer): ArrayBuffer { - const pakoOptions: pako.InflateOptions = this.options?.deflate || {}; + const pakoOptions: pako.InflateOptions = this.options?.gzip || {}; const inputArray = new Uint8Array(input); return pako.ungzip(inputArray, pakoOptions).buffer; } @@ -54,7 +57,7 @@ export class GZipCompression extends Compression { async *compressBatches( asyncIterator: AsyncIterable | Iterable ): AsyncIterable { - const pakoOptions = this._getDeflateOptions(); + const pakoOptions = this._getPakoOptions(); const pakoProcessor = new pako.Deflate(pakoOptions); yield* this.transformBatches(pakoProcessor, asyncIterator); } @@ -62,12 +65,12 @@ export class GZipCompression extends Compression { async *decompressBatches( asyncIterator: AsyncIterable | Iterable ): AsyncIterable { - const pakoOptions: pako.InflateOptions = this.options?.deflate || {}; + const pakoOptions: pako.InflateOptions = this.options?.gzip || {}; const pakoProcessor = new pako.Inflate(pakoOptions); yield* this.transformBatches(pakoProcessor, asyncIterator); } - async *transformBatches( + private async *transformBatches( pakoProcessor: pako.Inflate | pako.Deflate, asyncIterator: AsyncIterable | Iterable ): AsyncIterable { @@ -94,28 +97,27 @@ export class GZipCompression extends Compression { yield* chunks; } - protected _onData(chunk) { + private _onData(chunk) { this._chunks.push(chunk); } - protected _onEnd(status) { + private _onEnd(status) { if (status !== 0) { throw new Error(getPakoError(status) + this._chunks.length); } } - protected _getChunks(): ArrayBuffer[] { + private _getChunks(): ArrayBuffer[] { const chunks = this._chunks; this._chunks = []; return chunks; } - protected _getDeflateOptions(): pako.DeflateOptions { - const pakoOptions: pako.DeflateOptions = this.options?.deflate || {}; - if (this.options.quality) { - // @ts-expect-error - pakoOptions.level = this.options.quality || 5; - } - return pakoOptions; + private _getPakoOptions(): pako.DeflateOptions { + return { + // @ts-ignore level is too strongly typed + level: this.options.quality || Compression.DEFAULT_COMPRESSION_LEVEL, + ...this.options?.gzipZlib + }; } } diff --git a/modules/compression/src/lib/gzip-compression-zlib.ts b/modules/compression/src/lib/gzip-compression-zlib.ts index 78f913402c..44e8b556c7 100644 --- a/modules/compression/src/lib/gzip-compression-zlib.ts +++ b/modules/compression/src/lib/gzip-compression-zlib.ts @@ -3,26 +3,28 @@ import type {CompressionOptions} from './compression'; import {Compression} from './compression'; import {isBrowser, toArrayBuffer} from '@loaders.gl/loader-utils'; import {promisify2} from '@loaders.gl/loader-utils'; -import * as zlib from 'zlib'; import type {ZlibOptions} from 'zlib'; +import * as zlib from 'zlib'; -export type DeflateCompressionOptions = CompressionOptions & { - deflate?: ZlibOptions; +const DEFAULT_COMPRESSION_LEVEL = 6; + +export type GZipCompressionZlibOptions = CompressionOptions & { + gzipZlib?: ZlibOptions; }; /** * GZIP compression / decompression * Using Node.js zlib library (works under Node only) */ -export class GZipCompression extends Compression { +export class GZipCompressionZlib extends Compression { readonly name: string = 'gzip'; readonly extensions = ['gz', 'gzip']; readonly contentEncodings = ['gzip', 'x-gzip']; readonly isSupported = isBrowser; - readonly options: DeflateCompressionOptions; + readonly options: GZipCompressionZlibOptions; - constructor(options: DeflateCompressionOptions = {}) { + constructor(options: GZipCompressionZlibOptions = {}) { super(options); this.options = options; if (isBrowser) { @@ -31,28 +33,33 @@ export class GZipCompression extends Compression { } async compress(input: ArrayBuffer): Promise { - const options: ZlibOptions = this.options?.deflate || { - level: this.options.quality || 6 - }; + const options = this._getZlibOptions(); const buffer = await promisify2(zlib.gzip)(input, options); return toArrayBuffer(buffer); } async decompress(input: ArrayBuffer): Promise { - const buffer = await promisify2(zlib.gunzip)(input, this.options.deflate || {}); + const options = this._getZlibOptions(); + const buffer = await promisify2(zlib.gunzip)(input, options); return toArrayBuffer(buffer); } compressSync(input: ArrayBuffer): ArrayBuffer { - const options: ZlibOptions = this.options?.deflate || { - level: this.options.quality || 6 - }; + const options = this._getZlibOptions(); const buffer = zlib.gzipSync(input, options); return toArrayBuffer(buffer); } decompressSync(input: ArrayBuffer): ArrayBuffer { - const buffer = zlib.gunzipSync(input, this.options.deflate || {}); + const options = this._getZlibOptions(); + const buffer = zlib.gunzipSync(input, options); return toArrayBuffer(buffer); } + + protected _getZlibOptions(): ZlibOptions { + return { + level: this.options.quality || DEFAULT_COMPRESSION_LEVEL, + ...this.options?.gzipZlib + }; + } } diff --git a/modules/compression/src/lib/zstd-compression.ts b/modules/compression/src/lib/zstd-compression.ts index 54aa2cf8f7..0584465744 100644 --- a/modules/compression/src/lib/zstd-compression.ts +++ b/modules/compression/src/lib/zstd-compression.ts @@ -13,7 +13,9 @@ export class ZstdCompression extends Compression { readonly name: string = 'zstd'; readonly extensions = []; readonly contentEncodings = []; - get isSupported() { return zstd; }; + get isSupported() { + return zstd; + } readonly options: CompressionOptions; diff --git a/modules/compression/src/workers/worker.ts b/modules/compression/src/workers/worker.ts index b827cbe6a7..1665e43880 100644 --- a/modules/compression/src/workers/worker.ts +++ b/modules/compression/src/workers/worker.ts @@ -1,10 +1,12 @@ import {createWorker} from '@loaders.gl/worker-utils'; +import type {Compression} from '../lib/compression'; + // Compressors import {NoCompression} from '../lib/no-compression'; import {BrotliCompression} from '../lib/brotli-compression'; -import {DeflateCompression} from '../lib/deflate-compression'; -import {GZipCompression} from '../lib/gzip-compression'; +import {DeflateCompression} from '../lib/deflate-compression-pako'; +import {GZipCompression} from '../lib/gzip-compression-pako'; import {LZ4Compression} from '../lib/lz4-compression'; // import {LZOCompression} from '../lib/lzo-compression'; import {SnappyCompression} from '../lib/snappy-compression'; @@ -13,7 +15,7 @@ import {ZstdCompression} from '../lib/zstd-compression'; // Import big dependencies // import brotli from 'brotli'; - brotli has problems with decompress in browsers -// import brotliDecompress from 'brotli/decompress'; +import brotliDecompress from 'brotli/decompress'; import lz4js from 'lz4js'; // import lzo from 'lzo'; // import {ZstdCodec} from 'zstd-codec'; @@ -24,27 +26,26 @@ import lz4js from 'lz4js'; // Inject large dependencies through Compression constructor options const modules = { // brotli has problems with decompress in browsers - // brotli: { - // decompress: brotliDecompress, - // compress: () => { - // throw new Error('brotli compress'); - // } - // }, + brotli: { + decompress: brotliDecompress, + compress: () => { + throw new Error('brotli compress'); + } + }, lz4js + // 'zstd-codec': ZstdCodec, // lzo, - // 'zstd-codec': ZstdCodec }; -/** @type {Compression[]} */ -const COMPRESSIONS = [ +const COMPRESSIONS: Compression[] = [ new NoCompression({modules}), new BrotliCompression({modules}), new DeflateCompression({modules}), new GZipCompression({modules}), - // new LZOCompression({modules}), - new LZ4Compression({modules}), new SnappyCompression({modules}), + new LZ4Compression({modules}), new ZstdCompression({modules}) + // new LZOCompression({modules}) ]; createWorker(async (data, options = {}) => { diff --git a/modules/compression/test/compression.bench.ts b/modules/compression/test/compression.bench.ts index 3113dedd68..0914d6ab40 100644 --- a/modules/compression/test/compression.bench.ts +++ b/modules/compression/test/compression.bench.ts @@ -1,26 +1,27 @@ +import type {Bench} from '@probe.gl/bench'; import {isBrowser} from '@loaders.gl/loader-utils'; import { Compression, - NoCompression, - GZipCompression, + // NoCompression, // DeflateCompression, - LZ4Compression, - ZstdCompression, + // DeflateCompressionZlib, + GZipCompression, + GZipCompressionZlib, + // LZ4Compression, + // ZstdCompression, SnappyCompression, BrotliCompression, BrotliCompressionZlib, // LZOCompression, // CompressionWorker, _GZipCompressionFflate, - _DeflateCompressionFflate, - _GZipCompressionPako, - _DeflateCompressionPako, - _GZipCompressionZlib, - _DeflateCompressionZlib, + _GZipCompressionPako + // _DeflateCompressionFflate, + // _DeflateCompressionPako } from '@loaders.gl/compression'; import {getData} from './utils/test-utils'; -// import * as brotli from 'brotli-compress'; +// import * as brotli from 'brotli-compress'; // import brotliPromise, {BrotliWasmType} from 'brotli-wasm'; // Import the default export // let brotli: BrotliWasmType | undefined; @@ -32,7 +33,6 @@ import {ZstdCodec} from 'zstd-codec'; // Inject large dependencies through Compression constructor options Object.assign(Compression.modules, { lz4js, - // lzo, 'zstd-codec': ZstdCodec, // brotli module has big problems with compress in browsers brotli: { @@ -41,30 +41,57 @@ Object.assign(Compression.modules, { throw new Error('brotli compress'); } } + // lzo }); +// Prepare data +const {binaryData} = getData(); +const gzippedData = new GZipCompression().compressSync(binaryData); +const snappyData = new SnappyCompression().compressSync(binaryData); +let brotliData: ArrayBuffer | undefined; +try { + brotliData = new BrotliCompressionZlib().compressSync(binaryData); +} catch { + // ignore errors +} + // const noCompression = new NoCompression(); // const gzipFflate = new GZipCompression(); // const gzipPako = new _GZipCompressionPako(); -// const gzipZlib = new _GZipCompressionZlib(); +// const gzipZlib = new GZipCompressionZlib(); // const brotli = new BrotliCompression(); // const brotliZlib = new BrotliCompressionZlib(); // const snappy = new SnappyCompression(); // const lz4 = new LZ4Compression(); // const zstd = new ZstdCompression(); +export async function compressionBench(bench: Bench): Promise { + await compressionBenchDecompression(bench); -export default async function compressionBench(bench) { - const {binaryData} = getData(); - const gzippedData = new GZipCompression().compressSync(binaryData); - const snappyData = new SnappyCompression().compressSync(binaryData) - let brotliData: ArrayBuffer | undefined; - try { - brotliData = new BrotliCompressionZlib().compressSync(binaryData); - } catch { - // ignore errors - } + await compressionBenchCompression(bench); + + // bench = bench.addAsync( + // 'GZip - compress - Pako - async', + // {multiplier: binaryData.byteLength, unit: 'bytes'}, + // () => { + // new _GZipCompressionPako().compress(binaryData); + // } + // ); + // bench = bench.addAsync( + // 'GZip - compress - Fflate - async', + // {multiplier: binaryData.byteLength, unit: 'bytes'}, + // () => new _GZipCompressionFflate().compress(binaryData) + // ); + // bench = bench.addAsync( + // 'GZip - compress - Zlib - async', + // {multiplier: binaryData.byteLength, unit: 'bytes'}, + // () => new GZipCompressionZlib().compress(binaryData) + // ); + return bench; +} + +async function compressionBenchDecompression(bench: Bench): Promise { bench = bench.group('Decompression'); bench = bench.add( @@ -77,7 +104,7 @@ export default async function compressionBench(bench) { bench = bench.add( 'GZip - decompress - Zlib - sync', {multiplier: binaryData.byteLength, unit: 'bytes'}, - () => new _GZipCompressionZlib().compressSync(gzippedData) + () => new GZipCompressionZlib().compressSync(gzippedData) ); } @@ -106,7 +133,9 @@ export default async function compressionBench(bench) { () => new BrotliCompression().decompressSync(data) ); } +} +async function compressionBenchCompression(bench: Bench): Promise { bench = bench.group('Compression'); bench = bench.addAsync( @@ -119,7 +148,7 @@ export default async function compressionBench(bench) { bench = bench.add( 'GZip - compress 6 - Zlib - sync', {multiplier: binaryData.byteLength, unit: 'bytes'}, - () => new _GZipCompressionZlib().compressSync(binaryData) + () => new GZipCompressionZlib().compressSync(binaryData) ); } bench = bench.add( @@ -148,24 +177,4 @@ export default async function compressionBench(bench) { () => new BrotliCompression().compressSync(binaryData) ); } - - // bench = bench.addAsync( - // 'GZip - compress - Pako - async', - // {multiplier: binaryData.byteLength, unit: 'bytes'}, - // () => { - // new _GZipCompressionPako().compress(binaryData); - // } - // ); - // bench = bench.addAsync( - // 'GZip - compress - Fflate - async', - // {multiplier: binaryData.byteLength, unit: 'bytes'}, - // () => new _GZipCompressionFflate().compress(binaryData) - // ); - // bench = bench.addAsync( - // 'GZip - compress - Zlib - async', - // {multiplier: binaryData.byteLength, unit: 'bytes'}, - // () => new _GZipCompressionZlib().compress(binaryData) - // ); - - return bench; } diff --git a/yarn.lock b/yarn.lock index 95badcf71e..cf62fb8225 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1973,7 +1973,7 @@ resolved "https://registry.yarnpkg.com/@luma.gl/constants/-/constants-8.5.20.tgz#91de116f68110fb28a000b59747d34d54bc06ab2" integrity sha512-5yG+ybkUZ4j6kLPWMZjN4Hun2yLB0MyEpNCRKAUN9/yS9UIWA7unyVxjSf2vnE7k/7dywtxlbXegASNFgNVGxw== -"@luma.gl/core@^8.5.20": +"@luma.gl/core@8.5.20", "@luma.gl/core@^8.5.20": version "8.5.20" resolved "https://registry.yarnpkg.com/@luma.gl/core/-/core-8.5.20.tgz#8b6cea7b5d7230e8b2848c310fc092af2c652571" integrity sha512-xJr96G6vhYcznYHC84fbeOG3fgNM4lFwj9bd0VPcg/Kfe8otUeN1Hl0AKHCCtNn48PiMSg3LKbaiRfNUMhaffQ== @@ -12299,10 +12299,10 @@ typedarray@^0.0.6: resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" integrity sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA== -typescript@^4.3.4: - version "4.9.5" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.5.tgz#095979f9bcc0d09da324d58d03ce8f8374cbe65a" - integrity sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g== +typescript@^4.3.4, typescript@^5.0.4: + version "5.0.4" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.0.4.tgz#b217fd20119bd61a94d4011274e0ab369058da3b" + integrity sha512-cW9T5W9xY37cc+jfEnaUvX91foxtHkza3Nw3wkoF4sSlKn0MONdkdEndig/qPBWXNkmplh3NzayQzCiHM4/hqw== typical@^4.0.0: version "4.0.0"