feat(compression): Integrate fflate library

modules/3d-tiles/test/lib/utils/load-utils.ts

@@ -3,22 +3,17 @@
 
 import {fetchFile, load} from '@loaders.gl/core';
 import {Tiles3DLoader} from '@loaders.gl/3d-tiles';
-import {Tileset3D} from '@loaders.gl/tiles';
-
-/** @typedef {import('@loaders.gl/tiles').Tile3D} Tile3D */
+import {Tileset3D, Tile3D} from '@loaders.gl/tiles';
 
 /**
- * @returns {Promise<Tile3D>}
  */
-export async function loadRootTile(t, tilesetUrl) {
+export async function loadRootTile(t, tilesetUrl): Promise<Tile3D> {
   try {
     // Load tileset
     const tilesetJson = await load(tilesetUrl, Tiles3DLoader);
     const tileset = new Tileset3D(tilesetJson, tilesetUrl);
 
     // Load root tile
-    /** @type {Tile3D} */
-    // @ts-ignore
     const sourceRootTile = tileset.root as Tile3D;
     await tileset._loadTile(sourceRootTile);
     return sourceRootTile;

modules/compression/package.json

@@ -41,20 +41,21 @@
     "@babel/runtime": "^7.3.1",
     "@loaders.gl/loader-utils": "4.0.0-alpha.6",
     "@loaders.gl/worker-utils": "4.0.0-alpha.6",
-    "@types/brotli": "^1.3.0",
     "@types/pako": "^1.0.1",
+    "brotli": "^1.3.3",
+    "brotli-wasm": "^1.3.1",
     "fflate": "0.7.4",
     "lzo-wasm": "^0.0.4",
     "pako": "1.0.11",
     "snappyjs": "^0.6.1"
   },
   "optionalDependencies": {
-    "brotli": "^1.3.2",
+    "brotli-compress": "^1.3.3",
     "lz4js": "^0.2.0",
     "zstd-codec": "^0.1"
   },
   "devDependencies": {
-    "brotli": "^1.3.2",
+    "brotli-compress": "^1.3.3",
     "lz4js": "^0.2.0",
     "zstd-codec": "^0.1"
   },

modules/compression/src/index.ts

@@ -3,13 +3,29 @@ export type {CompressionOptions} from './lib/compression';
 export {Compression} from './lib/compression';
 
 export {NoCompression} from './lib/no-compression';
-export {DeflateCompression} from './lib/deflate-compression';
-export {GZipCompression} from './lib/gzip-compression';
+
+export {DeflateCompression} from './lib/deflate-compression-pako';
+export {DeflateCompressionZlib} from './lib/deflate-compression-zlib';
+export {GZipCompression} from './lib/gzip-compression-pako';
+export {GZipCompressionZlib} from './lib/gzip-compression-zlib';
+
 export {BrotliCompression} from './lib/brotli-compression';
+export {BrotliCompressionZlib} from './lib/brotli-compression-zlib';
+
 export {SnappyCompression} from './lib/snappy-compression';
+
 export {LZ4Compression} from './lib/lz4-compression';
+
 export {ZstdCompression} from './lib/zstd-compression';
+
 export {LZOCompression} from './lib/lzo-compression';
 
 export type {CompressionWorkerOptions} from './compression-worker';
 export {CompressionWorker, compressOnWorker} from './compression-worker';
+
+// Versions
+export {DeflateCompression as _DeflateCompressionFflate} from './lib/deflate-compression-fflate';
+export {GZipCompression as _GZipCompressionFflate} from './lib/gzip-compression-fflate';
+
+export {DeflateCompression as _DeflateCompressionPako} from './lib/deflate-compression-pako';
+export {GZipCompression as _GZipCompressionPako} from './lib/gzip-compression-pako';

modules/compression/src/lib/brotli-compression-zlib.ts

@@ -0,0 +1,64 @@
+// BROTLI
+import type {CompressionOptions} from './compression';
+import {Compression} from './compression';
+import {isBrowser, toArrayBuffer} from '@loaders.gl/loader-utils';
+import zlib, {BrotliOptions} from 'zlib';
+import {promisify1, promisify2} from '@loaders.gl/loader-utils';
+
+export type BrotliCompressionZlibOptions = CompressionOptions & {
+  brotliZlib?: BrotliOptions;
+};
+
+/**
+ * brotli compression / decompression
+ * zlib implementation
+ * @note Node uses compression level 11 by default which is 100x slower!!
+ */
+export class BrotliCompressionZlib extends Compression {
+  readonly name: string = 'brotli';
+  readonly extensions = ['br'];
+  readonly contentEncodings = ['br'];
+  readonly isSupported = true;
+  readonly options: BrotliCompressionZlibOptions;
+
+  constructor(options: BrotliCompressionZlibOptions = {}) {
+    super(options);
+    this.options = options;
+    if (isBrowser) {
+      throw new Error('zlib only available under Node.js');
+    }
+  }
+
+  async compress(input: ArrayBuffer): Promise<ArrayBuffer> {
+    const options = this._getBrotliZlibOptions();
+    // @ts-expect-error promisify type failure on overload
+    const buffer = await promisify2(zlib.brotliCompress)(input, options);
+    return toArrayBuffer(buffer);
+  }
+
+  compressSync(input: ArrayBuffer): ArrayBuffer {
+    const options = this._getBrotliZlibOptions();
+    const buffer = zlib.brotliCompressSync(input, options);
+    return toArrayBuffer(buffer);
+  }
+
+  async decompress(input: ArrayBuffer): Promise<ArrayBuffer> {
+    const buffer = await promisify1(zlib.brotliDecompress)(input);
+    return toArrayBuffer(buffer);
+  }
+
+  decompressSync(input: ArrayBuffer): ArrayBuffer {
+    const buffer = zlib.brotliDecompressSync(input);
+    return toArrayBuffer(buffer);
+  }
+
+  private _getBrotliZlibOptions(): BrotliOptions {
+    // {params: {[zlib.constants.BROTLI_PARAM_QUALITY]: 4}}
+    return {
+      params: {
+        [zlib.constants.BROTLI_PARAM_QUALITY]: Compression.DEFAULT_COMPRESSION_LEVEL,
+        ...this.options?.brotliZlib
+      }
+    };
+  }
+}

modules/compression/src/lib/brotli-compression.ts

@@ -1,111 +1,91 @@
 // BROTLI
-import type {CompressionOptions} from './compression';
+import {isBrowser} from '@loaders.gl/loader-utils';
 import {Compression} from './compression';
-import {isBrowser, toArrayBuffer} from '@loaders.gl/loader-utils';
+import {BrotliCompressionZlib, BrotliCompressionZlibOptions} from './brotli-compression-zlib';
 import type brotliNamespace from 'brotli';
-// import brotli from 'brotli';  // https://bundlephobia.com/package/brotli
-import {BrotliDecode} from '../brotli/decode';
-import zlib from 'zlib';
-import {promisify1} from '@loaders.gl/loader-utils';
+import type {BrotliOptions} from 'brotli';
+// import brotli from 'brotli';
+// import {BrotliDecode} from '../brotli/decode';
 
-export type BrotliCompressionOptions = CompressionOptions & {
-  brotli?: {
-    mode?: number;
-    quality?: number;
-    lgwin?: number;
-    useZlib?: boolean;
-  };
-};
-
-const DEFAULT_BROTLI_OPTIONS = {
-  brotli: {
-    mode: 0,
-    quality: 8,
-    lgwin: 22
-  }
+export type BrotliCompressionOptions = BrotliCompressionZlibOptions & {
+  brotli?: {};
 };
 
 let brotli: typeof brotliNamespace;
 
 /**
  * brotli compression / decompression
+ * Implemented with brotli package
+ * @see https://bundlephobia.com/package/brotli
  */
 export class BrotliCompression extends Compression {
   readonly name: string = 'brotli';
   readonly extensions = ['br'];
   readonly contentEncodings = ['br'];
-  readonly isSupported = true;
+
+  get isSupported() {
+    return brotli;
+  }
+  get isCompressionSupported() {
+    return false;
+  }
+
   readonly options: BrotliCompressionOptions;
 
-  constructor(options: BrotliCompressionOptions) {
+  constructor(options: BrotliCompressionOptions = {}) {
     super(options);
     this.options = options;
+
+    // dependency injection
+    brotli = brotli || this.options?.modules?.brotli || Compression.modules.brotli;
+
+    if (!isBrowser && this.options.useZlib) {
+      // @ts-ignore public API is equivalent
+      return new BrotliCompressionZlib(options);
+    }
   }
 
   /**
    * brotli is an injectable dependency due to big size
    * @param options
    */
   async preload(): Promise<void> {
-    brotli = brotli || this.options?.modules?.brotli;
+    brotli = brotli || (await this.options?.modules?.brotli);
     if (!brotli) {
       // eslint-disable-next-line no-console
       console.warn(`${this.name} library not installed`);
     }
   }
 
-  async compress(input: ArrayBuffer): Promise<ArrayBuffer> {
-    // On Node.js we can use built-in zlib
-    if (!isBrowser && this.options.brotli?.useZlib) {
-      const buffer = await promisify1(zlib.brotliCompress)(input);
-      return toArrayBuffer(buffer);
-    }
-    return this.compressSync(input);
-  }
-
   compressSync(input: ArrayBuffer): ArrayBuffer {
-    // On Node.js we can use built-in zlib
-    if (!isBrowser && this.options.brotli?.useZlib) {
-      const buffer = zlib.brotliCompressSync(input);
-      return toArrayBuffer(buffer);
-    }
-    const brotliOptions = {...DEFAULT_BROTLI_OPTIONS.brotli, ...this.options?.brotli};
-    const inputArray = new Uint8Array(input);
-
     if (!brotli) {
       throw new Error('brotli compression: brotli module not installed');
     }
-
-    // @ts-ignore brotli types state that only Buffers are accepted...
-    const outputArray = brotli.compress(inputArray, brotliOptions);
+    const options = this._getBrotliOptions();
+    const inputArray = new Uint8Array(input);
+    const outputArray = brotli.compress(inputArray, options);
     return outputArray.buffer;
   }
 
-  async decompress(input: ArrayBuffer): Promise<ArrayBuffer> {
-    // On Node.js we can use built-in zlib
-    if (!isBrowser && this.options.brotli?.useZlib) {
-      const buffer = await promisify1(zlib.brotliDecompress)(input);
-      return toArrayBuffer(buffer);
-    }
-    return this.decompressSync(input);
-  }
-
   decompressSync(input: ArrayBuffer): ArrayBuffer {
-    // On Node.js we can use built-in zlib
-    if (!isBrowser && this.options.brotli?.useZlib) {
-      const buffer = zlib.brotliDecompressSync(input);
-      return toArrayBuffer(buffer);
+    if (!brotli) {
+      throw new Error('brotli compression: brotli module not installed');
     }
 
-    const brotliOptions = {...DEFAULT_BROTLI_OPTIONS.brotli, ...this.options?.brotli};
+    const options = this._getBrotliOptions();
     const inputArray = new Uint8Array(input);
 
-    if (brotli) {
-      // @ts-ignore brotli types state that only Buffers are accepted...
-      const outputArray = brotli.decompress(inputArray, brotliOptions);
-      return outputArray.buffer;
-    }
-    const outputArray = BrotliDecode(inputArray, undefined);
+    // @ts-ignore brotli types state that only Buffers are accepted...
+    const outputArray = brotli.decompress(inputArray, options);
     return outputArray.buffer;
+    // const outputArray = BrotliDecode(inputArray, undefined);
+    // return outputArray.buffer;
+  }
+
+  private _getBrotliOptions(): BrotliOptions {
+    return {
+      level: this.options.quality || Compression.DEFAULT_COMPRESSION_LEVEL,
+      ...this.options?.brotli
+    };
   }
 }

modules/compression/src/lib/compression.ts

@@ -3,18 +3,57 @@ import {concatenateArrayBuffersAsync} from '@loaders.gl/loader-utils';
 
 /** Compression options */
 export type CompressionOptions = {
-  // operation: 'compress' | 'decompress';
-  modules?: {[moduleName: string]: any};
+  /**
+   * Compression quality (higher values better compression but exponentially slower)
+   * brotli goes from 1-11
+   * zlib goes from 1-9
+   * 5 or 6 is usually a good compromise
+   */
+  quality?: number;
+
+  /**
+   * Whether to use built-in Zlib on node.js for max performance (doesn't handle incremental compression)
+   * Currently only deflate, gzip and brotli are supported.
+   */
+  useZlib?: boolean;
+
+  /**
+   * Injection of npm modules - keeps large compression libraries out of standard bundle
+   */
+  modules?: CompressionModules;
+};
+
+/**
+ * Injection of npm modules - keeps large compression libraries out of standard bundle
+ */
+export type CompressionModules = {
+  brotli?: any;
+  lz4js?: any;
+  lzo?: any;
+  'zstd-codec'?: any;
 };
 
 /** Compression */
 export abstract class Compression {
+  /** Default compression level for gzip, brotli etc */
+  static DEFAULT_COMPRESSION_LEVEL = 5;
+
+  /** Name of the compression */
   abstract readonly name: string;
+  /** File extensions used for this */
   abstract readonly extensions: string[];
+  /** Strings used for Content-Encoding headers in browser */
   abstract readonly contentEncodings: string[];
+  /** Whether decompression is supported */
   abstract readonly isSupported: boolean;
+  /** Whether compression is supported */
+  get isCompressionSupported(): boolean {
+    return this.isSupported;
+  }
+
+  static modules: CompressionModules = {};
 
-  constructor(options?: CompressionOptions) {
+  constructor(options) {
     this.compressBatches = this.compressBatches.bind(this);
     this.decompressBatches = this.decompressBatches.bind(this);
   }