Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@ibgreen
ibgreen committed May 23, 2023
1 parent 33be688 commit 9ee6aa2
Show file tree
Hide file tree
Showing 18 changed files with 414 additions and 185 deletions.
7 changes: 4 additions & 3 deletions modules/compression/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,20 +41,21 @@
"@babel/runtime": "^7.3.1",
"@loaders.gl/loader-utils": "4.0.0-alpha.6",
"@loaders.gl/worker-utils": "4.0.0-alpha.6",
"@types/brotli": "^1.3.0",
"@types/pako": "^1.0.1",
"brotli": "^1.3.3",
"brotli-wasm": "^1.3.1",
"fflate": "0.7.4",
"lzo-wasm": "^0.0.4",
"pako": "1.0.11",
"snappyjs": "^0.6.1"
},
"optionalDependencies": {
"brotli": "^1.3.2",
"brotli-compress": "^1.3.3",
"lz4js": "^0.2.0",
"zstd-codec": "^0.1"
},
"devDependencies": {
"brotli": "^1.3.2",
"brotli-compress": "^1.3.3",
"lz4js": "^0.2.0",
"zstd-codec": "^0.1"
},
Expand Down
18 changes: 18 additions & 0 deletions modules/compression/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,31 @@ export type {CompressionOptions} from './lib/compression';
export {Compression} from './lib/compression';

export {NoCompression} from './lib/no-compression';

export {DeflateCompression} from './lib/deflate-compression';
export {DeflateCompression as _DeflateCompressionZlib} from './lib/deflate-compression-zlib';
export {GZipCompression} from './lib/gzip-compression';
export {GZipCompression as _GZipCompressionZlib} from './lib/gzip-compression-zlib';

export {BrotliCompression} from './lib/brotli-compression';
export {BrotliCompressionZlib} from './lib/brotli-compression-zlib';

export {SnappyCompression} from './lib/snappy-compression';

export {LZ4Compression} from './lib/lz4-compression';

export {ZstdCompression} from './lib/zstd-compression';

export {LZOCompression} from './lib/lzo-compression';

export type {CompressionWorkerOptions} from './compression-worker';
export {CompressionWorker, compressOnWorker} from './compression-worker';

// Versions
export {DeflateCompression as _DeflateCompressionFflate} from './lib/deflate-compression';
export {GZipCompression as _GZipCompressionFflate} from './lib/gzip-compression';

export {DeflateCompression as _DeflateCompressionPako} from './lib/deflate-compression-pako';
export {GZipCompression as _GZipCompressionPako} from './lib/gzip-compression-pako';


55 changes: 55 additions & 0 deletions modules/compression/src/lib/brotli-compression-zlib.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
// BROTLI
import type {CompressionOptions} from './compression';
import {Compression} from './compression';
import {isBrowser, toArrayBuffer} from '@loaders.gl/loader-utils';
import zlib from 'zlib';
import {promisify1, promisify2} from '@loaders.gl/loader-utils';

export type BrotliCompressionOptions = CompressionOptions & {
brotli?: {
mode?: number;
quality?: number;
lgwin?: number;
useZlib?: boolean;
};
};

/**
* brotli compression / decompression using zlib
*/
export class BrotliCompressionZlib extends Compression {
readonly name: string = 'brotli';
readonly extensions = ['br'];
readonly contentEncodings = ['br'];
readonly isSupported = true;
readonly options: BrotliCompressionOptions;

constructor(options: BrotliCompressionOptions = {}) {
super(options);
this.options = options;
if (isBrowser) {
throw new Error('zlib only available under Node.js');
}
}

async compress(input: ArrayBuffer): Promise<ArrayBuffer> {
// @ts-expect-error overloads - Node uses compression level 11 by default which is 100x slower
const buffer = await promisify2(zlib.brotliCompress)(input, {params: {[zlib.constants.BROTLI_PARAM_QUALITY]: 4}});
return toArrayBuffer(buffer);
}

compressSync(input: ArrayBuffer): ArrayBuffer {
const buffer = zlib.brotliCompressSync(input, {params: {[zlib.constants.BROTLI_PARAM_QUALITY]: 4}});
return toArrayBuffer(buffer);
}

async decompress(input: ArrayBuffer): Promise<ArrayBuffer> {
const buffer = await promisify1(zlib.brotliDecompress)(input);
return toArrayBuffer(buffer);
}

decompressSync(input: ArrayBuffer): ArrayBuffer {
const buffer = zlib.brotliDecompressSync(input);
return toArrayBuffer(buffer);
}
}
75 changes: 30 additions & 45 deletions modules/compression/src/lib/brotli-compression.ts
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
// BROTLI
import {isBrowser} from '@loaders.gl/loader-utils';
import type {CompressionOptions} from './compression';
import {Compression} from './compression';
import {isBrowser, toArrayBuffer} from '@loaders.gl/loader-utils';
import {BrotliCompressionZlib} from './brotli-compression-zlib';
import type brotliNamespace from 'brotli';
// import brotli from 'brotli'; // https://bundlephobia.com/package/brotli
import {BrotliDecode} from '../brotli/decode';
import zlib from 'zlib';
import {promisify1} from '@loaders.gl/loader-utils';
// import {BrotliDecode} from '../brotli/decode';

export type BrotliCompressionOptions = CompressionOptions & {
brotli?: {
Expand Down Expand Up @@ -34,78 +33,64 @@ export class BrotliCompression extends Compression {
readonly name: string = 'brotli';
readonly extensions = ['br'];
readonly contentEncodings = ['br'];
readonly isSupported = true;

get isSupported() {
return brotli;
}
get isCompressionSupported() {
return false;
}

readonly options: BrotliCompressionOptions;

constructor(options: BrotliCompressionOptions) {
constructor(options: BrotliCompressionOptions = {}) {
super(options);
this.options = options;

// dependency injection
brotli = brotli || this.options?.modules?.brotli || Compression.modules.brotli;

if (!isBrowser && this.options.brotli?.useZlib) {
return new BrotliCompressionZlib(options);
}
}

/**
* brotli is an injectable dependency due to big size
* @param options
*/
async preload(): Promise<void> {
brotli = brotli || this.options?.modules?.brotli;
brotli = brotli || (await this.options?.modules?.brotli);
if (!brotli) {
// eslint-disable-next-line no-console
console.warn(`${this.name} library not installed`);
}
}

async compress(input: ArrayBuffer): Promise<ArrayBuffer> {
// On Node.js we can use built-in zlib
if (!isBrowser && this.options.brotli?.useZlib) {
const buffer = await promisify1(zlib.brotliCompress)(input);
return toArrayBuffer(buffer);
}
return this.compressSync(input);
}

compressSync(input: ArrayBuffer): ArrayBuffer {
// On Node.js we can use built-in zlib
if (!isBrowser && this.options.brotli?.useZlib) {
const buffer = zlib.brotliCompressSync(input);
return toArrayBuffer(buffer);
}
const brotliOptions = {...DEFAULT_BROTLI_OPTIONS.brotli, ...this.options?.brotli};
const inputArray = new Uint8Array(input);

if (!brotli) {
throw new Error('brotli compression: brotli module not installed');
}

// @ts-ignore brotli types state that only Buffers are accepted...
const outputArray = brotli.compress(inputArray, brotliOptions);
return outputArray.buffer;
}
const brotliOptions = {...DEFAULT_BROTLI_OPTIONS.brotli, ...this.options?.brotli};
const inputArray = new Uint8Array(input);

async decompress(input: ArrayBuffer): Promise<ArrayBuffer> {
// On Node.js we can use built-in zlib
if (!isBrowser && this.options.brotli?.useZlib) {
const buffer = await promisify1(zlib.brotliDecompress)(input);
return toArrayBuffer(buffer);
}
return this.decompressSync(input);
const outputArray = brotli.compress(inputArray, {quality: 5, brotliOptions});
return outputArray.buffer;
}

decompressSync(input: ArrayBuffer): ArrayBuffer {
// On Node.js we can use built-in zlib
if (!isBrowser && this.options.brotli?.useZlib) {
const buffer = zlib.brotliDecompressSync(input);
return toArrayBuffer(buffer);
if (!brotli) {
throw new Error('brotli compression: brotli module not installed');
}

const brotliOptions = {...DEFAULT_BROTLI_OPTIONS.brotli, ...this.options?.brotli};
const inputArray = new Uint8Array(input);

if (brotli) {
// @ts-ignore brotli types state that only Buffers are accepted...
const outputArray = brotli.decompress(inputArray, brotliOptions);
return outputArray.buffer;
}
const outputArray = BrotliDecode(inputArray, undefined);
// @ts-ignore brotli types state that only Buffers are accepted...
const outputArray = brotli.decompress(inputArray, brotliOptions);
return outputArray.buffer;
// const outputArray = BrotliDecode(inputArray, undefined);
// return outputArray.buffer;
}
}
24 changes: 21 additions & 3 deletions modules/compression/src/lib/compression.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,36 @@ import {concatenateArrayBuffersAsync} from '@loaders.gl/loader-utils';

/** Compression options */
export type CompressionOptions = {
// operation: 'compress' | 'decompress';
modules?: {[moduleName: string]: any};
/** Compression quality typically goes from 1-11 (higher values better but slower) */
quality?: number;
/** Injection of npm modules for large libraries */
modules?: CompressionModules;
};

export type CompressionModules = {
brotli?: any;
lz4js?: any;
lzo?: any;
'zstd-codec'?: any;
}


/** Compression */
export abstract class Compression {
/** Name of the compression */
abstract readonly name: string;
/** File extensions used for this */
abstract readonly extensions: string[];
/** Strings used for Content-Encoding headers in browser */
abstract readonly contentEncodings: string[];
/** Whether decompression is supported */
abstract readonly isSupported: boolean;
/** Whether compression is supported */
get isCompressionSupported(): boolean { return this.isSupported; };

static modules: CompressionModules = {};

constructor(options?: CompressionOptions) {
constructor(options) {
this.compressBatches = this.compressBatches.bind(this);
this.decompressBatches = this.decompressBatches.bind(this);
}
Expand Down
27 changes: 14 additions & 13 deletions modules/compression/src/lib/deflate-compression.ts
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
// loaders.gl, MIT license
import {promisify1} from '@loaders.gl/loader-utils';
import type {CompressionOptions} from './compression';
import {Compression} from './compression';

import {deflate, inflate, deflateSync, inflateSync} from 'fflate';
import {deflateSync, inflateSync} from 'fflate';
import type {DeflateOptions} from 'fflate'; // https://bundlephobia.com/package/pako

export type DeflateCompressionOptions = CompressionOptions & {
Expand All @@ -26,18 +25,20 @@ export class DeflateCompression extends Compression {
this.options = options;
}

async compress(input: ArrayBuffer): Promise<ArrayBuffer> {
// const options = this.options?.gzip || {};
const inputArray = new Uint8Array(input);
const outputArray = await promisify1(deflate)(inputArray); // options - overload pick
return outputArray.buffer;
}
// Async fflate uses Workers which interferes with loaders.gl

async decompress(input: ArrayBuffer): Promise<ArrayBuffer> {
const inputArray = new Uint8Array(input);
const outputArray = await promisify1(inflate)(inputArray);
return outputArray.buffer;
}
// async compress(input: ArrayBuffer): Promise<ArrayBuffer> {
// // const options = this.options?.gzip || {};
// const inputArray = new Uint8Array(input);
// const outputArray = await promisify1(deflate)(inputArray); // options - overload pick
// return outputArray.buffer;
// }

// async decompress(input: ArrayBuffer): Promise<ArrayBuffer> {
// const inputArray = new Uint8Array(input);
// const outputArray = await promisify1(inflate)(inputArray);
// return outputArray.buffer;
// }

compressSync(input: ArrayBuffer): ArrayBuffer {
const options = this.options?.deflate || {};
Expand Down
37 changes: 16 additions & 21 deletions modules/compression/src/lib/gzip-compression-pako.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,20 +17,6 @@ export type GZipCompressionOptions = CompressionOptions & {
* Using PAKO library.
*/
export class GZipCompression extends Compression {
readonly name: string = 'gzip';
readonly extensions = ['gz', 'gzip'];
readonly contentEncodings = ['gzip', 'x-gzip'];
readonly isSupported = true;

constructor(options?: GZipCompressionOptions) {
super({...options});
}
}

/**
* DEFLATE compression / decompression
*/
export class DeflateCompression extends Compression {
readonly name: string = 'deflate';
readonly extensions: string[] = [];
readonly contentEncodings = ['deflate'];
Expand All @@ -54,21 +40,21 @@ export class DeflateCompression extends Compression {
}

compressSync(input: ArrayBuffer): ArrayBuffer {
const pakoOptions: pako.DeflateOptions = this.options?.deflate || {};
const pakoOptions = this._getDeflateOptions();
const inputArray = new Uint8Array(input);
return pako.deflate(inputArray, pakoOptions).buffer;
return pako.gzip(inputArray, pakoOptions).buffer;
}

decompressSync(input: ArrayBuffer): ArrayBuffer {
const pakoOptions: pako.InflateOptions = this.options?.deflate || {};
const inputArray = new Uint8Array(input);
return pako.inflate(inputArray, pakoOptions).buffer;
return pako.ungzip(inputArray, pakoOptions).buffer;
}

async *compressBatches(
asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>
): AsyncIterable<ArrayBuffer> {
const pakoOptions: pako.DeflateOptions = this.options?.deflate || {};
const pakoOptions = this._getDeflateOptions();
const pakoProcessor = new pako.Deflate(pakoOptions);
yield* this.transformBatches(pakoProcessor, asyncIterator);
}
Expand Down Expand Up @@ -108,19 +94,28 @@ export class DeflateCompression extends Compression {
yield* chunks;
}

_onData(chunk) {
protected _onData(chunk) {
this._chunks.push(chunk);
}

_onEnd(status) {
protected _onEnd(status) {
if (status !== 0) {
throw new Error(getPakoError(status) + this._chunks.length);
}
}

_getChunks(): ArrayBuffer[] {
protected _getChunks(): ArrayBuffer[] {
const chunks = this._chunks;
this._chunks = [];
return chunks;
}

protected _getDeflateOptions(): pako.DeflateOptions {
const pakoOptions: pako.DeflateOptions = this.options?.deflate || {};
if (this.options.quality) {
// @ts-expect-error
pakoOptions.level = this.options.quality || 5;
}
return pakoOptions;
}
}
Loading

0 comments on commit 9ee6aa2

Please sign in to comment.