zlib: add brotli support

Refs: #20458 Co-authored-by: Hackzzila <[email protected]> PR-URL: #24938 Reviewed-By: Ruben Bridgewater <[email protected]> Reviewed-By: Matteo Collina <[email protected]> Reviewed-By: Jan Krems <[email protected]> Reviewed-By: James M Snell <[email protected]> Reviewed-By: Myles Borins <[email protected]> Reviewed-By: Ali Ijaz Sheikh <[email protected]> Reviewed-By: Daniel Bevenius <[email protected]>
nodejs · Jan 5, 2019 · 7edf8c7 · 7edf8c7
1 parent ec87b6c
commit 7edf8c7
Show file tree

Hide file tree

Showing 9 changed files with 540 additions and 37 deletions.
diff --git a/doc/api/errors.md b/doc/api/errors.md
@@ -624,6 +624,16 @@ An attempt was made to register something that is not a function as an
 The type of an asynchronous resource was invalid. Note that users are also able
 to define their own types if using the public embedder API.
 
+<a id="ERR_BROTLI_INVALID_PARAM"></a>
+### ERR_BROTLI_INVALID_PARAM
+
+An invalid parameter key was passed during construction of a Brotli stream.
+
+<a id="ERR_BROTLI_COMPRESSION_FAILED">
+### ERR_BROTLI_COMPRESSION_FAILED
+
+Data passed to a Brotli stream was not successfully compressed.
+
 <a id="ERR_BUFFER_CONTEXT_NOT_AVAILABLE"></a>
 ### ERR_BUFFER_CONTEXT_NOT_AVAILABLE
 

diff --git a/lib/internal/errors.js b/lib/internal/errors.js
@@ -546,6 +546,7 @@ E('ERR_ARG_NOT_ITERABLE', '%s must be iterable', TypeError);
 E('ERR_ASSERTION', '%s', Error);
 E('ERR_ASYNC_CALLBACK', '%s must be a function', TypeError);
 E('ERR_ASYNC_TYPE', 'Invalid name for async "type": %s', TypeError);
+E('ERR_BROTLI_INVALID_PARAM', '%s is not a valid Brotli parameter', RangeError);
 E('ERR_BUFFER_OUT_OF_BOUNDS',
   // Using a default argument here is important so the argument is not counted
   // towards `Function#length`.

diff --git a/lib/zlib.js b/lib/zlib.js
@@ -22,10 +22,11 @@
 'use strict';
 
 const {
+  ERR_BROTLI_INVALID_PARAM,
   ERR_BUFFER_TOO_LARGE,
   ERR_INVALID_ARG_TYPE,
   ERR_OUT_OF_RANGE,
-  ERR_ZLIB_INITIALIZATION_FAILED
+  ERR_ZLIB_INITIALIZATION_FAILED,
 } = require('internal/errors').codes;
 const Transform = require('_stream_transform');
 const {
@@ -47,11 +48,18 @@ const { owner_symbol } = require('internal/async_hooks').symbols;
 
 const constants = internalBinding('constants').zlib;
 const {
+  // Zlib flush levels
   Z_NO_FLUSH, Z_BLOCK, Z_PARTIAL_FLUSH, Z_SYNC_FLUSH, Z_FULL_FLUSH, Z_FINISH,
+  // Zlib option values
   Z_MIN_CHUNK, Z_MIN_WINDOWBITS, Z_MAX_WINDOWBITS, Z_MIN_LEVEL, Z_MAX_LEVEL,
   Z_MIN_MEMLEVEL, Z_MAX_MEMLEVEL, Z_DEFAULT_CHUNK, Z_DEFAULT_COMPRESSION,
   Z_DEFAULT_STRATEGY, Z_DEFAULT_WINDOWBITS, Z_DEFAULT_MEMLEVEL, Z_FIXED,
-  DEFLATE, DEFLATERAW, INFLATE, INFLATERAW, GZIP, GUNZIP, UNZIP
+  // Node's compression stream modes (node_zlib_mode)
+  DEFLATE, DEFLATERAW, INFLATE, INFLATERAW, GZIP, GUNZIP, UNZIP,
+  BROTLI_DECODE, BROTLI_ENCODE,
+  // Brotli operations (~flush levels)
+  BROTLI_OPERATION_PROCESS, BROTLI_OPERATION_FLUSH,
+  BROTLI_OPERATION_FINISH
 } = constants;
 
 // translation table for return codes.
@@ -212,7 +220,7 @@ function ZlibBase(opts, mode, handle, { flush, finishFlush, fullFlush }) {
   // The ZlibBase class is not exported to user land, the mode should only be
   // passed in by us.
   assert(typeof mode === 'number');
-  assert(mode >= DEFLATE && mode <= UNZIP);
+  assert(mode >= DEFLATE && mode <= BROTLI_ENCODE);
 
   if (opts) {
     chunkSize = opts.chunkSize;
@@ -481,7 +489,7 @@ function processCallback() {
   // important to null out the values once they are no longer needed since
   // `_handle` can stay in memory long after the buffer is needed.
   var handle = this;
-  var self = this.jsref;
+  var self = this[owner_symbol];
   var state = self._writeState;
 
   if (self._hadError) {
@@ -622,6 +630,9 @@ function Zlib(opts, mode) {
                    this._writeState,
                    processCallback,
                    dictionary)) {
+    // TODO(addaleax): Sometimes we generate better error codes in C++ land,
+    // e.g. ERR_BROTLI_PARAM_SET_FAILED -- it's hard to access them with
+    // the current bindings setup, though.
     throw new ERR_ZLIB_INITIALIZATION_FAILED();
   }
 
@@ -726,6 +737,70 @@ function createConvenienceMethod(ctor, sync) {
   }
 }
 
+const kMaxBrotliParam = Math.max(...Object.keys(constants).map((key) => {
+  return key.startsWith('BROTLI_PARAM_') ? constants[key] : 0;
+}));
+
+const brotliInitParamsArray = new Uint32Array(kMaxBrotliParam + 1);
+
+const brotliDefaultOpts = {
+  flush: BROTLI_OPERATION_PROCESS,
+  finishFlush: BROTLI_OPERATION_FINISH,
+  fullFlush: BROTLI_OPERATION_FLUSH
+};
+function Brotli(opts, mode) {
+  assert(mode === BROTLI_DECODE || mode === BROTLI_ENCODE);
+
+  brotliInitParamsArray.fill(-1);
+  if (opts && opts.params) {
+    for (const origKey of Object.keys(opts.params)) {
+      const key = +origKey;
+      if (Number.isNaN(key) || key < 0 || key > kMaxBrotliParam ||
+          (brotliInitParamsArray[key] | 0) !== -1) {
+        throw new ERR_BROTLI_INVALID_PARAM(origKey);
+      }
+
+      const value = opts.params[origKey];
+      if (typeof value !== 'number' && typeof value !== 'boolean') {
+        throw new ERR_INVALID_ARG_TYPE('options.params[key]',
+                                       'number', opts.params[origKey]);
+      }
+      brotliInitParamsArray[key] = value;
+    }
+  }
+
+  const handle = mode === BROTLI_DECODE ?
+    new binding.BrotliDecoder(mode) : new binding.BrotliEncoder(mode);
+
+  this._writeState = new Uint32Array(2);
+  if (!handle.init(brotliInitParamsArray,
+                   this._writeState,
+                   processCallback)) {
+    throw new ERR_ZLIB_INITIALIZATION_FAILED();
+  }
+
+  ZlibBase.call(this, opts, mode, handle, brotliDefaultOpts);
+}
+Object.setPrototypeOf(Brotli.prototype, Zlib.prototype);
+Object.setPrototypeOf(Brotli, Zlib);
+
+function BrotliCompress(opts) {
+  if (!(this instanceof BrotliCompress))
+    return new BrotliCompress(opts);
+  Brotli.call(this, opts, BROTLI_ENCODE);
+}
+Object.setPrototypeOf(BrotliCompress.prototype, Brotli.prototype);
+Object.setPrototypeOf(BrotliCompress, Brotli);
+
+function BrotliDecompress(opts) {
+  if (!(this instanceof BrotliDecompress))
+    return new BrotliDecompress(opts);
+  Brotli.call(this, opts, BROTLI_DECODE);
+}
+Object.setPrototypeOf(BrotliDecompress.prototype, Brotli.prototype);
+Object.setPrototypeOf(BrotliDecompress, Brotli);
+
+
 function createProperty(ctor) {
   return {
     configurable: true,
@@ -751,6 +826,8 @@ module.exports = {
   DeflateRaw,
   InflateRaw,
   Unzip,
+  BrotliCompress,
+  BrotliDecompress,
 
   // Convenience methods.
   // compress/decompress a string or buffer in one step.
@@ -767,7 +844,11 @@ module.exports = {
   gunzip: createConvenienceMethod(Gunzip, false),
   gunzipSync: createConvenienceMethod(Gunzip, true),
   inflateRaw: createConvenienceMethod(InflateRaw, false),
-  inflateRawSync: createConvenienceMethod(InflateRaw, true)
+  inflateRawSync: createConvenienceMethod(InflateRaw, true),
+  brotliCompress: createConvenienceMethod(BrotliCompress, false),
+  brotliCompressSync: createConvenienceMethod(BrotliCompress, true),
+  brotliDecompress: createConvenienceMethod(BrotliDecompress, false),
+  brotliDecompressSync: createConvenienceMethod(BrotliDecompress, true),
 };
 
 Object.defineProperties(module.exports, {
@@ -778,6 +859,8 @@ Object.defineProperties(module.exports, {
   createGzip: createProperty(Gzip),
   createGunzip: createProperty(Gunzip),
   createUnzip: createProperty(Unzip),
+  createBrotliCompress: createProperty(BrotliCompress),
+  createBrotliDecompress: createProperty(BrotliDecompress),
   constants: {
     configurable: false,
     enumerable: true,
@@ -795,6 +878,7 @@ Object.defineProperties(module.exports, {
 const bkeys = Object.keys(constants);
 for (var bk = 0; bk < bkeys.length; bk++) {
   var bkey = bkeys[bk];
+  if (bkey.startsWith('BROTLI')) continue;
   Object.defineProperty(module.exports, bkey, {
     enumerable: true, value: constants[bkey], writable: false
   });

diff --git a/node.gyp b/node.gyp
@@ -11,6 +11,7 @@
     'node_shared%': 'false',
     'force_dynamic_crt%': 0,
     'node_module_version%': '',
+    'node_shared_brotli%': 'false',
     'node_shared_zlib%': 'false',
     'node_experimental_http_parser%': 'false',
     'node_shared_http_parser%': 'false',

diff --git a/node.gypi b/node.gypi
@@ -212,6 +212,10 @@
       'dependencies': [ 'deps/nghttp2/nghttp2.gyp:nghttp2' ],
     }],
 
+    [ 'node_shared_brotli=="false"', {
+      'dependencies': [ 'deps/brotli/brotli.gyp:brotli' ],
+    }],
+
     [ 'OS=="mac"', {
       # linking Corefoundation is needed since certain OSX debugging tools
       # like Instruments require it for some features

diff --git a/src/node_metadata.cc b/src/node_metadata.cc
@@ -1,5 +1,6 @@
 #include "node_metadata.h"
 #include "ares.h"
+#include "brotli/encode.h"
 #include "nghttp2/nghttp2ver.h"
 #include "node.h"
 #include "node_internals.h"
@@ -30,6 +31,13 @@ Metadata::Versions::Versions() {
   llhttp = llhttp_version;
   http_parser = http_parser_version;
 
+  brotli =
+    std::to_string(BrotliEncoderVersion() >> 24) +
+    "." +
+    std::to_string((BrotliEncoderVersion() & 0xFFF000) >> 12) +
+    "." +
+    std::to_string(BrotliEncoderVersion() & 0xFFF);
+
 #if HAVE_OPENSSL
   openssl = crypto::GetOpenSSLVersion();
 #endif

diff --git a/src/node_metadata.h b/src/node_metadata.h
@@ -12,6 +12,7 @@ namespace node {
   V(v8)                                                                        \
   V(uv)                                                                        \
   V(zlib)                                                                      \
+  V(brotli)                                                                    \
   V(ares)                                                                      \
   V(modules)                                                                   \
   V(nghttp2)                                                                   \