diff --git a/binary/decoder.js b/binary/decoder.js
index 1186add..c44053f 100644
--- a/binary/decoder.js
+++ b/binary/decoder.js
@@ -47,7 +47,7 @@
goog.provide('jspb.BinaryDecoder');
goog.require('jspb.asserts');
-goog.require('goog.crypt');
+goog.require('jspb.binary.utf8');
goog.require('jspb.utils');
@@ -256,7 +256,7 @@ jspb.BinaryDecoder.prototype.setCursor = function(cursor) {
*/
jspb.BinaryDecoder.prototype.advance = function(count) {
this.cursor_ += count;
- jspb.asserts.assert(this.cursor_ <= this.end_);
+ this.checkCursor();
};
@@ -397,6 +397,17 @@ jspb.BinaryDecoder.prototype.readSplitFixed64 = function(convert) {
return convert(lowBits, highBits);
};
+/**
+ * Asserts that our cursor is in bounds.
+ *
+ * @private
+ * @return {void}
+ */
+jspb.BinaryDecoder.prototype.checkCursor = function () {
+ if (this.cursor_ > this.end_) {
+ asserts.fail('Read past the end ' + this.cursor_ + ' > ' + this.end_);
+ }
+}
/**
* Skips over a varint in the block without decoding it.
@@ -452,7 +463,7 @@ jspb.BinaryDecoder.prototype.readUnsignedVarint32 = function() {
var x = (temp & 0x7F);
if (temp < 128) {
this.cursor_ += 1;
- jspb.asserts.assert(this.cursor_ <= this.end_);
+ this.checkCursor();
return x;
}
@@ -460,7 +471,7 @@ jspb.BinaryDecoder.prototype.readUnsignedVarint32 = function() {
x |= (temp & 0x7F) << 7;
if (temp < 128) {
this.cursor_ += 2;
- jspb.asserts.assert(this.cursor_ <= this.end_);
+ this.checkCursor();
return x;
}
@@ -468,7 +479,7 @@ jspb.BinaryDecoder.prototype.readUnsignedVarint32 = function() {
x |= (temp & 0x7F) << 14;
if (temp < 128) {
this.cursor_ += 3;
- jspb.asserts.assert(this.cursor_ <= this.end_);
+ this.checkCursor();
return x;
}
@@ -476,7 +487,7 @@ jspb.BinaryDecoder.prototype.readUnsignedVarint32 = function() {
x |= (temp & 0x7F) << 21;
if (temp < 128) {
this.cursor_ += 4;
- jspb.asserts.assert(this.cursor_ <= this.end_);
+ this.checkCursor();
return x;
}
@@ -486,7 +497,7 @@ jspb.BinaryDecoder.prototype.readUnsignedVarint32 = function() {
// We're reading the high bits of an unsigned varint. The byte we just read
// also contains bits 33 through 35, which we're going to discard.
this.cursor_ += 5;
- jspb.asserts.assert(this.cursor_ <= this.end_);
+ this.checkCursor();
return x >>> 0;
}
@@ -500,7 +511,7 @@ jspb.BinaryDecoder.prototype.readUnsignedVarint32 = function() {
jspb.asserts.assert(false);
}
- jspb.asserts.assert(this.cursor_ <= this.end_);
+ this.checkCursor();
return x;
};
@@ -679,7 +690,7 @@ jspb.BinaryDecoder.prototype.readZigzagVarint64String = function() {
jspb.BinaryDecoder.prototype.readUint8 = function() {
var a = this.bytes_[this.cursor_ + 0];
this.cursor_ += 1;
- jspb.asserts.assert(this.cursor_ <= this.end_);
+ this.checkCursor();
return a;
};
@@ -694,7 +705,7 @@ jspb.BinaryDecoder.prototype.readUint16 = function() {
var a = this.bytes_[this.cursor_ + 0];
var b = this.bytes_[this.cursor_ + 1];
this.cursor_ += 2;
- jspb.asserts.assert(this.cursor_ <= this.end_);
+ this.checkCursor();
return (a << 0) | (b << 8);
};
@@ -711,7 +722,7 @@ jspb.BinaryDecoder.prototype.readUint32 = function() {
var c = this.bytes_[this.cursor_ + 2];
var d = this.bytes_[this.cursor_ + 3];
this.cursor_ += 4;
- jspb.asserts.assert(this.cursor_ <= this.end_);
+ this.checkCursor();
return ((a << 0) | (b << 8) | (c << 16) | (d << 24)) >>> 0;
};
@@ -756,7 +767,7 @@ jspb.BinaryDecoder.prototype.readUint64String = function() {
jspb.BinaryDecoder.prototype.readInt8 = function() {
var a = this.bytes_[this.cursor_ + 0];
this.cursor_ += 1;
- jspb.asserts.assert(this.cursor_ <= this.end_);
+ this.checkCursor();
return (a << 24) >> 24;
};
@@ -771,7 +782,7 @@ jspb.BinaryDecoder.prototype.readInt16 = function() {
var a = this.bytes_[this.cursor_ + 0];
var b = this.bytes_[this.cursor_ + 1];
this.cursor_ += 2;
- jspb.asserts.assert(this.cursor_ <= this.end_);
+ this.checkCursor();
return (((a << 0) | (b << 8)) << 16) >> 16;
};
@@ -788,7 +799,7 @@ jspb.BinaryDecoder.prototype.readInt32 = function() {
var c = this.bytes_[this.cursor_ + 2];
var d = this.bytes_[this.cursor_ + 3];
this.cursor_ += 4;
- jspb.asserts.assert(this.cursor_ <= this.end_);
+ this.checkCursor();
return (a << 0) | (b << 8) | (c << 16) | (d << 24);
};
@@ -858,7 +869,9 @@ jspb.BinaryDecoder.prototype.readDouble = function() {
* @export
*/
jspb.BinaryDecoder.prototype.readBool = function() {
- return !!this.bytes_[this.cursor_++];
+ const b = !!this.bytes_[this.cursor_++];
+ this.checkCursor();
+ return b;
};
@@ -879,59 +892,17 @@ jspb.BinaryDecoder.prototype.readEnum = function() {
* Supports codepoints from U+0000 up to U+10FFFF.
* (http://en.wikipedia.org/wiki/UTF-8).
* @param {number} length The length of the string to read.
+ * @param {boolean} requireUtf8 Whether to throw when invalid utf8 is found.
* @return {string} The decoded string.
* @export
*/
-jspb.BinaryDecoder.prototype.readString = function(length) {
- var bytes = this.bytes_;
- var cursor = this.cursor_;
- var end = cursor + length;
- var codeUnits = [];
-
- var result = '';
- while (cursor < end) {
- var c = bytes[cursor++];
- if (c < 128) { // Regular 7-bit ASCII.
- codeUnits.push(c);
- } else if (c < 192) {
- // UTF-8 continuation mark. We are out of sync. This
- // might happen if we attempted to read a character
- // with more than four bytes.
- continue;
- } else if (c < 224) { // UTF-8 with two bytes.
- var c2 = bytes[cursor++];
- codeUnits.push(((c & 31) << 6) | (c2 & 63));
- } else if (c < 240) { // UTF-8 with three bytes.
- var c2 = bytes[cursor++];
- var c3 = bytes[cursor++];
- codeUnits.push(((c & 15) << 12) | ((c2 & 63) << 6) | (c3 & 63));
- } else if (c < 248) { // UTF-8 with 4 bytes.
- var c2 = bytes[cursor++];
- var c3 = bytes[cursor++];
- var c4 = bytes[cursor++];
- // Characters written on 4 bytes have 21 bits for a codepoint.
- // We can't fit that on 16bit characters, so we use surrogates.
- var codepoint =
- ((c & 7) << 18) | ((c2 & 63) << 12) | ((c3 & 63) << 6) | (c4 & 63);
- // Surrogates formula from wikipedia.
- // 1. Subtract 0x10000 from codepoint
- codepoint -= 0x10000;
- // 2. Split this into the high 10-bit value and the low 10-bit value
- // 3. Add 0xD800 to the high value to form the high surrogate
- // 4. Add 0xDC00 to the low value to form the low surrogate:
- var low = (codepoint & 1023) + 0xDC00;
- var high = ((codepoint >> 10) & 1023) + 0xD800;
- codeUnits.push(high, low);
- }
- // Avoid exceeding the maximum stack size when calling `apply`.
- if (codeUnits.length >= 8192) {
- result += String.fromCharCode.apply(null, codeUnits);
- codeUnits.length = 0;
- }
- }
- result += goog.crypt.byteArrayToString(codeUnits);
- this.cursor_ = cursor;
+jspb.BinaryDecoder.prototype.readString = function (length, requireUtf8) {
+ const cursor = this.cursor_;
+ this.cursor_ += length;
+ this.checkCursor();
+ const result =
+ jspb.binary.utf8.decodeUtf8(jspb.asserts.assert(this.bytes_), cursor, length, requireUtf8);
return result;
};
@@ -966,7 +937,7 @@ jspb.BinaryDecoder.prototype.readBytes = function(length) {
var result = this.bytes_.subarray(this.cursor_, this.cursor_ + length);
this.cursor_ += length;
- jspb.asserts.assert(this.cursor_ <= this.end_);
+ this.checkCursor();
return result;
};
diff --git a/binary/decoder_test.js b/binary/decoder_test.js
index 77f6877..b8e1084 100644
--- a/binary/decoder_test.js
+++ b/binary/decoder_test.js
@@ -354,7 +354,7 @@ describe('binaryDecoderTest', () => {
const decoder = jspb.BinaryDecoder.alloc(encoder.end());
- expect(decoder.readString(len)).toEqual(long_string);
+ expect(decoder.readString(len, true)).toEqual(long_string);
});
/**
@@ -375,11 +375,11 @@ describe('binaryDecoderTest', () => {
const decoder = jspb.BinaryDecoder.alloc(encoder.end());
- expect(decoder.readString(ascii.length)).toEqual(ascii);
- expect(utf8_two_bytes).toEqual(decoder.readString(utf8_two_bytes.length));
+ expect(decoder.readString(ascii.length, true)).toEqual(ascii);
+ expect(utf8_two_bytes).toEqual(decoder.readString(2, true));
expect(utf8_three_bytes)
- .toEqual(decoder.readString(utf8_three_bytes.length));
- expect(utf8_four_bytes).toEqual(decoder.readString(utf8_four_bytes.length));
+ .toEqual(decoder.readString(3, true));
+ expect(utf8_four_bytes).toEqual(decoder.readString(4, true));
});
/**
diff --git a/binary/reader.js b/binary/reader.js
index 7be3b58..0f8c961 100644
--- a/binary/reader.js
+++ b/binary/reader.js
@@ -52,6 +52,26 @@ goog.require('jspb.BinaryConstants');
goog.require('jspb.BinaryDecoder');
goog.require('jspb.utils');
+/**
+ * Whether to enforce that string fields are valid utf8.
+ *
+ *
Currently set to `ALWAYS`, can be set to `DEPRECATED_PROTO3_ONLY` to only
+ * enforce utf8 for proto3 string fields, for proto2 string fields it will use
+ * replacement characters when encoding errors are found.
+ *
+ *
TODO: Remove the flag, simplify BinaryReader to remove
+ * readStringRequireUtf8 and related support in the code generator et. al.
+ *
+ * @define {string}
+ */
+const ENFORCE_UTF8 = goog.define('jspb.binary.ENFORCE_UTF8', 'ALWAYS');
+
+// Constrain the set of values to only these two.
+jspb.asserts.assert(
+ ENFORCE_UTF8 === 'DEPRECATED_PROTO3_ONLY' || ENFORCE_UTF8 === 'ALWAYS');
+
+const /** boolean */ UTF8_PARSING_ERRORS_ARE_FATAL = ENFORCE_UTF8 === 'ALWAYS';
+
/**
@@ -996,10 +1016,29 @@ jspb.BinaryReader.prototype.readEnum = function() {
* @export
*/
jspb.BinaryReader.prototype.readString = function() {
+ // delegate to the other reader so that inlining can eliminate this method
+ // in the common case.
+ if (UTF8_PARSING_ERRORS_ARE_FATAL) {
+ return this.readStringRequireUtf8();
+ }
+
jspb.asserts.assert(
this.nextWireType_ == jspb.BinaryConstants.WireType.DELIMITED);
var length = this.decoder_.readUnsignedVarint32();
- return this.decoder_.readString(length);
+ return this.decoder_.readString(length, /*requireUtf8=*/ false);
+};
+
+/**
+ * Reads a string field from the binary stream, or throws an error if the next
+ * field in the stream is not of the correct wire type, or if the string is
+ * not valid utf8.
+ *
+ * @return {string} The value of the string field.
+ */
+jspb.BinaryReader.prototype.readStringRequireUtf8 = function () {
+ jspb.asserts.assert(this.nextWireType_ == jspb.BinaryConstants.WireType.DELIMITED);
+ const length = this.decoder_.readUnsignedVarint32();
+ return this.decoder_.readString(length, /*requireUtf8=*/ true);
};
diff --git a/binary/utf8.js b/binary/utf8.js
new file mode 100644
index 0000000..b77602b
--- /dev/null
+++ b/binary/utf8.js
@@ -0,0 +1,426 @@
+/**
+ * @fileoverview UTF8 encoding and decoding routines
+ */
+goog.provide('jspb.binary.utf8');
+
+goog.require('jspb.asserts');
+
+
+/**
+ * Whether to use the browser based `TextEncoder` and `TextDecoder` APIs for
+ * handling utf8.
+ *
+ *
Enabled by default for `goog.FEATURESET_YEAR >= 2020`. The code also
+ * performs feature detection for this API and will always use it if available,
+ * this variable enables us to not ship the polyfill.
+ *
+ *
See http://go/jscompiler-flags#browser-featureset-year-options for the
+ * behavior here.
+ *
+ * @define {boolean}
+ */
+const USE_TEXT_ENCODING =
+ goog.define('jspb.binary.USE_TEXTENCODING', goog.FEATURESET_YEAR >= 2020);
+
+const /** number */ MIN_SURROGATE = 0xD800;
+const /** number */ MIN_HIGH_SURROGATE = MIN_SURROGATE;
+const /** number */ MAX_HIGH_SURROGATE = 0xDBFF;
+const /** number */ MIN_LOW_SURROGATE = 0xDC00;
+const /** number */ MAX_LOW_SURROGATE = 0xDFFF;
+const /** number */ MAX_SURROGATE = MAX_LOW_SURROGATE;
+
+/**
+ * Returns whether the byte is not a valid continuation of the form
+ * '10XXXXXX'.
+ * @return {boolean}
+ */
+function isNotTrailingByte(/** number */ byte) {
+ // 0xC0 is '11000000' in binary
+ // 0x80 is '10000000' in binary
+ return (byte & 0xC0) !== 0x80;
+}
+
+
+/**
+ * Either throws an error or appends a replacement codepoint of invalid utf8
+ */
+function invalid(
+ /** boolean */ parsingErrorsAreFatal, /** !Array */ codeUnits) {
+ if (parsingErrorsAreFatal) {
+ throw new Error('Invalid UTF8');
+ }
+ codeUnits.push(0xFFFD); // utf8 replacement character
+}
+
+/** @return {string} */
+function codeUnitsToString(
+ /** string? */ accum, /** !Array */ utf16CodeUnits) {
+ const suffix = String.fromCharCode.apply(null, utf16CodeUnits);
+ return accum == null ? suffix : accum + suffix;
+}
+
+/**
+ * Our handwritten UTF8 decoder.
+ *
+ * https://en.wikipedia.org/wiki/UTF-8#Encoding describes the bit layout
+ *
+ * https://en.wikipedia.org/wiki/UTF-8#Invalid_sequences_and_error_handling
+ * describes important cases to check for which are namely:
+ * - overlong encodings, meaning a value expressable in N bytes could have been
+ * expressed in fewer bytes
+ * - invalid bytes, meaning bytes that are generally out of range
+ * - surrogate codepoints, utf8 never encodes directly a utf16 surrogate value
+ * - underflow where there aren't enough bytes for the sequence we are parsing
+ * - out of range codepoints.
+ *
+ * @return {string}
+ */
+jspb.binary.utf8.polyfillDecodeUtf8 = function (
+ /** !Uint8Array */ bytes, /** number */ offset, /** number */ length,
+ /** boolean */ parsingErrorsAreFatal) {
+ let cursor = offset;
+ const end = cursor + length;
+ const codeUnits = [];
+ let result = null;
+
+ // This is significantly slower than the TextDecoder implementation.
+ // Ideas for improving performance:
+ // 1. Reduce branching with non-shortcircuting operators, e.g.
+ // https://stackoverflow.com/q/5652363
+ // 2. improve isNotTrailingByte using xor?
+ // 3. consider having a dedicate ascii loop (java impls do this)
+ let c1, c2, c3, c4;
+ while (cursor < end) {
+ c1 = bytes[cursor++];
+ if (c1 < 0x80) { // Regular 7-bit ASCII.
+ codeUnits.push(c1);
+ } else if (c1 < 0xE0) { // UTF-8 with two bytes.
+ if (cursor >= end) {
+ invalid(parsingErrorsAreFatal, codeUnits);
+ } else {
+ c2 = bytes[cursor++];
+ // Make sure that c1 is a valid leading byte and c2 is a valid
+ // trailing byte
+ // 0xC2 is '11000010', if c1 is less than this then we have an overlong
+ // encoding because there would only be 7 significant bits.
+ if (c1 < 0xC2 || isNotTrailingByte(c2)) {
+ cursor--; // push c2 back since it isn't 'accepted'
+ invalid(parsingErrorsAreFatal, codeUnits);
+ } else {
+ // The codeUnit is the lower 6 bits from c2 and the lower 5 bits from
+ // c1
+ const codeUnit = ((c1 & 0x1F) << 6) | (c2 & 0x3F);
+ // Consistency check that the computed code is in range for a 2 byte
+ // sequence.
+ jspb.asserts.assert(codeUnit >= 0x80 && codeUnit <= 0x07FF);
+ codeUnits.push(codeUnit);
+ }
+ }
+ } else if (c1 < 0xF0) { // UTF-8 with three bytes.
+ if (cursor >= end - 1) {
+ invalid(parsingErrorsAreFatal, codeUnits);
+ } else {
+ c2 = bytes[cursor++];
+ if (isNotTrailingByte(c2) ||
+ // These checks were taken from
+ // java/com/google/protobuf/Utf8.java
+ // overlong? 5 most significant bits must not all be zero
+ (c1 === 0xE0 && c2 < 0xA0)
+ // check for illegal surrogate codepoints
+ || (c1 === 0xED && c2 >= 0xA0) ||
+ // We delay reading c3 until now so than an error in c2 or c1 will
+ // preserve c3 for the next loop iteration
+ isNotTrailingByte(c3 = bytes[cursor++])) {
+ cursor--; // push back c2 or c3, depending on how far we made it
+ invalid(parsingErrorsAreFatal, codeUnits);
+ } else {
+ // 4 bits from the first byte
+ // 6 bits from each of the two lower bytes
+ // == 16 bits total
+ const codeUnit =
+ ((c1 & 0xF) << 12) | ((c2 & 0x3F) << 6) | (c3 & 0x3F);
+ // Consistency check, this is the valid range for a 3 byte character
+ jspb.asserts.assert(codeUnit >= 0x800 && codeUnit <= 0xFFFF);
+ // And that Utf16 surrogates are disallowed
+ jspb.asserts.assert(codeUnit < MIN_SURROGATE || codeUnit > MAX_SURROGATE);
+ codeUnits.push(codeUnit);
+ }
+ }
+ } else if (c1 <= 0xF4) { // UTF-8 with 4 bytes.
+ // 0xF8 matches the bitpattern for utf8 with 4 bytes, but all leading
+ // bytes > 0xF4 are either overlong encodings or exceed the valid range.
+ if (cursor >= end - 2) {
+ invalid(parsingErrorsAreFatal, codeUnits);
+ } else {
+ c2 = bytes[cursor++];
+ if (isNotTrailingByte(c2) ||
+ // This check was inspired by
+ // java/com/google/protobuf/Utf8.java
+ // Tricky optimized form of:
+ // valid 4-byte leading byte?
+ // if (byte1 > (byte) 0xF4 ||
+ // overlong? 4 most significant bits must not all be zero
+ // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 ||
+ // codepoint larger than the highest code point (U+10FFFF)?
+ // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F)
+ (((c1 << 28) + (c2 - 0x90)) >> 30) !== 0 ||
+ // We delay reading c3 and c4 until now so than an error in c2 or c1
+ // will preserve them for the next loop iteration.
+ isNotTrailingByte(c3 = bytes[cursor++]) ||
+ isNotTrailingByte(c4 = bytes[cursor++])) {
+ cursor--; // push back c2, c3 or c4 depending on how far we made it
+ invalid(parsingErrorsAreFatal, codeUnits);
+ } else {
+ // Characters written on 4 bytes have 21 bits for a codepoint.
+ // We can't fit that on 16bit characters, so we use surrogates.
+ // 3 bits from the uppermost byte, 6 bits from each of the lower 3
+ // bytes. This is 21 bits which is too big for a 16 bit utf16 code
+ // unit so we use surrogates.
+ let codepoint = ((c1 & 0x7) << 18) | ((c2 & 0x3F) << 12) |
+ ((c3 & 0x3F) << 6) | (c4 & 0x3F);
+ // Consistency check, this is the valid range for a 4 byte character.
+ jspb.asserts.assert(codepoint >= 0x10000 && codepoint <= 0x10FFFF);
+ // Surrogates formula from wikipedia.
+ // 1. Subtract 0x10000 from codepoint
+ codepoint -= 0x10000;
+ // 2. Split this into the high 10-bit value and the low 10-bit value
+ // 3. Add 0xD800 to the high value to form the high surrogate
+ // 4. Add 0xDC00 to the low value to form the low surrogate:
+ const low = (codepoint & 0x3FF) + MIN_LOW_SURROGATE;
+ const high = ((codepoint >> 10) & 0x3FF) + MIN_HIGH_SURROGATE;
+ codeUnits.push(high, low);
+ }
+ }
+ } else {
+ // initial byte is too large for utf8
+ invalid(parsingErrorsAreFatal, codeUnits);
+ }
+ // Accumulate as we go to avoid exceeding the maximum stack size when
+ // calling `apply`.
+ if (codeUnits.length >= 8192) {
+ result = codeUnitsToString(result, codeUnits);
+ codeUnits.length = 0;
+ }
+ }
+ // ensure we don't overflow or underflow
+ jspb.asserts.assert(cursor === end, `expected ${cursor} === ${end}`);
+ return codeUnitsToString(result, codeUnits);
+}
+
+
+/** @type {boolean|undefined} */
+let isFatalTextDecoderCachableAfterThrowing_ =
+ // chrome version >= 2020 are not subject to https://crbug.com/910292
+ goog.FEATURESET_YEAR >= 2020 ? true : undefined;
+
+/** @return {boolean} */
+function isFatalTextDecoderCachableAfterThrowing(/** !TextDecoder */ decoder) {
+ // Test if the decoder is subject to https://crbug.com/910292
+ // chrome versions with this bug cause one failed decode to cause all later
+ // decodes to throw.
+ if (isFatalTextDecoderCachableAfterThrowing_ === undefined) {
+ // In theory we shouldn't need to generate an error here since this function
+ // is only called in the context of a failed decode. However, the buggy
+ // chrome versions are not 'consistent' in corrupting their internal state
+ // since it depends on where in the decode stream the error occurs. This
+ // error however does consistently trigger the bug based on manual testing.
+ try {
+ // A lonely continuation byte
+ decoder.decode(new Uint8Array([0x80]));
+ } catch (e) {
+ // expected
+ }
+ try {
+ // 'a' in hex
+ decoder.decode(new Uint8Array([0x61]));
+ isFatalTextDecoderCachableAfterThrowing_ = true;
+ } catch (e) {
+ // This decode should not throw, if it does it means our chrome version
+ // is buggy and we need to flush our cached decoder when failures occur
+ isFatalTextDecoderCachableAfterThrowing_ = false;
+ }
+ }
+ return isFatalTextDecoderCachableAfterThrowing_;
+}
+
+/** @type {!TextDecoder|undefined} */
+let fatalDecoderInstance;
+
+/** @return {!TextDecoder}*/
+function getFatalDecoderInstance() {
+ let instance = fatalDecoderInstance;
+ if (!instance) {
+ instance = fatalDecoderInstance = new TextDecoder('utf-8', { fatal: true });
+ }
+ return instance;
+}
+
+/** @type {!TextDecoder|undefined} */
+let nonFatalDecoderInstance;
+
+/** @return {!TextDecoder}*/
+function getNonFatalDecoderInstance() {
+ let instance = nonFatalDecoderInstance;
+ if (!instance) {
+ instance = nonFatalDecoderInstance =
+ new TextDecoder('utf-8', { fatal: false });
+ }
+ return instance;
+}
+
+/**
+ * A `subarray` implementation that avoids calling `subarray` if it isn't needed
+ *
+ * `subarray` tends to be surprisingly slow.
+ * @return {!Uint8Array}
+ */
+function subarray(
+ /** !Uint8Array*/ bytes, /** number */ offset, /** number */ end) {
+ return offset === 0 && end === bytes.length ? bytes :
+ bytes.subarray(offset, end);
+}
+
+/**
+ * @return {string}
+ */
+jspb.binary.utf8.textDecoderDecodeUtf8 = function (
+ /** !Uint8Array*/ bytes, /** number */ offset, /** number */ length,
+ /** boolean*/ parsingErrorsAreFatal) {
+ const /** !TextDecoder */ decoder = parsingErrorsAreFatal ?
+ getFatalDecoderInstance() :
+ getNonFatalDecoderInstance();
+
+ bytes = subarray(bytes, offset, offset + length);
+ try {
+ return decoder.decode(bytes);
+ } catch (e) {
+ if (parsingErrorsAreFatal &&
+ !isFatalTextDecoderCachableAfterThrowing(decoder)) {
+ fatalDecoderInstance = undefined;
+ }
+ throw e;
+ }
+}
+
+/** @const {boolean} */
+const useTextDecoderDecode =
+ USE_TEXT_ENCODING || typeof TextDecoder !== 'undefined';
+
+/**
+ * A utf8 decoding routine either based upon TextDecoder if available or using
+ * our polyfill implementation
+ * @return {string}
+ */
+jspb.binary.utf8.decodeUtf8 = function (
+ /** !Uint8Array*/ bytes, /** number */ offset, /** number */ length,
+ /** boolean*/ parsingErrorsAreFatal) {
+ return useTextDecoderDecode ?
+ jspb.binary.utf8.textDecoderDecodeUtf8(bytes, offset, length, parsingErrorsAreFatal) :
+ jspb.binary.utf8.polyfillDecodeUtf8(bytes, offset, length, parsingErrorsAreFatal);
+}
+
+/** @type {!TextEncoder|undefined} */
+let textEncoderInstance;
+
+/** @return {!Uint8Array} */
+jspb.binary.utf8.textEncoderEncode = function (
+ /** string */ s, /** boolean */ rejectUnpairedSurrogates) {
+ if (rejectUnpairedSurrogates) {
+ checkWellFormed(s);
+ }
+
+ if (!textEncoderInstance) {
+ textEncoderInstance = new TextEncoder();
+ }
+ return textEncoderInstance.encode(s);
+}
+
+// isWellFormed landed in major browsers in early 2023 so it will only be
+// definitely available in 2024 See
+// http://go/mdn/JavaScript/Reference/Global_Objects/String/isWellFormed
+const /** boolean */ HAS_WELL_FORMED_METHOD = goog.FEATURESET_YEAR > 2023 ||
+ typeof String.prototype.isWellFormed === 'function';
+
+jspb.binary.utf8.checkWellFormed = function (/** string */ text) {
+ if (HAS_WELL_FORMED_METHOD ?
+ // Externs don't contain the definition of this function yet.
+ // http://go/mdn/JavaScript/Reference/Global_Objects/String/isWellFormed
+ !(/** @type{{isWellFormed:function():boolean}}*/ (
+ /** @type {?} */ (text))
+ .isWellFormed()) :
+ /(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])/
+ .test(text)) {
+ throw new Error('Found an unpaired surrogate');
+ }
+}
+
+
+/** @return {!Uint8Array} */
+jspb.binary.utf8.polyfillEncode = function (
+ /** string */ s, /** boolean */ rejectUnpairedSurrogates) {
+ let bi = 0;
+ // The worse case is that every character requires 3 output bytes, so we
+ // allocate for this. This assumes that the buffer will be short lived.
+ // Callers can always `slice` if needed
+ const buffer = new Uint8Array(3 * s.length);
+ for (let ci = 0; ci < s.length; ci++) {
+ let c = s.charCodeAt(ci);
+ if (c < 0x80) {
+ buffer[bi++] = c;
+ } else if (c < 0x800) {
+ buffer[bi++] = (c >> 6) | 0xC0;
+ buffer[bi++] = (c & 63) | 0x80;
+ } else {
+ jspb.asserts.assert(c < 65536);
+ // Look for surrogates
+ // First check if it is surrogate range
+ if (c >= MIN_SURROGATE && c <= MAX_SURROGATE) {
+ // is it a high surrogate?
+ if (c <= MAX_HIGH_SURROGATE && ci < s.length) {
+ const c2 = s.charCodeAt(++ci);
+ if (c2 >= MIN_LOW_SURROGATE && c2 <= MAX_LOW_SURROGATE) {
+ // http://mathiasbynens.be/notes/javascript-encoding#surrogate-formulae
+ const codePoint =
+ (c - MIN_SURROGATE) * 0x400 + c2 - MIN_LOW_SURROGATE + 0x10000;
+ buffer[bi++] = (codePoint >> 18) | 0xF0;
+ buffer[bi++] = ((codePoint >> 12) & 63) | 0x80;
+ buffer[bi++] = ((codePoint >> 6) & 63) | 0x80;
+ buffer[bi++] = (codePoint & 63) | 0x80;
+ continue;
+ } else {
+ // else c2 not in low surrogate range, treat c as a lone surrogate
+ // and back up ci so we process c2 on the next loop as an
+ // independent character
+ ci--;
+ }
+ } // else c not a high surrogate
+ if (rejectUnpairedSurrogates) {
+ throw new Error('Found an unpaired surrogate');
+ }
+ c = 0xFFFD; // Error! Unpaired surrogate
+ }
+ buffer[bi++] = (c >> 12) | 0xE0;
+ buffer[bi++] = ((c >> 6) & 63) | 0x80;
+ buffer[bi++] = (c & 63) | 0x80;
+ }
+ }
+ return subarray(buffer, 0, bi);
+}
+
+/** @const {boolean} */
+const useTextEncoderEncode =
+ (USE_TEXT_ENCODING || typeof TextEncoder !== 'undefined');
+
+/**
+ * A utf8 encoding routine either based upon TextEncoder if available or using
+ * our polyfill implementation
+ * @return {!Uint8Array}
+ */
+jspb.binary.utf8.encodeUtf8 = function (
+ /**string*/ string, /** boolean=*/ rejectUnpairedSurrogates = false) {
+ jspb.asserts.assertString(string);
+ return useTextEncoderEncode ?
+ jspb.binary.utf8.textEncoderEncode(string, rejectUnpairedSurrogates) :
+ jspb.binary.utf8.polyfillEncode(string, rejectUnpairedSurrogates);
+}
+
diff --git a/generator/js_generator.cc b/generator/js_generator.cc
index 84365dc..8136f58 100644
--- a/generator/js_generator.cc
+++ b/generator/js_generator.cc
@@ -1073,17 +1073,21 @@ std::string JSFieldTypeAnnotation(const GeneratorOptions& options,
return jstype;
}
-std::string JSBinaryReaderMethodType(const FieldDescriptor* field) {
+std::string JSBinaryMethodType(const FieldDescriptor* field, bool is_writer) {
std::string name = field->type_name();
if (name[0] >= 'a' && name[0] <= 'z') {
name[0] = (name[0] - 'a') + 'A';
}
+ if (!is_writer && field->type() == FieldDescriptor::TYPE_STRING &&
+ field->file()->syntax() == FileDescriptor::SYNTAX_PROTO3) {
+ name = name + "RequireUtf8";
+ }
return IsIntegralFieldWithStringJSType(field) ? (name + "String") : name;
}
std::string JSBinaryReadWriteMethodName(const FieldDescriptor* field,
bool is_writer) {
- std::string name = JSBinaryReaderMethodType(field);
+ std::string name = JSBinaryMethodType(field, is_writer);
if (field->is_packed()) {
name = "Packed" + name;
} else if (is_writer && field->is_repeated()) {
@@ -3128,11 +3132,11 @@ void Generator::GenerateClassDeserializeBinaryField(
printer->Print(
" var values = /** @type {$fieldtype$} */ "
"(reader.isDelimited() "
- "? reader.readPacked$reader$() : [reader.read$reader$()]);\n",
+ "? reader.read$reader$() : [reader.read$reader$()]);\n",
"fieldtype",
JSFieldTypeAnnotation(options, field, false, true,
/* singular_if_not_packed */ false, BYTES_U8),
- "reader", JSBinaryReaderMethodType(field));
+ "reader", JSBinaryReadWriteMethodName(field, /* is_writer=*/false));
} else {
printer->Print(
" var value = /** @type {$fieldtype$} */ "
diff --git a/gulpfile.js b/gulpfile.js
index e7f7511..426e0f1 100644
--- a/gulpfile.js
+++ b/gulpfile.js
@@ -145,6 +145,7 @@ function getClosureCompilerCommand(exportsFile, outputFile) {
'--js=binary/decoder.js',
'--js=binary/encoder.js',
'--js=binary/reader.js',
+ '--js=binary/utf8.js',
'--js=binary/utils.js',
'--js=binary/writer.js',
`--js=${exportsFile}`,
@@ -194,7 +195,7 @@ function commonjs_out(cb) {
function closure_make_deps(cb) {
exec(
- './node_modules/.bin/closure-make-deps --closure-path=. --file=node_modules/google-closure-library/closure/goog/deps.js binary/arith.js binary/constants.js binary/decoder.js binary/encoder.js binary/reader.js binary/utils.js binary/writer.js asserts.js debug.js map.js message.js node_loader.js test_bootstrap.js > deps.js',
+ './node_modules/.bin/closure-make-deps --closure-path=. --file=node_modules/google-closure-library/closure/goog/deps.js binary/arith.js binary/constants.js binary/decoder.js binary/encoder.js binary/reader.js binary/utf8.js binary/utils.js binary/writer.js asserts.js debug.js map.js message.js node_loader.js test_bootstrap.js > deps.js',
make_exec_logging_callback(cb));
}