From 8ae6c6a750aa7b734fd39b2bb5c3f93b2930f998 Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Sat, 8 Jun 2024 14:11:28 +0100 Subject: [PATCH 01/11] count tokens more changes don't increment token count for NOT_AVAILABLE handle case when token is set to null use _updateTokenToNull null check not needed now add test Update TokenCountTest.java Update StreamReadConstraints.java add validation try to reduce overhead for when maxTokenCount is not needed Update StreamReadConstraints.java Update TokenCountTest.java --- .../fasterxml/jackson/core/JsonParser.java | 12 ++ .../jackson/core/StreamReadConstraints.java | 112 +++++++++++++++++- .../jackson/core/base/ParserMinimalBase.java | 29 +++++ .../core/json/ReaderBasedJsonParser.java | 56 ++++----- .../core/json/UTF8DataInputJsonParser.java | 47 ++++---- .../core/json/UTF8StreamJsonParser.java | 66 +++++------ .../json/async/NonBlockingJsonParserBase.java | 21 ++-- .../async/NonBlockingUtf8JsonParserBase.java | 78 ++++++------ .../jackson/core/read/TokenCountTest.java | 95 +++++++++++++++ 9 files changed, 375 insertions(+), 141 deletions(-) create mode 100644 src/test/java/com/fasterxml/jackson/core/read/TokenCountTest.java diff --git a/src/main/java/com/fasterxml/jackson/core/JsonParser.java b/src/main/java/com/fasterxml/jackson/core/JsonParser.java index 4265d48437..db8d93bf2e 100644 --- a/src/main/java/com/fasterxml/jackson/core/JsonParser.java +++ b/src/main/java/com/fasterxml/jackson/core/JsonParser.java @@ -2546,6 +2546,18 @@ public T readValueAsTree() throws IOException { return (T) _codec().readTree(this); } + /** + * Get an approximate count of the number of tokens that have been read. + * This count is likely to be only updated if {@link StreamReadConstraints.Builder.maxTokenCount(long)} + * has been used to set a limit on the number of tokens that can be read. + * + * @return the number of tokens that have been read (-1 if the count is not available) + * @since 2.18 + */ + public long getTokenCount() { + return -1L; + } + protected ObjectCodec _codec() { ObjectCodec c = getCodec(); if (c == null) { diff --git a/src/main/java/com/fasterxml/jackson/core/StreamReadConstraints.java b/src/main/java/com/fasterxml/jackson/core/StreamReadConstraints.java index c7586d2d3c..b460ab4152 100644 --- a/src/main/java/com/fasterxml/jackson/core/StreamReadConstraints.java +++ b/src/main/java/com/fasterxml/jackson/core/StreamReadConstraints.java @@ -43,6 +43,12 @@ public class StreamReadConstraints */ public static final long DEFAULT_MAX_DOC_LEN = -1L; + /** + * Default setting for maximum token count: + * see {@link Builder#maxTokenCount} for details. + */ + public static final long DEFAULT_MAX_TOKEN_COUNT = -1L; + /** * @since 2.16 */ @@ -74,6 +80,7 @@ public class StreamReadConstraints protected final int _maxNestingDepth; protected final long _maxDocLen; + protected final long _maxTokenCount; protected final int _maxNumLen; protected final int _maxStringLen; @@ -112,6 +119,7 @@ public static void overrideDefaultStreamReadConstraints(final StreamReadConstrai public static final class Builder { private long maxDocLen; + private long maxTokenCount; private int maxNestingDepth; private int maxNumLen; private int maxStringLen; @@ -156,6 +164,31 @@ public Builder maxDocumentLength(long maxDocLen) { return this; } + /** + * Sets the maximum allowed token count (for positive values over 0) or + * indicate that any count is acceptable ({@code 0} or negative number). + * + *

+ * A token is a single unit of input, such as a number, a string, an object + * start or end, or an array start or end. + *

+ * + * @param maxTokenCount the maximum allowed token count if positive number above 0; otherwise + * ({@code 0} or negative number) means "unlimited". + * + * @return this builder + * + * @since 2.18 + */ + public Builder maxTokenCount(long maxTokenCount) { + // Negative values and 0 mean "unlimited", mark with -1L + if (maxTokenCount <= 0L) { + maxTokenCount = -1L; + } + this.maxTokenCount = maxTokenCount; + return this; + } + /** * Sets the maximum number length (in chars or bytes, depending on input context). * The default is 1000. @@ -220,14 +253,15 @@ public Builder maxNameLength(final int maxNameLen) { } Builder() { - this(DEFAULT_MAX_DEPTH, DEFAULT_MAX_DOC_LEN, + this(DEFAULT_MAX_DEPTH, DEFAULT_MAX_DOC_LEN, DEFAULT_MAX_TOKEN_COUNT, DEFAULT_MAX_NUM_LEN, DEFAULT_MAX_STRING_LEN, DEFAULT_MAX_NAME_LEN); } - Builder(final int maxNestingDepth, final long maxDocLen, + Builder(final int maxNestingDepth, final long maxDocLen, final long maxTokenCount, final int maxNumLen, final int maxStringLen, final int maxNameLen) { this.maxNestingDepth = maxNestingDepth; this.maxDocLen = maxDocLen; + this.maxTokenCount = maxTokenCount; this.maxNumLen = maxNumLen; this.maxStringLen = maxStringLen; this.maxNameLen = maxNameLen; @@ -236,6 +270,7 @@ public Builder maxNameLength(final int maxNameLen) { Builder(StreamReadConstraints src) { maxNestingDepth = src._maxNestingDepth; maxDocLen = src._maxDocLen; + maxTokenCount = src._maxTokenCount; maxNumLen = src._maxNumLen; maxStringLen = src._maxStringLen; maxNameLen = src._maxNameLen; @@ -243,7 +278,7 @@ public Builder maxNameLength(final int maxNameLen) { public StreamReadConstraints build() { return new StreamReadConstraints(maxNestingDepth, maxDocLen, - maxNumLen, maxStringLen, maxNameLen); + maxNumLen, maxStringLen, maxNameLen, maxTokenCount); } } @@ -257,7 +292,7 @@ public StreamReadConstraints build() { protected StreamReadConstraints(final int maxNestingDepth, final long maxDocLen, final int maxNumLen, final int maxStringLen) { this(maxNestingDepth, maxDocLen, - maxNumLen, maxStringLen, DEFAULT_MAX_NAME_LEN); + maxNumLen, maxStringLen, DEFAULT_MAX_NAME_LEN, DEFAULT_MAX_TOKEN_COUNT); } /** @@ -269,13 +304,30 @@ protected StreamReadConstraints(final int maxNestingDepth, final long maxDocLen, * * @since 2.16 */ + @Deprecated // since 2.18 + protected StreamReadConstraints(final int maxNestingDepth, final long maxDocLen, + final int maxNumLen, final int maxStringLen, final int maxNameLen) { + this(maxNestingDepth, maxDocLen, maxNumLen, maxStringLen, maxNameLen, DEFAULT_MAX_TOKEN_COUNT); + } + + /** + * @param maxNestingDepth Maximum input document nesting to allow + * @param maxDocLen Maximum input document length to allow + * @param maxNumLen Maximum number representation length to allow + * @param maxStringLen Maximum String value length to allow + * @param maxNameLen Maximum Object property name length to allow + * @param maxTokenCount Maximum number of tokens to allow + * + * @since 2.18 + */ protected StreamReadConstraints(final int maxNestingDepth, final long maxDocLen, - final int maxNumLen, final int maxStringLen, final int maxNameLen) { + final int maxNumLen, final int maxStringLen, final int maxNameLen, final long maxTokenCount) { _maxNestingDepth = maxNestingDepth; _maxDocLen = maxDocLen; _maxNumLen = maxNumLen; _maxStringLen = maxStringLen; _maxNameLen = maxNameLen; + _maxTokenCount = maxTokenCount; } public static Builder builder() { @@ -337,6 +389,31 @@ public boolean hasMaxDocumentLength() { return _maxDocLen > 0L; } + /** + * Accessor for maximum token count. + * see {@link Builder#maxTokenCount(long)} for details. + * + * @return Maximum allowed token count + * @since 2.18 + */ + public long getMaxTokenCount() { + return _maxTokenCount; + } + + /** + * Convenience method, basically same as: + *
+     *  getMaxTokenCount() > 0L
+     *
+ * + * @return {@code True} if this constraints instance has a limit for maximum + * token count to enforce; {@code false} otherwise. + * @since 2.18 + */ + public boolean hasMaxTokenCount() { + return _maxTokenCount > 0L; + } + /** * Accessor for maximum length of numbers to decode. * see {@link Builder#maxNumberLength(int)} for details. @@ -419,6 +496,31 @@ public void validateDocumentLength(long len) throws StreamConstraintsException } } + /** + * Convenience method that can be used to verify that the + * token count does not exceed the maximum specified by this + * constraints object (if any): if it does, a + * {@link StreamConstraintsException} + * is thrown. + * + * @param count Current token count for processed document content + * + * @throws StreamConstraintsException If length exceeds maximum + * + * @since 2.18 + */ + public void validateTokenCount(long count) throws StreamConstraintsException + { + // for performance reasons, it is assumed that users check hasMaxTokenCount() + // before calling this method - this method will not work properly if hasMaxTokenCount() is false + if (count > _maxTokenCount) { + throw _constructException( + "Token count (%d) exceeds the maximum allowed (%d, from %s)", + count, _maxTokenCount, + _constrainRef("getMaxTokenCount")); + } + } + /* /********************************************************************** /* Convenience methods for validation, token lengths diff --git a/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java b/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java index c9f438bb66..4da5b9cce5 100644 --- a/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java +++ b/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.core.*; import com.fasterxml.jackson.core.exc.InputCoercionException; +import com.fasterxml.jackson.core.exc.StreamConstraintsException; import com.fasterxml.jackson.core.io.JsonEOFException; import com.fasterxml.jackson.core.io.NumberInput; import com.fasterxml.jackson.core.util.ByteArrayBuilder; @@ -158,6 +159,13 @@ public abstract class ParserMinimalBase extends JsonParser */ protected JsonToken _currToken; + /** + * Current count of tokens + * + * @since 2.18 + */ + protected long _tokenCount; + /** * Last cleared token, if any: that is, value that was in * effect when {@link #clearCurrentToken} was called. @@ -538,6 +546,11 @@ public String getValueAsString(String defaultValue) throws IOException { return getText(); } + @Override + public long getTokenCount() { + return _tokenCount; + } + /* /********************************************************** /* Base64 decoding @@ -818,6 +831,22 @@ protected final void _wrapError(String msg, Throwable t) throws JsonParseExcepti throw _constructReadException(msg, t); } + protected final JsonToken _updateToken(final JsonToken token) throws StreamConstraintsException { + _currToken = token; + if (streamReadConstraints().hasMaxTokenCount() && token != JsonToken.NOT_AVAILABLE) { + streamReadConstraints().validateTokenCount(++_tokenCount); + } + return token; + } + + protected final JsonToken _updateTokenToNull() { + return (_currToken = null); + } + + protected final JsonToken _updateTokenToNA() { + return (_currToken = JsonToken.NOT_AVAILABLE); + } + @Deprecated // since 2.11 protected static byte[] _asciiBytes(String str) { byte[] b = new byte[str.length()]; diff --git a/src/main/java/com/fasterxml/jackson/core/json/ReaderBasedJsonParser.java b/src/main/java/com/fasterxml/jackson/core/json/ReaderBasedJsonParser.java index 1876b9e1d4..29517f05f1 100644 --- a/src/main/java/com/fasterxml/jackson/core/json/ReaderBasedJsonParser.java +++ b/src/main/java/com/fasterxml/jackson/core/json/ReaderBasedJsonParser.java @@ -3,6 +3,7 @@ import java.io.*; import com.fasterxml.jackson.core.*; +import com.fasterxml.jackson.core.exc.StreamConstraintsException; import com.fasterxml.jackson.core.io.CharTypes; import com.fasterxml.jackson.core.io.IOContext; import com.fasterxml.jackson.core.sym.CharsToNameCanonicalizer; @@ -675,7 +676,7 @@ public final JsonToken nextToken() throws IOException // Should actually close/release things // like input source, symbol table and recyclable buffers now. close(); - return (_currToken = null); + return _updateTokenToNull(); } // clear any data retained so far _binaryValue = null; @@ -708,7 +709,7 @@ public final JsonToken nextToken() throws IOException _updateNameLocation(); String name = (i == INT_QUOTE) ? _parseName() : _handleOddName(i); _parsingContext.setCurrentName(name); - _currToken = JsonToken.FIELD_NAME; + _updateToken(JsonToken.FIELD_NAME); i = _skipColon(); } _updateLocation(); @@ -785,7 +786,7 @@ public final JsonToken nextToken() throws IOException _nextToken = t; return _currToken; } - _currToken = t; + _updateToken(t); return t; } @@ -803,7 +804,7 @@ private final JsonToken _nextAfterName() throws IOException } else if (t == JsonToken.START_OBJECT) { createChildObjectContext(_tokenInputRow, _tokenInputCol); } - return (_currToken = t); + return _updateToken(t); } @Override @@ -837,7 +838,7 @@ public boolean nextFieldName(SerializableString sstr) throws IOException int i = _skipWSOrEnd(); if (i < 0) { close(); - _currToken = null; + _updateTokenToNull(); return false; } _binaryValue = null; @@ -913,7 +914,7 @@ public String nextFieldName() throws IOException int i = _skipWSOrEnd(); if (i < 0) { close(); - _currToken = null; + _updateTokenToNull(); return null; } _binaryValue = null; @@ -939,7 +940,7 @@ public String nextFieldName() throws IOException _updateNameLocation(); String name = (i == INT_QUOTE) ? _parseName() : _handleOddName(i); _parsingContext.setCurrentName(name); - _currToken = JsonToken.FIELD_NAME; + _updateToken(JsonToken.FIELD_NAME); i = _skipColon(); _updateLocation(); @@ -1007,7 +1008,7 @@ public String nextFieldName() throws IOException private final void _isNextTokenNameYes(int i) throws IOException { - _currToken = JsonToken.FIELD_NAME; + _updateToken(JsonToken.FIELD_NAME); _updateLocation(); switch (i) { @@ -1067,7 +1068,7 @@ protected boolean _isNextTokenNameMaybe(int i, String nameToMatch) throws IOExce // // // and this is back to standard nextToken() String name = (i == INT_QUOTE) ? _parseName() : _handleOddName(i); _parsingContext.setCurrentName(name); - _currToken = JsonToken.FIELD_NAME; + _updateToken(JsonToken.FIELD_NAME); i = _skipColon(); _updateLocation(); if (i == INT_QUOTE) { @@ -1133,32 +1134,32 @@ private final JsonToken _nextTokenNotInObject(int i) throws IOException { if (i == INT_QUOTE) { _tokenIncomplete = true; - return (_currToken = JsonToken.VALUE_STRING); + return _updateToken(JsonToken.VALUE_STRING); } switch (i) { case '[': createChildArrayContext(_tokenInputRow, _tokenInputCol); - return (_currToken = JsonToken.START_ARRAY); + return _updateToken(JsonToken.START_ARRAY); case '{': createChildObjectContext(_tokenInputRow, _tokenInputCol); - return (_currToken = JsonToken.START_OBJECT); + return _updateToken(JsonToken.START_OBJECT); case 't': _matchToken("true", 1); - return (_currToken = JsonToken.VALUE_TRUE); + return _updateToken(JsonToken.VALUE_TRUE); case 'f': _matchToken("false", 1); - return (_currToken = JsonToken.VALUE_FALSE); + return _updateToken(JsonToken.VALUE_FALSE); case 'n': _matchToken("null", 1); - return (_currToken = JsonToken.VALUE_NULL); + return _updateToken(JsonToken.VALUE_NULL); case '-': - return (_currToken = _parseSignedNumber(true)); + return _updateToken(_parseSignedNumber(true)); /* Should we have separate handling for plus? Although * it is not allowed per se, it may be erroneously used, * and could be indicated by a more specific error message. */ case '.': // [core#61]] - return (_currToken = _parseFloatThatStartsWithPeriod(false)); + return _updateToken(_parseFloatThatStartsWithPeriod(false)); case '0': case '1': case '2': @@ -1169,7 +1170,7 @@ private final JsonToken _nextTokenNotInObject(int i) throws IOException case '7': case '8': case '9': - return (_currToken = _parseUnsignedNumber(i)); + return _updateToken(_parseUnsignedNumber(i)); /* * This check proceeds only if the Feature.ALLOW_MISSING_VALUES is enabled * The Check is for missing values. In case of missing values in an array, the next token will be either ',' or ']'. @@ -1184,11 +1185,11 @@ private final JsonToken _nextTokenNotInObject(int i) throws IOException if (!_parsingContext.inRoot()) { if ((_features & FEAT_MASK_ALLOW_MISSING) != 0) { --_inputPtr; - return (_currToken = JsonToken.VALUE_NULL); + return _updateToken(JsonToken.VALUE_NULL); } } } - return (_currToken = _handleOddValue(i)); + return _updateToken(_handleOddValue(i)); } // note: identical to one in UTF8StreamJsonParser @Override @@ -1198,7 +1199,7 @@ public final String nextTextValue() throws IOException _nameCopied = false; JsonToken t = _nextToken; _nextToken = null; - _currToken = t; + _updateToken(t); if (t == JsonToken.VALUE_STRING) { if (_tokenIncomplete) { _tokenIncomplete = false; @@ -1225,7 +1226,7 @@ public final int nextIntValue(int defaultValue) throws IOException _nameCopied = false; JsonToken t = _nextToken; _nextToken = null; - _currToken = t; + _updateToken(t); if (t == JsonToken.VALUE_NUMBER_INT) { return getIntValue(); } @@ -1248,7 +1249,7 @@ public final long nextLongValue(long defaultValue) throws IOException _nameCopied = false; JsonToken t = _nextToken; _nextToken = null; - _currToken = t; + _updateToken(t); if (t == JsonToken.VALUE_NUMBER_INT) { return getLongValue(); } @@ -1271,7 +1272,7 @@ public final Boolean nextBooleanValue() throws IOException _nameCopied = false; JsonToken t = _nextToken; _nextToken = null; - _currToken = t; + _updateToken(t); if (t == JsonToken.VALUE_TRUE) { return Boolean.TRUE; } @@ -3024,15 +3025,14 @@ protected void _reportInvalidToken(String matchedPart, String msg) throws IOExce /********************************************************** */ - private void _closeScope(int i) throws JsonParseException - { + private void _closeScope(int i) throws JsonParseException, StreamConstraintsException { if (i == INT_RBRACKET) { _updateLocation(); if (!_parsingContext.inArray()) { _reportMismatchedEndMarker(i, '}'); } _parsingContext = _parsingContext.clearAndGetParent(); - _currToken = JsonToken.END_ARRAY; + _updateToken(JsonToken.END_ARRAY); } if (i == INT_RCURLY) { _updateLocation(); @@ -3040,7 +3040,7 @@ private void _closeScope(int i) throws JsonParseException _reportMismatchedEndMarker(i, ']'); } _parsingContext = _parsingContext.clearAndGetParent(); - _currToken = JsonToken.END_OBJECT; + _updateToken(JsonToken.END_OBJECT); } } } diff --git a/src/main/java/com/fasterxml/jackson/core/json/UTF8DataInputJsonParser.java b/src/main/java/com/fasterxml/jackson/core/json/UTF8DataInputJsonParser.java index 8d2954dd02..9d6a01315b 100644 --- a/src/main/java/com/fasterxml/jackson/core/json/UTF8DataInputJsonParser.java +++ b/src/main/java/com/fasterxml/jackson/core/json/UTF8DataInputJsonParser.java @@ -560,7 +560,7 @@ public JsonToken nextToken() throws IOException if (i < 0) { // end-of-input // Close/release things like input source, symbol table and recyclable buffers close(); - return (_currToken = null); + return _updateTokenToNull(); } // clear any data retained so far _binaryValue = null; @@ -598,7 +598,7 @@ public JsonToken nextToken() throws IOException // So first parse the field name itself: String n = _parseName(i); _parsingContext.setCurrentName(n); - _currToken = JsonToken.FIELD_NAME; + _updateToken(JsonToken.FIELD_NAME); i = _skipColon(); @@ -666,33 +666,33 @@ private final JsonToken _nextTokenNotInObject(int i) throws IOException { if (i == INT_QUOTE) { _tokenIncomplete = true; - return (_currToken = JsonToken.VALUE_STRING); + return _updateToken(JsonToken.VALUE_STRING); } switch (i) { case '[': createChildArrayContext(_tokenInputRow, _tokenInputCol); - return (_currToken = JsonToken.START_ARRAY); + return _updateToken(JsonToken.START_ARRAY); case '{': createChildObjectContext(_tokenInputRow, _tokenInputCol); - return (_currToken = JsonToken.START_OBJECT); + return _updateToken(JsonToken.START_OBJECT); case 't': _matchToken("true", 1); - return (_currToken = JsonToken.VALUE_TRUE); + return _updateToken(JsonToken.VALUE_TRUE); case 'f': _matchToken("false", 1); - return (_currToken = JsonToken.VALUE_FALSE); + return _updateToken(JsonToken.VALUE_FALSE); case 'n': _matchToken("null", 1); - return (_currToken = JsonToken.VALUE_NULL); + return _updateToken(JsonToken.VALUE_NULL); case '-': - return (_currToken = _parseNegNumber()); + return _updateToken(_parseNegNumber()); case '+': if (isEnabled(JsonReadFeature.ALLOW_LEADING_PLUS_SIGN_FOR_NUMBERS.mappedFeature())) { - return (_currToken = _parsePosNumber()); + return _updateToken(_parsePosNumber()); } - return (_currToken = _handleUnexpectedValue(i)); + return _updateToken(_handleUnexpectedValue(i)); case '.': // as per [core#611] - return (_currToken = _parseFloatThatStartsWithPeriod(false, false)); + return _updateToken(_parseFloatThatStartsWithPeriod(false, false)); case '0': case '1': case '2': @@ -703,9 +703,9 @@ private final JsonToken _nextTokenNotInObject(int i) throws IOException case '7': case '8': case '9': - return (_currToken = _parseUnsignedNumber(i)); + return _updateToken(_parseUnsignedNumber(i)); } - return (_currToken = _handleUnexpectedValue(i)); + return _updateToken(_handleUnexpectedValue(i)); } private final JsonToken _nextAfterName() throws IOException @@ -720,7 +720,7 @@ private final JsonToken _nextAfterName() throws IOException } else if (t == JsonToken.START_OBJECT) { createChildObjectContext(_tokenInputRow, _tokenInputCol); } - return (_currToken = t); + return _updateToken(t); } @Override @@ -785,7 +785,7 @@ public String nextFieldName() throws IOException final String nameStr = _parseName(i); _parsingContext.setCurrentName(nameStr); - _currToken = JsonToken.FIELD_NAME; + _updateToken(JsonToken.FIELD_NAME); i = _skipColon(); if (i == INT_QUOTE) { @@ -853,7 +853,7 @@ public String nextTextValue() throws IOException _nameCopied = false; JsonToken t = _nextToken; _nextToken = null; - _currToken = t; + _updateToken(t); if (t == JsonToken.VALUE_STRING) { if (_tokenIncomplete) { _tokenIncomplete = false; @@ -879,7 +879,7 @@ public int nextIntValue(int defaultValue) throws IOException _nameCopied = false; JsonToken t = _nextToken; _nextToken = null; - _currToken = t; + _updateToken(t); if (t == JsonToken.VALUE_NUMBER_INT) { return getIntValue(); } @@ -901,7 +901,7 @@ public long nextLongValue(long defaultValue) throws IOException _nameCopied = false; JsonToken t = _nextToken; _nextToken = null; - _currToken = t; + _updateToken(t); if (t == JsonToken.VALUE_NUMBER_INT) { return getLongValue(); } @@ -923,7 +923,7 @@ public Boolean nextBooleanValue() throws IOException _nameCopied = false; JsonToken t = _nextToken; _nextToken = null; - _currToken = t; + _updateToken(t); if (t == JsonToken.VALUE_TRUE) { return Boolean.TRUE; } @@ -2958,21 +2958,20 @@ public JsonLocation currentTokenLocation() { /********************************************************** */ - private void _closeScope(int i) throws JsonParseException - { + private void _closeScope(int i) throws JsonParseException, StreamConstraintsException { if (i == INT_RBRACKET) { if (!_parsingContext.inArray()) { _reportMismatchedEndMarker(i, '}'); } _parsingContext = _parsingContext.clearAndGetParent(); - _currToken = JsonToken.END_ARRAY; + _updateToken(JsonToken.END_ARRAY); } if (i == INT_RCURLY) { if (!_parsingContext.inObject()) { _reportMismatchedEndMarker(i, ']'); } _parsingContext = _parsingContext.clearAndGetParent(); - _currToken = JsonToken.END_OBJECT; + _updateToken(JsonToken.END_OBJECT); } } diff --git a/src/main/java/com/fasterxml/jackson/core/json/UTF8StreamJsonParser.java b/src/main/java/com/fasterxml/jackson/core/json/UTF8StreamJsonParser.java index 80f7c8a836..a8164858ac 100644 --- a/src/main/java/com/fasterxml/jackson/core/json/UTF8StreamJsonParser.java +++ b/src/main/java/com/fasterxml/jackson/core/json/UTF8StreamJsonParser.java @@ -717,7 +717,7 @@ public JsonToken nextToken() throws IOException if (i < 0) { // end-of-input // Close/release things like input source, symbol table and recyclable buffers close(); - return (_currToken = null); + return _updateTokenToNull(); } // clear any data retained so far _binaryValue = null; @@ -725,11 +725,11 @@ public JsonToken nextToken() throws IOException // Closing scope? if (i == INT_RBRACKET) { _closeArrayScope(); - return (_currToken = JsonToken.END_ARRAY); + return _updateToken(JsonToken.END_ARRAY); } if (i == INT_RCURLY) { _closeObjectScope(); - return (_currToken = JsonToken.END_OBJECT); + return _updateToken(JsonToken.END_OBJECT); } // Nope: do we then expect a comma? @@ -756,7 +756,7 @@ public JsonToken nextToken() throws IOException _updateNameLocation(); String n = _parseName(i); _parsingContext.setCurrentName(n); - _currToken = JsonToken.FIELD_NAME; + _updateToken(JsonToken.FIELD_NAME); i = _skipColon(); _updateLocation(); @@ -825,33 +825,33 @@ private final JsonToken _nextTokenNotInObject(int i) throws IOException { if (i == INT_QUOTE) { _tokenIncomplete = true; - return (_currToken = JsonToken.VALUE_STRING); + return _updateToken(JsonToken.VALUE_STRING); } switch (i) { case '[': createChildArrayContext(_tokenInputRow, _tokenInputCol); - return (_currToken = JsonToken.START_ARRAY); + return _updateToken(JsonToken.START_ARRAY); case '{': createChildObjectContext(_tokenInputRow, _tokenInputCol); - return (_currToken = JsonToken.START_OBJECT); + return _updateToken(JsonToken.START_OBJECT); case 't': _matchTrue(); - return (_currToken = JsonToken.VALUE_TRUE); + return _updateToken(JsonToken.VALUE_TRUE); case 'f': _matchFalse(); - return (_currToken = JsonToken.VALUE_FALSE); + return _updateToken(JsonToken.VALUE_FALSE); case 'n': _matchNull(); - return (_currToken = JsonToken.VALUE_NULL); + return _updateToken(JsonToken.VALUE_NULL); case '-': - return (_currToken = _parseSignedNumber(true)); + return _updateToken(_parseSignedNumber(true)); case '+': if (!isEnabled(JsonReadFeature.ALLOW_LEADING_PLUS_SIGN_FOR_NUMBERS.mappedFeature())) { - return (_currToken = _handleUnexpectedValue(i)); + return _updateToken(_handleUnexpectedValue(i)); } - return (_currToken = _parseSignedNumber(false)); + return _updateToken(_parseSignedNumber(false)); case '.': // [core#611]: - return (_currToken = _parseFloatThatStartsWithPeriod(false, false)); + return _updateToken(_parseFloatThatStartsWithPeriod(false, false)); case '0': case '1': case '2': @@ -862,9 +862,9 @@ private final JsonToken _nextTokenNotInObject(int i) throws IOException case '7': case '8': case '9': - return (_currToken = _parseUnsignedNumber(i)); + return _updateToken(_parseUnsignedNumber(i)); } - return (_currToken = _handleUnexpectedValue(i)); + return _updateToken(_handleUnexpectedValue(i)); } private final JsonToken _nextAfterName() throws IOException @@ -881,7 +881,7 @@ private final JsonToken _nextAfterName() throws IOException } else if (t == JsonToken.START_OBJECT) { createChildObjectContext(_tokenInputRow, _tokenInputCol); } - return (_currToken = t); + return _updateToken(t); } @Override @@ -913,7 +913,7 @@ public boolean nextFieldName(SerializableString str) throws IOException int i = _skipWSOrEnd(); if (i < 0) { // end-of-input close(); - _currToken = null; + _updateTokenToNull(); return false; } _binaryValue = null; @@ -921,12 +921,12 @@ public boolean nextFieldName(SerializableString str) throws IOException // Closing scope? if (i == INT_RBRACKET) { _closeArrayScope(); - _currToken = JsonToken.END_ARRAY; + _updateToken(JsonToken.END_ARRAY); return false; } if (i == INT_RCURLY) { _closeObjectScope(); - _currToken = JsonToken.END_OBJECT; + _updateToken(JsonToken.END_OBJECT); return false; } @@ -999,19 +999,19 @@ public String nextFieldName() throws IOException int i = _skipWSOrEnd(); if (i < 0) { close(); - _currToken = null; + _updateTokenToNull(); return null; } _binaryValue = null; if (i == INT_RBRACKET) { _closeArrayScope(); - _currToken = JsonToken.END_ARRAY; + _updateToken(JsonToken.END_ARRAY); return null; } if (i == INT_RCURLY) { _closeObjectScope(); - _currToken = JsonToken.END_OBJECT; + _updateToken(JsonToken.END_OBJECT); return null; } @@ -1039,7 +1039,7 @@ public String nextFieldName() throws IOException _updateNameLocation(); final String nameStr = _parseName(i); _parsingContext.setCurrentName(nameStr); - _currToken = JsonToken.FIELD_NAME; + _updateToken(JsonToken.FIELD_NAME); i = _skipColon(); _updateLocation(); @@ -1152,7 +1152,7 @@ private final int _skipColonFast(int ptr) throws IOException private final void _isNextTokenNameYes(int i) throws IOException { - _currToken = JsonToken.FIELD_NAME; + _updateToken(JsonToken.FIELD_NAME); _updateLocation(); switch (i) { @@ -1214,7 +1214,7 @@ private final boolean _isNextTokenNameMaybe(int i, SerializableString str) throw String n = _parseName(i); _parsingContext.setCurrentName(n); final boolean match = n.equals(str.getValue()); - _currToken = JsonToken.FIELD_NAME; + _updateToken(JsonToken.FIELD_NAME); i = _skipColon(); _updateLocation(); @@ -1285,7 +1285,7 @@ public String nextTextValue() throws IOException _nameCopied = false; JsonToken t = _nextToken; _nextToken = null; - _currToken = t; + _updateToken(t); if (t == JsonToken.VALUE_STRING) { if (_tokenIncomplete) { _tokenIncomplete = false; @@ -1312,7 +1312,7 @@ public int nextIntValue(int defaultValue) throws IOException _nameCopied = false; JsonToken t = _nextToken; _nextToken = null; - _currToken = t; + _updateToken(t); if (t == JsonToken.VALUE_NUMBER_INT) { return getIntValue(); } @@ -1335,7 +1335,7 @@ public long nextLongValue(long defaultValue) throws IOException _nameCopied = false; JsonToken t = _nextToken; _nextToken = null; - _currToken = t; + _updateToken(t); if (t == JsonToken.VALUE_NUMBER_INT) { return getLongValue(); } @@ -1358,7 +1358,7 @@ public Boolean nextBooleanValue() throws IOException _nameCopied = false; JsonToken t = _nextToken; _nextToken = null; - _currToken = t; + _updateToken(t); if (t == JsonToken.VALUE_TRUE) { return Boolean.TRUE; } @@ -3874,13 +3874,13 @@ private final void _updateNameLocation() /********************************************************** */ - private final JsonToken _closeScope(int i) throws JsonParseException { + private final JsonToken _closeScope(int i) throws JsonParseException, StreamConstraintsException { if (i == INT_RCURLY) { _closeObjectScope(); - return (_currToken = JsonToken.END_OBJECT); + return _updateToken(JsonToken.END_OBJECT); } _closeArrayScope(); - return (_currToken = JsonToken.END_ARRAY); + return _updateToken(JsonToken.END_ARRAY); } private final void _closeArrayScope() throws JsonParseException { diff --git a/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParserBase.java b/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParserBase.java index 55ce3d41fe..46ca776907 100644 --- a/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParserBase.java +++ b/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParserBase.java @@ -588,7 +588,7 @@ protected final JsonToken _startArrayScope() throws IOException createChildArrayContext(-1, -1); _majorState = MAJOR_ARRAY_ELEMENT_FIRST; _majorStateAfterValue = MAJOR_ARRAY_ELEMENT_NEXT; - return (_currToken = JsonToken.START_ARRAY); + return _updateToken(JsonToken.START_ARRAY); } protected final JsonToken _startObjectScope() throws IOException @@ -596,7 +596,7 @@ protected final JsonToken _startObjectScope() throws IOException createChildObjectContext(-1, -1); _majorState = MAJOR_OBJECT_FIELD_FIRST; _majorStateAfterValue = MAJOR_OBJECT_FIELD_NEXT; - return (_currToken = JsonToken.START_OBJECT); + return _updateToken(JsonToken.START_OBJECT); } protected final JsonToken _closeArrayScope() throws IOException @@ -616,7 +616,7 @@ protected final JsonToken _closeArrayScope() throws IOException } _majorState = st; _majorStateAfterValue = st; - return (_currToken = JsonToken.END_ARRAY); + return _updateToken(JsonToken.END_ARRAY); } protected final JsonToken _closeObjectScope() throws IOException @@ -636,7 +636,7 @@ protected final JsonToken _closeObjectScope() throws IOException } _majorState = st; _majorStateAfterValue = st; - return (_currToken = JsonToken.END_OBJECT); + return _updateToken(JsonToken.END_OBJECT); } /* @@ -826,21 +826,20 @@ protected final JsonToken _eofAsNextToken() throws IOException { _handleEOF(); } close(); - return (_currToken = null); + return _updateTokenToNull(); } protected final JsonToken _fieldComplete(String name) throws IOException { _majorState = MAJOR_OBJECT_VALUE; _parsingContext.setCurrentName(name); - return (_currToken = JsonToken.FIELD_NAME); + return _updateToken(JsonToken.FIELD_NAME); } protected final JsonToken _valueComplete(JsonToken t) throws IOException { _majorState = _majorStateAfterValue; - _currToken = t; - return t; + return _updateToken(t); } protected final JsonToken _valueCompleteInt(int value, String asText) throws IOException @@ -850,9 +849,7 @@ protected final JsonToken _valueCompleteInt(int value, String asText) throws IOE _numTypesValid = NR_INT; // to force parsing _numberInt = value; _majorState = _majorStateAfterValue; - JsonToken t = JsonToken.VALUE_NUMBER_INT; - _currToken = t; - return t; + return _updateToken(JsonToken.VALUE_NUMBER_INT); } @SuppressWarnings("deprecation") @@ -868,7 +865,7 @@ protected final JsonToken _valueNonStdNumberComplete(int type) throws IOExceptio _numTypesValid = NR_DOUBLE; _numberDouble = NON_STD_TOKEN_VALUES[type]; _majorState = _majorStateAfterValue; - return (_currToken = JsonToken.VALUE_NUMBER_FLOAT); + return _updateToken(JsonToken.VALUE_NUMBER_FLOAT); } protected final String _nonStdToken(int type) { diff --git a/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingUtf8JsonParserBase.java b/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingUtf8JsonParserBase.java index 0f9594dce4..9951d436c6 100644 --- a/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingUtf8JsonParserBase.java +++ b/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingUtf8JsonParserBase.java @@ -467,7 +467,7 @@ private final JsonToken _finishBOM(int bytesHandled) throws IOException } _pending32 = bytesHandled; _minorState = MINOR_ROOT_BOM; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } /* @@ -532,7 +532,7 @@ private final JsonToken _startFieldNameAfterComma(int ch) throws IOException int ptr = _inputPtr; if (ptr >= _inputEnd) { _minorState = MINOR_FIELD_LEADING_WS; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } ch = getByteFromBuffer(ptr); _inputPtr = ptr+1; @@ -675,7 +675,7 @@ private final JsonToken _startValueExpectComma(int ch) throws IOException int ptr = _inputPtr; if (ptr >= _inputEnd) { _minorState = MINOR_VALUE_WS_AFTER_COMMA; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } ch = getByteFromBuffer(ptr); _inputPtr = ptr+1; @@ -765,7 +765,7 @@ private final JsonToken _startValueExpectColon(int ch) throws IOException int ptr = _inputPtr; if (ptr >= _inputEnd) { _minorState = MINOR_VALUE_LEADING_WS; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } ch = getByteFromBuffer(ptr); _inputPtr = ptr+1; @@ -946,7 +946,7 @@ private final int _skipWS(int ch) throws IOException } } if (_inputPtr >= _inputEnd) { - _currToken = JsonToken.NOT_AVAILABLE; + _updateTokenToNA(); return 0; } ch = getNextUnsignedByteFromBuffer(); @@ -964,7 +964,7 @@ private final JsonToken _startSlashComment(int fromMinorState) throws IOExceptio if (_inputPtr >= _inputEnd) { _pending32 = fromMinorState; _minorState = MINOR_COMMENT_LEADING_SLASH; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } int ch = getNextSignedByteFromBuffer(); if (ch == INT_ASTERISK) { // c-style @@ -987,7 +987,7 @@ private final JsonToken _finishHashComment(int fromMinorState) throws IOExceptio if (_inputPtr >= _inputEnd) { _minorState = MINOR_COMMENT_YAML; _pending32 = fromMinorState; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } int ch = getNextUnsignedByteFromBuffer(); if (ch < 0x020) { @@ -1013,7 +1013,7 @@ private final JsonToken _finishCppComment(int fromMinorState) throws IOException if (_inputPtr >= _inputEnd) { _minorState = MINOR_COMMENT_CPP; _pending32 = fromMinorState; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } int ch = getNextUnsignedByteFromBuffer(); if (ch < 0x020) { @@ -1039,7 +1039,7 @@ private final JsonToken _finishCComment(int fromMinorState, boolean gotStar) thr if (_inputPtr >= _inputEnd) { _minorState = gotStar ? MINOR_COMMENT_CLOSING_ASTERISK : MINOR_COMMENT_C; _pending32 = fromMinorState; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } int ch = getNextUnsignedByteFromBuffer(); if (ch < 0x020) { @@ -1070,7 +1070,7 @@ private final JsonToken _startAfterComment(int fromMinorState) throws IOExceptio // Ok, then, need one more character... if (_inputPtr >= _inputEnd) { _minorState = fromMinorState; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } int ch = getNextUnsignedByteFromBuffer(); switch (fromMinorState) { @@ -1161,7 +1161,7 @@ protected JsonToken _finishKeywordToken(String expToken, int matched, while (true) { if (_inputPtr >= _inputEnd) { _pending32 = matched; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } int ch = getByteFromBuffer(_inputPtr); if (matched == end) { // need to verify trailing separator @@ -1185,7 +1185,7 @@ protected JsonToken _finishKeywordTokenWithEOF(String expToken, int matched, JsonToken result) throws IOException { if (matched == expToken.length()) { - return (_currToken = result); + return _updateToken(result); } _textBuffer.resetWithCopy(expToken, 0, matched); return _finishErrorTokenWithEOF(); @@ -1201,7 +1201,7 @@ protected JsonToken _finishNonStdToken(int type, int matched) throws IOException _nonStdTokenType = type; _pending32 = matched; _minorState = MINOR_VALUE_TOKEN_NON_STD; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } int ch = getByteFromBuffer(_inputPtr); if (matched == end) { // need to verify trailing separator @@ -1250,7 +1250,7 @@ protected JsonToken _finishErrorToken() throws IOException } return _reportErrorToken(_textBuffer.contentsAsString()); } - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } protected JsonToken _finishErrorTokenWithEOF() throws IOException @@ -1290,7 +1290,7 @@ protected JsonToken _startPositiveNumber(int ch) throws IOException if (_inputPtr >= _inputEnd) { _minorState = MINOR_NUMBER_INTEGER_DIGITS; _textBuffer.setCurrentLength(1); - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } int outPtr = 1; @@ -1322,7 +1322,7 @@ protected JsonToken _startPositiveNumber(int ch) throws IOException if (++_inputPtr >= _inputEnd) { _minorState = MINOR_NUMBER_INTEGER_DIGITS; _textBuffer.setCurrentLength(outPtr); - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } ch = getByteFromBuffer(_inputPtr) & 0xFF; } @@ -1336,7 +1336,7 @@ protected JsonToken _startNegativeNumber() throws IOException _numberNegative = true; if (_inputPtr >= _inputEnd) { _minorState = MINOR_NUMBER_MINUS; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } int ch = getNextUnsignedByteFromBuffer(); if (ch <= INT_0) { @@ -1358,7 +1358,7 @@ protected JsonToken _startNegativeNumber() throws IOException _minorState = MINOR_NUMBER_INTEGER_DIGITS; _textBuffer.setCurrentLength(2); _intLength = 1; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } ch = getByteFromBuffer(_inputPtr); int outPtr = 2; @@ -1388,7 +1388,7 @@ protected JsonToken _startNegativeNumber() throws IOException if (++_inputPtr >= _inputEnd) { _minorState = MINOR_NUMBER_INTEGER_DIGITS; _textBuffer.setCurrentLength(outPtr); - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } ch = getByteFromBuffer(_inputPtr) & 0xFF; } @@ -1402,7 +1402,7 @@ protected JsonToken _startPositiveNumber() throws IOException _numberNegative = false; if (_inputPtr >= _inputEnd) { _minorState = MINOR_NUMBER_PLUS; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } int ch = getNextUnsignedByteFromBuffer(); if (ch <= INT_0) { @@ -1430,7 +1430,7 @@ protected JsonToken _startPositiveNumber() throws IOException _minorState = MINOR_NUMBER_INTEGER_DIGITS; _textBuffer.setCurrentLength(2); _intLength = 1; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } ch = getByteFromBuffer(_inputPtr); int outPtr = 2; @@ -1460,7 +1460,7 @@ protected JsonToken _startPositiveNumber() throws IOException if (++_inputPtr >= _inputEnd) { _minorState = MINOR_NUMBER_INTEGER_DIGITS; _textBuffer.setCurrentLength(outPtr); - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } ch = getByteFromBuffer(_inputPtr) & 0xFF; } @@ -1474,7 +1474,7 @@ protected JsonToken _startNumberLeadingZero() throws IOException int ptr = _inputPtr; if (ptr >= _inputEnd) { _minorState = MINOR_NUMBER_ZERO; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } // While we could call `_finishNumberLeadingZeroes()`, let's try checking @@ -1579,7 +1579,7 @@ protected JsonToken _finishNumberLeadingZeroes() throws IOException while (true) { if (_inputPtr >= _inputEnd) { _minorState = MINOR_NUMBER_ZERO; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } int ch = getNextUnsignedByteFromBuffer(); if (ch < INT_0) { @@ -1637,7 +1637,7 @@ protected JsonToken _finishNumberLeadingPosNegZeroes(final boolean negative) thr while (true) { if (_inputPtr >= _inputEnd) { _minorState = negative ? MINOR_NUMBER_MINUSZERO : MINOR_NUMBER_ZERO; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } int ch = getNextUnsignedByteFromBuffer(); if (ch < INT_0) { @@ -1691,7 +1691,7 @@ protected JsonToken _finishNumberIntegralPart(char[] outBuf, int outPtr) throws if (_inputPtr >= _inputEnd) { _minorState = MINOR_NUMBER_INTEGER_DIGITS; _textBuffer.setCurrentLength(outPtr); - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } int ch = getByteFromBuffer(_inputPtr) & 0xFF; if (ch < INT_0) { @@ -1736,7 +1736,7 @@ protected JsonToken _startFloat(char[] outBuf, int outPtr, int ch) throws IOExce _textBuffer.setCurrentLength(outPtr); _minorState = MINOR_NUMBER_FRACTION_DIGITS; _fractLength = fractLen; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } ch = getNextSignedByteFromBuffer(); // ok to have sign extension for now if (ch < INT_0 || ch > INT_9) { @@ -1767,7 +1767,7 @@ protected JsonToken _startFloat(char[] outBuf, int outPtr, int ch) throws IOExce _textBuffer.setCurrentLength(outPtr); _minorState = MINOR_NUMBER_EXPONENT_MARKER; _expLength = 0; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } ch = getNextSignedByteFromBuffer(); // ok to have sign extension for now if (ch == INT_MINUS || ch == INT_PLUS) { @@ -1779,7 +1779,7 @@ protected JsonToken _startFloat(char[] outBuf, int outPtr, int ch) throws IOExce _textBuffer.setCurrentLength(outPtr); _minorState = MINOR_NUMBER_EXPONENT_DIGITS; _expLength = 0; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } ch = getNextSignedByteFromBuffer(); } @@ -1793,7 +1793,7 @@ protected JsonToken _startFloat(char[] outBuf, int outPtr, int ch) throws IOExce _textBuffer.setCurrentLength(outPtr); _minorState = MINOR_NUMBER_EXPONENT_DIGITS; _expLength = expLen; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } ch = getNextSignedByteFromBuffer(); } @@ -2086,7 +2086,7 @@ private final JsonToken _parseEscapedName(int qlen, int currQuad, int currQuadBy _pending32 = currQuad; _pendingBytes = currQuadBytes; _minorState = MINOR_FIELD_NAME; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } int ch = getNextUnsignedByteFromBuffer(); if (codes[ch] == 0) { @@ -2121,7 +2121,7 @@ private final JsonToken _parseEscapedName(int qlen, int currQuad, int currQuadBy _quadLength = qlen; _pending32 = currQuad; _pendingBytes = currQuadBytes; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } } @@ -2246,7 +2246,7 @@ private JsonToken _finishUnquotedName(int qlen, int currQuad, int currQuadBytes) _pending32 = currQuad; _pendingBytes = currQuadBytes; _minorState = MINOR_FIELD_UNQUOTED_NAME; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } int ch = getByteFromBuffer(_inputPtr) & 0xFF; if (codes[ch] != 0) { @@ -2292,7 +2292,7 @@ private JsonToken _finishAposName(int qlen, int currQuad, int currQuadBytes) _pending32 = currQuad; _pendingBytes = currQuadBytes; _minorState = MINOR_FIELD_APOS_NAME; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } int ch = getNextUnsignedByteFromBuffer(); if (ch == INT_APOS) { @@ -2312,7 +2312,7 @@ private JsonToken _finishAposName(int qlen, int currQuad, int currQuadBytes) _quadLength = qlen; _pending32 = currQuad; _pendingBytes = currQuadBytes; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } } if (ch > 127) { @@ -2548,7 +2548,7 @@ private final JsonToken _finishRegularString() throws IOException _inputPtr = ptr; _minorState = MINOR_VALUE_STRING; _textBuffer.setCurrentLength(outPtr); - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } if (outPtr >= outBuf.length) { outBuf = _textBuffer.finishCurrentSegment(); @@ -2575,7 +2575,7 @@ private final JsonToken _finishRegularString() throws IOException _textBuffer.setCurrentLength(outPtr); if (!_decodeSplitMultiByte(c, codes[c], ptr < _inputEnd)) { _minorStateAfterSplit = MINOR_VALUE_STRING; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } outBuf = _textBuffer.getBufferWithoutReset(); outPtr = _textBuffer.getCurrentSegmentSize(); @@ -2671,7 +2671,7 @@ private final JsonToken _finishAposString() throws IOException _inputPtr = ptr; _minorState = MINOR_VALUE_APOS_STRING; _textBuffer.setCurrentLength(outPtr); - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } if (outPtr >= outBuf.length) { outBuf = _textBuffer.finishCurrentSegment(); @@ -2698,7 +2698,7 @@ private final JsonToken _finishAposString() throws IOException _textBuffer.setCurrentLength(outPtr); if (!_decodeSplitMultiByte(c, codes[c], ptr < _inputEnd)) { _minorStateAfterSplit = MINOR_VALUE_APOS_STRING; - return (_currToken = JsonToken.NOT_AVAILABLE); + return _updateTokenToNA(); } outBuf = _textBuffer.getBufferWithoutReset(); outPtr = _textBuffer.getCurrentSegmentSize(); diff --git a/src/test/java/com/fasterxml/jackson/core/read/TokenCountTest.java b/src/test/java/com/fasterxml/jackson/core/read/TokenCountTest.java new file mode 100644 index 0000000000..a716e561f7 --- /dev/null +++ b/src/test/java/com/fasterxml/jackson/core/read/TokenCountTest.java @@ -0,0 +1,95 @@ +package com.fasterxml.jackson.core.read; + + +import com.fasterxml.jackson.core.JUnit5TestBase; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.core.json.async.NonBlockingByteBufferJsonParser; +import com.fasterxml.jackson.core.json.async.NonBlockingJsonParser; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Set of basic unit tests for verifying that the token count + * functionality works as expected. + */ +public class TokenCountTest extends JUnit5TestBase { + private final static JsonFactory JSON_FACTORY = JsonFactory.builder() + .streamReadConstraints(StreamReadConstraints.builder().maxTokenCount(Long.MAX_VALUE).build()) + .build(); + private final static String ARRAY_DOC = a2q("{ 'nums': [1,2,3,4,5,6,7,8,9,10] }"); + + @Test + void arrayDoc() throws Exception + { + for (int mode : ALL_MODES) { + _testArrayDoc(mode); + } + } + + @Test + void arrayDocNonBlockingArray() throws Exception + { + final byte[] input = ARRAY_DOC.getBytes("UTF-8"); + try (NonBlockingJsonParser p = (NonBlockingJsonParser) JSON_FACTORY.createNonBlockingByteArrayParser()) { + p.feedInput(input, 0, input.length); + p.endOfInput(); + _testArrayDoc(p); + } + } + + @Test + void arrayDocNonBlockingBuffer() throws Exception + { + final byte[] input = ARRAY_DOC.getBytes("UTF-8"); + try (NonBlockingByteBufferJsonParser p = (NonBlockingByteBufferJsonParser) JSON_FACTORY.createNonBlockingByteBufferParser()) { + p.feedInput(ByteBuffer.wrap(input, 0, input.length)); + p.endOfInput(); + _testArrayDoc(p); + } + } + + @Test + void sampleDoc() throws Exception + { + for (int mode : ALL_MODES) { + _testSampleDoc(mode); + } + } + + private void _testArrayDoc(int mode) throws Exception + { + try (JsonParser p = createParser(JSON_FACTORY, mode, ARRAY_DOC)) { + _testArrayDoc(p); + } + } + + private void _testArrayDoc(JsonParser p) throws Exception + { + assertEquals(0, p.getTokenCount()); + while (p.nextToken() != null) { + // read all tokens + } + assertEquals(15, p.getTokenCount()); + } + + private void _testSampleDoc(int mode) throws Exception + { + try (JsonParser p = createParser(JSON_FACTORY, mode, SAMPLE_DOC_JSON_SPEC)) { + _testSampleDoc(p); + } + } + + private void _testSampleDoc(JsonParser p) throws Exception + { + assertEquals(0, p.getTokenCount()); + while (p.nextToken() != null) { + // read all tokens + } + assertEquals(27, p.getTokenCount()); + } +} From 52bb0707cbf241b540f7a2177bfc983cb346a43d Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Thu, 13 Jun 2024 09:18:26 +0100 Subject: [PATCH 02/11] Update ParserMinimalBase.java --- .../com/fasterxml/jackson/core/base/ParserMinimalBase.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java b/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java index 4da5b9cce5..cc53d045c0 100644 --- a/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java +++ b/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java @@ -833,8 +833,8 @@ protected final void _wrapError(String msg, Throwable t) throws JsonParseExcepti protected final JsonToken _updateToken(final JsonToken token) throws StreamConstraintsException { _currToken = token; - if (streamReadConstraints().hasMaxTokenCount() && token != JsonToken.NOT_AVAILABLE) { - streamReadConstraints().validateTokenCount(++_tokenCount); + if (_streamReadConstraints.hasMaxTokenCount() && token != JsonToken.NOT_AVAILABLE) { + _streamReadConstraints.validateTokenCount(++_tokenCount); } return token; } From 9b1763b1b328307a0324e328966a50de894e9837 Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Thu, 13 Jun 2024 09:37:39 +0100 Subject: [PATCH 03/11] Update ParserMinimalBase.java --- .../jackson/core/base/ParserMinimalBase.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java b/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java index cc53d045c0..4e3543a006 100644 --- a/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java +++ b/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java @@ -166,6 +166,13 @@ public abstract class ParserMinimalBase extends JsonParser */ protected long _tokenCount; + /** + * Whether or not to track the token count due a {@link StreamReadConstraints} maxTokenCount > 0. + * + * @since 2.18 + */ + protected final boolean _hasMaxTokenCount; + /** * Last cleared token, if any: that is, value that was in * effect when {@link #clearCurrentToken} was called. @@ -182,6 +189,7 @@ public abstract class ParserMinimalBase extends JsonParser protected ParserMinimalBase() { super(); _streamReadConstraints = StreamReadConstraints.defaults(); + _hasMaxTokenCount = _streamReadConstraints.hasMaxTokenCount(); } @Deprecated // since 2.18 @@ -193,12 +201,14 @@ protected ParserMinimalBase(int features) { protected ParserMinimalBase(StreamReadConstraints src) { super(); _streamReadConstraints = (src == null) ? StreamReadConstraints.defaults() : src; + _hasMaxTokenCount = _streamReadConstraints.hasMaxTokenCount(); } // @since 2.18 protected ParserMinimalBase(int features, StreamReadConstraints src) { super(features); _streamReadConstraints = (src == null) ? StreamReadConstraints.defaults() : src; + _hasMaxTokenCount = _streamReadConstraints.hasMaxTokenCount(); } // NOTE: had base impl in 2.3 and before; but shouldn't @@ -833,7 +843,7 @@ protected final void _wrapError(String msg, Throwable t) throws JsonParseExcepti protected final JsonToken _updateToken(final JsonToken token) throws StreamConstraintsException { _currToken = token; - if (_streamReadConstraints.hasMaxTokenCount() && token != JsonToken.NOT_AVAILABLE) { + if (_hasMaxTokenCount && token != JsonToken.NOT_AVAILABLE) { _streamReadConstraints.validateTokenCount(++_tokenCount); } return token; From aa9c9344eb1c157dc56fb3101b1de56c220c4389 Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Thu, 13 Jun 2024 10:19:07 +0100 Subject: [PATCH 04/11] small optimisation --- .../fasterxml/jackson/core/base/ParserMinimalBase.java | 8 ++++++++ .../core/json/async/NonBlockingJsonParserBase.java | 2 +- .../core/json/async/NonBlockingUtf8JsonParserBase.java | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java b/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java index 4e3543a006..2a7a9b8c00 100644 --- a/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java +++ b/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java @@ -842,6 +842,14 @@ protected final void _wrapError(String msg, Throwable t) throws JsonParseExcepti } protected final JsonToken _updateToken(final JsonToken token) throws StreamConstraintsException { + _currToken = token; + if (_hasMaxTokenCount) { + _streamReadConstraints.validateTokenCount(++_tokenCount); + } + return token; + } + + protected final JsonToken _updateTokenWithPossibleNA(final JsonToken token) throws StreamConstraintsException { _currToken = token; if (_hasMaxTokenCount && token != JsonToken.NOT_AVAILABLE) { _streamReadConstraints.validateTokenCount(++_tokenCount); diff --git a/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParserBase.java b/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParserBase.java index 46ca776907..71fb235bfa 100644 --- a/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParserBase.java +++ b/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParserBase.java @@ -839,7 +839,7 @@ protected final JsonToken _fieldComplete(String name) throws IOException protected final JsonToken _valueComplete(JsonToken t) throws IOException { _majorState = _majorStateAfterValue; - return _updateToken(t); + return _updateTokenWithPossibleNA(t); } protected final JsonToken _valueCompleteInt(int value, String asText) throws IOException diff --git a/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingUtf8JsonParserBase.java b/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingUtf8JsonParserBase.java index 9951d436c6..aafdabee24 100644 --- a/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingUtf8JsonParserBase.java +++ b/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingUtf8JsonParserBase.java @@ -1185,7 +1185,7 @@ protected JsonToken _finishKeywordTokenWithEOF(String expToken, int matched, JsonToken result) throws IOException { if (matched == expToken.length()) { - return _updateToken(result); + return _updateTokenWithPossibleNA(result); } _textBuffer.resetWithCopy(expToken, 0, matched); return _finishErrorTokenWithEOF(); From ec30b27854ee5c7f6c95cdb3043775fa20a79ad4 Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Thu, 13 Jun 2024 10:36:09 +0100 Subject: [PATCH 05/11] remove unnecessary check --- .../fasterxml/jackson/core/base/ParserMinimalBase.java | 8 -------- .../core/json/async/NonBlockingJsonParserBase.java | 2 +- .../core/json/async/NonBlockingUtf8JsonParserBase.java | 2 +- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java b/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java index 2a7a9b8c00..7efbdbc324 100644 --- a/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java +++ b/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java @@ -849,14 +849,6 @@ protected final JsonToken _updateToken(final JsonToken token) throws StreamConst return token; } - protected final JsonToken _updateTokenWithPossibleNA(final JsonToken token) throws StreamConstraintsException { - _currToken = token; - if (_hasMaxTokenCount && token != JsonToken.NOT_AVAILABLE) { - _streamReadConstraints.validateTokenCount(++_tokenCount); - } - return token; - } - protected final JsonToken _updateTokenToNull() { return (_currToken = null); } diff --git a/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParserBase.java b/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParserBase.java index 71fb235bfa..46ca776907 100644 --- a/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParserBase.java +++ b/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParserBase.java @@ -839,7 +839,7 @@ protected final JsonToken _fieldComplete(String name) throws IOException protected final JsonToken _valueComplete(JsonToken t) throws IOException { _majorState = _majorStateAfterValue; - return _updateTokenWithPossibleNA(t); + return _updateToken(t); } protected final JsonToken _valueCompleteInt(int value, String asText) throws IOException diff --git a/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingUtf8JsonParserBase.java b/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingUtf8JsonParserBase.java index aafdabee24..9951d436c6 100644 --- a/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingUtf8JsonParserBase.java +++ b/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingUtf8JsonParserBase.java @@ -1185,7 +1185,7 @@ protected JsonToken _finishKeywordTokenWithEOF(String expToken, int matched, JsonToken result) throws IOException { if (matched == expToken.length()) { - return _updateTokenWithPossibleNA(result); + return _updateToken(result); } _textBuffer.resetWithCopy(expToken, 0, matched); return _finishErrorTokenWithEOF(); From a982e1aa9fa63bf944b51f1bf94a606ed3b5ec53 Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Thu, 13 Jun 2024 16:29:01 +0100 Subject: [PATCH 06/11] add test --- .../core/constraints/LargeDocReadTest.java | 17 +++++++++++++++++ .../jackson/core/read/TokenCountTest.java | 14 ++++++++------ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/test/java/com/fasterxml/jackson/core/constraints/LargeDocReadTest.java b/src/test/java/com/fasterxml/jackson/core/constraints/LargeDocReadTest.java index b184ee87b2..14c4a84a9d 100644 --- a/src/test/java/com/fasterxml/jackson/core/constraints/LargeDocReadTest.java +++ b/src/test/java/com/fasterxml/jackson/core/constraints/LargeDocReadTest.java @@ -9,6 +9,7 @@ import com.fasterxml.jackson.core.exc.StreamConstraintsException; import com.fasterxml.jackson.core.testsupport.AsyncReaderWrapper; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; // [core#1047]: Add max-name-length constraints @@ -20,6 +21,10 @@ class LargeDocReadTest extends AsyncTestBase .streamReadConstraints(StreamReadConstraints.builder().maxDocumentLength(10_000L).build()) .build(); + private final JsonFactory JSON_F_MAX_TOKENS_1K = JsonFactory.builder() + .streamReadConstraints(StreamReadConstraints.builder().maxTokenCount(1_000L).build()) + .build(); + // Test name that is below default max name @Test void largeNameBytes() throws Exception { @@ -83,6 +88,18 @@ void largeNameWithSmallLimitAsync() throws Exception } } + @Test + void tokenLimitBytes() throws Exception { + final String doc = generateJSON(StreamReadConstraints.defaults().getMaxNameLength() - 100); + try (JsonParser p = createParserUsingStream(JSON_F_MAX_TOKENS_1K, doc, "UTF-8")) { + consumeTokens(p); + fail("expected StreamConstraintsException"); + } catch (StreamConstraintsException e) { + assertEquals("Token count (1001) exceeds the maximum allowed (1000, from `StreamReadConstraints.getMaxTokenCount()`)", + e.getMessage()); + } + } + private void consumeTokens(JsonParser p) throws IOException { while (p.nextToken() != null) { ; diff --git a/src/test/java/com/fasterxml/jackson/core/read/TokenCountTest.java b/src/test/java/com/fasterxml/jackson/core/read/TokenCountTest.java index a716e561f7..471cb85c84 100644 --- a/src/test/java/com/fasterxml/jackson/core/read/TokenCountTest.java +++ b/src/test/java/com/fasterxml/jackson/core/read/TokenCountTest.java @@ -1,6 +1,5 @@ package com.fasterxml.jackson.core.read; - import com.fasterxml.jackson.core.JUnit5TestBase; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; @@ -9,6 +8,7 @@ import com.fasterxml.jackson.core.json.async.NonBlockingJsonParser; import org.junit.jupiter.api.Test; +import java.io.IOException; import java.nio.ByteBuffer; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -71,9 +71,7 @@ private void _testArrayDoc(int mode) throws Exception private void _testArrayDoc(JsonParser p) throws Exception { assertEquals(0, p.getTokenCount()); - while (p.nextToken() != null) { - // read all tokens - } + consumeTokens(p); assertEquals(15, p.getTokenCount()); } @@ -87,9 +85,13 @@ private void _testSampleDoc(int mode) throws Exception private void _testSampleDoc(JsonParser p) throws Exception { assertEquals(0, p.getTokenCount()); + consumeTokens(p); + assertEquals(27, p.getTokenCount()); + } + + private void consumeTokens(JsonParser p) throws IOException { while (p.nextToken() != null) { - // read all tokens + ; } - assertEquals(27, p.getTokenCount()); } } From f3abbf3a478d4fe79d3928d898e6f77e36b9ab2f Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Sat, 15 Jun 2024 21:27:39 +0100 Subject: [PATCH 07/11] Update TokenCountTest.java --- .../jackson/core/read/TokenCountTest.java | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/src/test/java/com/fasterxml/jackson/core/read/TokenCountTest.java b/src/test/java/com/fasterxml/jackson/core/read/TokenCountTest.java index 471cb85c84..8ec81dedf7 100644 --- a/src/test/java/com/fasterxml/jackson/core/read/TokenCountTest.java +++ b/src/test/java/com/fasterxml/jackson/core/read/TokenCountTest.java @@ -22,6 +22,7 @@ public class TokenCountTest extends JUnit5TestBase { .streamReadConstraints(StreamReadConstraints.builder().maxTokenCount(Long.MAX_VALUE).build()) .build(); private final static String ARRAY_DOC = a2q("{ 'nums': [1,2,3,4,5,6,7,8,9,10] }"); + private final static String SHORT_ARRAY_DOC = a2q("{ 'nums': [1,2,3] }"); @Test void arrayDoc() throws Exception @@ -53,6 +54,37 @@ void arrayDocNonBlockingBuffer() throws Exception } } + @Test + void shortArrayDoc() throws Exception + { + for (int mode : ALL_MODES) { + _testShortArrayDoc(mode); + } + } + + @Test + void shortArrayDocNonBlockingArray() throws Exception + { + final byte[] input = SHORT_ARRAY_DOC.getBytes("UTF-8"); + try (NonBlockingJsonParser p = (NonBlockingJsonParser) JSON_FACTORY.createNonBlockingByteArrayParser()) { + p.feedInput(input, 0, input.length); + p.endOfInput(); + _testShortArrayDoc(p); + } + } + + @Test + void shortArrayDocNonBlockingBuffer() throws Exception + { + final byte[] input = SHORT_ARRAY_DOC.getBytes("UTF-8"); + try (NonBlockingByteBufferJsonParser p = (NonBlockingByteBufferJsonParser) + JSON_FACTORY.createNonBlockingByteBufferParser()) { + p.feedInput(ByteBuffer.wrap(input, 0, input.length)); + p.endOfInput(); + _testShortArrayDoc(p); + } + } + @Test void sampleDoc() throws Exception { @@ -61,6 +93,29 @@ void sampleDoc() throws Exception } } + @Test + void sampleDocNonBlockingArray() throws Exception + { + final byte[] input = SAMPLE_DOC_JSON_SPEC.getBytes("UTF-8"); + try (NonBlockingJsonParser p = (NonBlockingJsonParser) JSON_FACTORY.createNonBlockingByteArrayParser()) { + p.feedInput(input, 0, input.length); + p.endOfInput(); + _testSampleDoc(p); + } + } + + @Test + void sampleDocNonBlockingBuffer() throws Exception + { + final byte[] input = SAMPLE_DOC_JSON_SPEC.getBytes("UTF-8"); + try (NonBlockingByteBufferJsonParser p = (NonBlockingByteBufferJsonParser) + JSON_FACTORY.createNonBlockingByteBufferParser()) { + p.feedInput(ByteBuffer.wrap(input, 0, input.length)); + p.endOfInput(); + _testSampleDoc(p); + } + } + private void _testArrayDoc(int mode) throws Exception { try (JsonParser p = createParser(JSON_FACTORY, mode, ARRAY_DOC)) { @@ -75,6 +130,20 @@ private void _testArrayDoc(JsonParser p) throws Exception assertEquals(15, p.getTokenCount()); } + private void _testShortArrayDoc(int mode) throws Exception + { + try (JsonParser p = createParser(JSON_FACTORY, mode, SHORT_ARRAY_DOC)) { + _testShortArrayDoc(p); + } + } + + private void _testShortArrayDoc(JsonParser p) throws Exception + { + assertEquals(0, p.getTokenCount()); + consumeTokens(p); + assertEquals(8, p.getTokenCount()); + } + private void _testSampleDoc(int mode) throws Exception { try (JsonParser p = createParser(JSON_FACTORY, mode, SAMPLE_DOC_JSON_SPEC)) { From 95a6819defbd6c931c2ed4f92f0b4ffb5825c3eb Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Sat, 15 Jun 2024 18:31:14 -0700 Subject: [PATCH 08/11] Minor tweaks --- .../fasterxml/jackson/core/JsonParser.java | 24 +++++++++---------- .../jackson/core/base/ParserMinimalBase.java | 10 ++++---- .../jackson/core/read/TokenCountTest.java | 7 +++--- 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/fasterxml/jackson/core/JsonParser.java b/src/main/java/com/fasterxml/jackson/core/JsonParser.java index db8d93bf2e..6b5cda8816 100644 --- a/src/main/java/com/fasterxml/jackson/core/JsonParser.java +++ b/src/main/java/com/fasterxml/jackson/core/JsonParser.java @@ -864,6 +864,18 @@ public void setCurrentValue(Object v) { assignCurrentValue(v); } + /** + * Get an approximate count of the number of tokens that have been read. + * This count is likely to be only updated if {@link StreamReadConstraints.Builder#maxTokenCount(long)} + * has been used to set a limit on the number of tokens that can be read. + * + * @return the number of tokens that have been read (-1 if the count is not available) + * @since 2.18 + */ + public long getTokenCount() { + return -1L; + } + /* /********************************************************** /* Buffer handling @@ -2546,18 +2558,6 @@ public T readValueAsTree() throws IOException { return (T) _codec().readTree(this); } - /** - * Get an approximate count of the number of tokens that have been read. - * This count is likely to be only updated if {@link StreamReadConstraints.Builder.maxTokenCount(long)} - * has been used to set a limit on the number of tokens that can be read. - * - * @return the number of tokens that have been read (-1 if the count is not available) - * @since 2.18 - */ - public long getTokenCount() { - return -1L; - } - protected ObjectCodec _codec() { ObjectCodec c = getCodec(); if (c == null) { diff --git a/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java b/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java index bab450d68c..134b172c88 100644 --- a/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java +++ b/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java @@ -344,6 +344,11 @@ public JsonParser skipChildren() throws IOException // public abstract JsonLocation getCurrentLocation(); + @Override // since 2.18 + public long getTokenCount() { + return _tokenCount; + } + /* /********************************************************** /* Public API, token state overrides @@ -556,11 +561,6 @@ public String getValueAsString(String defaultValue) throws IOException { return getText(); } - @Override - public long getTokenCount() { - return _tokenCount; - } - /* /********************************************************** /* Base64 decoding diff --git a/src/test/java/com/fasterxml/jackson/core/read/TokenCountTest.java b/src/test/java/com/fasterxml/jackson/core/read/TokenCountTest.java index 8ec81dedf7..16c9b1a47f 100644 --- a/src/test/java/com/fasterxml/jackson/core/read/TokenCountTest.java +++ b/src/test/java/com/fasterxml/jackson/core/read/TokenCountTest.java @@ -1,5 +1,7 @@ package com.fasterxml.jackson.core.read; +import java.nio.ByteBuffer; + import com.fasterxml.jackson.core.JUnit5TestBase; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; @@ -8,9 +10,6 @@ import com.fasterxml.jackson.core.json.async.NonBlockingJsonParser; import org.junit.jupiter.api.Test; -import java.io.IOException; -import java.nio.ByteBuffer; - import static org.junit.jupiter.api.Assertions.assertEquals; /** @@ -158,7 +157,7 @@ private void _testSampleDoc(JsonParser p) throws Exception assertEquals(27, p.getTokenCount()); } - private void consumeTokens(JsonParser p) throws IOException { + private void consumeTokens(JsonParser p) throws Exception { while (p.nextToken() != null) { ; } From 57e03e9d59e42dbe4cc86a97d91f432807fa0953 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Mon, 17 Jun 2024 20:50:22 -0700 Subject: [PATCH 09/11] Add release note --- release-notes/VERSION-2.x | 3 +++ 1 file changed, 3 insertions(+) diff --git a/release-notes/VERSION-2.x b/release-notes/VERSION-2.x index ab06c101d3..728102e847 100644 --- a/release-notes/VERSION-2.x +++ b/release-notes/VERSION-2.x @@ -35,6 +35,9 @@ a pure JSON library. #1277: Add back Java 22 optimisation in FastDoubleParser #1305: Make helper methods of `WriterBasedJsonGenerator` non-final to allow overriding (contributed by @zhangOranges) +#1310: Add new `StreamReadConstraints` (`maxTokenCount`) to limit maximum number + of Tokens allowed per document + (implemented by @pjfanning) 2.17.2 (not yet released) From b25d3db3262cc41fc8ceb09dd915dcebe3138ffd Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Tue, 18 Jun 2024 16:32:54 -0700 Subject: [PATCH 10/11] Last bits of tweaking; add delegation in JsonParserDelegate, renaming, moving methods --- .../fasterxml/jackson/core/JsonParser.java | 24 +++++++++---------- .../jackson/core/base/ParserMinimalBase.java | 17 ++++++------- .../jackson/core/util/JsonParserDelegate.java | 3 +++ .../jackson/core/read/TokenCountTest.java | 12 +++++----- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/main/java/com/fasterxml/jackson/core/JsonParser.java b/src/main/java/com/fasterxml/jackson/core/JsonParser.java index 6b5cda8816..b0a38a4909 100644 --- a/src/main/java/com/fasterxml/jackson/core/JsonParser.java +++ b/src/main/java/com/fasterxml/jackson/core/JsonParser.java @@ -781,6 +781,18 @@ public JsonLocation currentTokenLocation() { return getTokenLocation(); } + /** + * Get an approximate count of the number of tokens that have been read. + * This count is likely to be only updated if {@link StreamReadConstraints.Builder#maxTokenCount(long)} + * has been used to set a limit on the number of tokens that can be read. + * + * @return the number of tokens that have been read (-1 if the count is not available) + * @since 2.18 + */ + public long currentTokenCount() { + return -1L; + } + /** * Deprecated alias for {@link #currentLocation()} (removed from Jackson 3.0). * @@ -864,18 +876,6 @@ public void setCurrentValue(Object v) { assignCurrentValue(v); } - /** - * Get an approximate count of the number of tokens that have been read. - * This count is likely to be only updated if {@link StreamReadConstraints.Builder#maxTokenCount(long)} - * has been used to set a limit on the number of tokens that can be read. - * - * @return the number of tokens that have been read (-1 if the count is not available) - * @since 2.18 - */ - public long getTokenCount() { - return -1L; - } - /* /********************************************************** /* Buffer handling diff --git a/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java b/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java index 134b172c88..9e73d7b12d 100644 --- a/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java +++ b/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java @@ -160,7 +160,7 @@ public abstract class ParserMinimalBase extends JsonParser protected JsonToken _currToken; /** - * Current count of tokens + * Current count of tokens, if tracked (see {@link #_trackMaxTokenCount}) * * @since 2.18 */ @@ -171,7 +171,7 @@ public abstract class ParserMinimalBase extends JsonParser * * @since 2.18 */ - protected final boolean _hasMaxTokenCount; + protected final boolean _trackMaxTokenCount; /** * Last cleared token, if any: that is, value that was in @@ -189,7 +189,7 @@ public abstract class ParserMinimalBase extends JsonParser protected ParserMinimalBase() { super(); _streamReadConstraints = StreamReadConstraints.defaults(); - _hasMaxTokenCount = _streamReadConstraints.hasMaxTokenCount(); + _trackMaxTokenCount = _streamReadConstraints.hasMaxTokenCount(); } @Deprecated // since 2.18 @@ -201,14 +201,14 @@ protected ParserMinimalBase(int features) { protected ParserMinimalBase(StreamReadConstraints src) { super(); _streamReadConstraints = (src == null) ? StreamReadConstraints.defaults() : src; - _hasMaxTokenCount = _streamReadConstraints.hasMaxTokenCount(); + _trackMaxTokenCount = _streamReadConstraints.hasMaxTokenCount(); } // @since 2.18 protected ParserMinimalBase(int features, StreamReadConstraints src) { super(features); _streamReadConstraints = (src == null) ? StreamReadConstraints.defaults() : src; - _hasMaxTokenCount = _streamReadConstraints.hasMaxTokenCount(); + _trackMaxTokenCount = _streamReadConstraints.hasMaxTokenCount(); } // NOTE: had base impl in 2.3 and before; but shouldn't @@ -328,9 +328,6 @@ public JsonParser skipChildren() throws IOException */ protected abstract void _handleEOF() throws JsonParseException; - //public JsonToken getCurrentToken() - //public boolean hasCurrentToken() - @Deprecated // since 2.17 -- still need to implement @Override public abstract String getCurrentName() throws IOException; @@ -345,7 +342,7 @@ public JsonParser skipChildren() throws IOException // public abstract JsonLocation getCurrentLocation(); @Override // since 2.18 - public long getTokenCount() { + public long currentTokenCount() { return _tokenCount; } @@ -849,7 +846,7 @@ protected final void _wrapError(String msg, Throwable t) throws JsonParseExcepti protected final JsonToken _updateToken(final JsonToken token) throws StreamConstraintsException { _currToken = token; - if (_hasMaxTokenCount) { + if (_trackMaxTokenCount) { _streamReadConstraints.validateTokenCount(++_tokenCount); } return token; diff --git a/src/main/java/com/fasterxml/jackson/core/util/JsonParserDelegate.java b/src/main/java/com/fasterxml/jackson/core/util/JsonParserDelegate.java index f69e85002a..950ccfa3d9 100644 --- a/src/main/java/com/fasterxml/jackson/core/util/JsonParserDelegate.java +++ b/src/main/java/com/fasterxml/jackson/core/util/JsonParserDelegate.java @@ -155,6 +155,9 @@ public boolean requiresCustomCodec() { @Override public JsonLocation currentLocation() { return delegate.currentLocation(); } @Override public JsonLocation currentTokenLocation() { return delegate.currentTokenLocation(); } + @Override // since 2.18 + public long currentTokenCount() { return delegate.currentTokenCount(); } + @Override @Deprecated public JsonToken getCurrentToken() { return delegate.getCurrentToken(); } diff --git a/src/test/java/com/fasterxml/jackson/core/read/TokenCountTest.java b/src/test/java/com/fasterxml/jackson/core/read/TokenCountTest.java index 16c9b1a47f..538adbbb2e 100644 --- a/src/test/java/com/fasterxml/jackson/core/read/TokenCountTest.java +++ b/src/test/java/com/fasterxml/jackson/core/read/TokenCountTest.java @@ -124,9 +124,9 @@ private void _testArrayDoc(int mode) throws Exception private void _testArrayDoc(JsonParser p) throws Exception { - assertEquals(0, p.getTokenCount()); + assertEquals(0, p.currentTokenCount()); consumeTokens(p); - assertEquals(15, p.getTokenCount()); + assertEquals(15, p.currentTokenCount()); } private void _testShortArrayDoc(int mode) throws Exception @@ -138,9 +138,9 @@ private void _testShortArrayDoc(int mode) throws Exception private void _testShortArrayDoc(JsonParser p) throws Exception { - assertEquals(0, p.getTokenCount()); + assertEquals(0, p.currentTokenCount()); consumeTokens(p); - assertEquals(8, p.getTokenCount()); + assertEquals(8, p.currentTokenCount()); } private void _testSampleDoc(int mode) throws Exception @@ -152,9 +152,9 @@ private void _testSampleDoc(int mode) throws Exception private void _testSampleDoc(JsonParser p) throws Exception { - assertEquals(0, p.getTokenCount()); + assertEquals(0, p.currentTokenCount()); consumeTokens(p); - assertEquals(27, p.getTokenCount()); + assertEquals(27, p.currentTokenCount()); } private void consumeTokens(JsonParser p) throws Exception { From a567787ce7afb5066439e8fb24c85ae7a547446b Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Tue, 18 Jun 2024 16:34:40 -0700 Subject: [PATCH 11/11] Move token-count test to under constraints as it is conceptually related --- .../jackson/core/{read => constraints}/TokenCountTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/test/java/com/fasterxml/jackson/core/{read => constraints}/TokenCountTest.java (99%) diff --git a/src/test/java/com/fasterxml/jackson/core/read/TokenCountTest.java b/src/test/java/com/fasterxml/jackson/core/constraints/TokenCountTest.java similarity index 99% rename from src/test/java/com/fasterxml/jackson/core/read/TokenCountTest.java rename to src/test/java/com/fasterxml/jackson/core/constraints/TokenCountTest.java index 538adbbb2e..787a33e37b 100644 --- a/src/test/java/com/fasterxml/jackson/core/read/TokenCountTest.java +++ b/src/test/java/com/fasterxml/jackson/core/constraints/TokenCountTest.java @@ -1,4 +1,4 @@ -package com.fasterxml.jackson.core.read; +package com.fasterxml.jackson.core.constraints; import java.nio.ByteBuffer;