Skip to content

Commit

Permalink
Provide mechanism to configure XContent parsing constraints (after up…
Browse files Browse the repository at this point in the history 
…date to Jackson 2.15.0 and above)

Signed-off-by: Andriy Redko <[email protected]>
  • Loading branch information
@reta
reta committed May 12, 2023
1 parent d9f840f commit 14be00c
Show file tree
Hide file tree
Showing 5 changed files with 125 additions and 0 deletions.
5 changes: 5 additions & 0 deletions libs/x-content/src/main/java/org/opensearch/common/xcontent/cbor/CborXContent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import com.fasterxml.jackson.core.JsonEncoding;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.StreamReadConstraints;
import com.fasterxml.jackson.dataformat.cbor.CBORFactory;
import org.opensearch.core.xcontent.DeprecationHandler;
import org.opensearch.core.xcontent.MediaType;
Expand All @@ -56,6 +57,9 @@
* A CBOR based content implementation using Jackson.
*/
public class CborXContent implements XContent {
public static final int DEFAULT_MAX_STRING_LEN = Integer.parseInt(
System.getProperty("opensearch.xcontent.string.length.max", "50000000" /* ~50 Mb */)
);

public static XContentBuilder contentBuilder() throws IOException {
return XContentBuilder.builder(cborXContent);
Expand All @@ -70,6 +74,7 @@ public static XContentBuilder contentBuilder() throws IOException {
// Do not automatically close unclosed objects/arrays in com.fasterxml.jackson.dataformat.cbor.CBORGenerator#close() method
cborFactory.configure(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT, false);
cborFactory.configure(JsonParser.Feature.STRICT_DUPLICATE_DETECTION, true);
cborFactory.setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(DEFAULT_MAX_STRING_LEN).build());
cborXContent = new CborXContent();
}

Expand Down
6 changes: 6 additions & 0 deletions libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.StreamReadConstraints;

import org.opensearch.core.xcontent.DeprecationHandler;
import org.opensearch.core.xcontent.MediaType;
import org.opensearch.core.xcontent.NamedXContentRegistry;
Expand All @@ -55,6 +57,9 @@
* A JSON based content implementation using Jackson.
*/
public class JsonXContent implements XContent {
public static final int DEFAULT_MAX_STRING_LEN = Integer.parseInt(
System.getProperty("opensearch.xcontent.string.length.max", "50000000" /* ~50 Mb */)
);

public static XContentBuilder contentBuilder() throws IOException {
return XContentBuilder.builder(jsonXContent);
Expand All @@ -72,6 +77,7 @@ public static XContentBuilder contentBuilder() throws IOException {
// Do not automatically close unclosed objects/arrays in com.fasterxml.jackson.core.json.UTF8JsonGenerator#close() method
jsonFactory.configure(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT, false);
jsonFactory.configure(JsonParser.Feature.STRICT_DUPLICATE_DETECTION, true);
jsonFactory.setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(DEFAULT_MAX_STRING_LEN).build());
jsonXContent = new JsonXContent();
}

Expand Down
5 changes: 5 additions & 0 deletions libs/x-content/src/main/java/org/opensearch/common/xcontent/smile/SmileXContent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import com.fasterxml.jackson.core.JsonEncoding;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.StreamReadConstraints;
import com.fasterxml.jackson.dataformat.smile.SmileFactory;
import com.fasterxml.jackson.dataformat.smile.SmileGenerator;
import org.opensearch.core.xcontent.DeprecationHandler;
Expand All @@ -56,6 +57,9 @@
* A Smile based content implementation using Jackson.
*/
public class SmileXContent implements XContent {
public static final int DEFAULT_MAX_STRING_LEN = Integer.parseInt(
System.getProperty("opensearch.xcontent.string.length.max", "50000000" /* ~50 Mb */)
);

public static XContentBuilder contentBuilder() throws IOException {
return XContentBuilder.builder(smileXContent);
Expand All @@ -72,6 +76,7 @@ public static XContentBuilder contentBuilder() throws IOException {
// Do not automatically close unclosed objects/arrays in com.fasterxml.jackson.dataformat.smile.SmileGenerator#close() method
smileFactory.configure(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT, false);
smileFactory.configure(JsonParser.Feature.STRICT_DUPLICATE_DETECTION, true);
smileFactory.setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(DEFAULT_MAX_STRING_LEN).build());
smileXContent = new SmileXContent();
}

Expand Down
5 changes: 5 additions & 0 deletions libs/x-content/src/main/java/org/opensearch/common/xcontent/yaml/YamlXContent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@

import com.fasterxml.jackson.core.JsonEncoding;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.StreamReadConstraints;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import org.opensearch.core.xcontent.DeprecationHandler;
import org.opensearch.core.xcontent.NamedXContentRegistry;
Expand All @@ -54,6 +55,9 @@
* A YAML based content implementation using Jackson.
*/
public class YamlXContent implements XContent {
public static final int DEFAULT_MAX_STRING_LEN = Integer.parseInt(
System.getProperty("opensearch.xcontent.string.length.max", "50000000" /* ~50 Mb */)
);

public static XContentBuilder contentBuilder() throws IOException {
return XContentBuilder.builder(yamlXContent);
Expand All @@ -65,6 +69,7 @@ public static XContentBuilder contentBuilder() throws IOException {
static {
yamlFactory = new YAMLFactory();
yamlFactory.configure(JsonParser.Feature.STRICT_DUPLICATE_DETECTION, true);
yamlFactory.setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(DEFAULT_MAX_STRING_LEN).build());
yamlXContent = new YamlXContent();
}

Expand Down
104 changes: 104 additions & 0 deletions libs/x-content/src/test/java/org/opensearch/common/xcontent/XContentParserTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,15 @@
package org.opensearch.common.xcontent;

import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.exc.StreamConstraintsException;
import com.fasterxml.jackson.dataformat.yaml.JacksonYAMLParseException;

import org.opensearch.common.CheckedSupplier;
import org.opensearch.common.Strings;
import org.opensearch.common.bytes.BytesReference;
import org.opensearch.common.xcontent.cbor.CborXContent;
import org.opensearch.common.xcontent.json.JsonXContent;
import org.opensearch.common.xcontent.smile.SmileXContent;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParseException;
import org.opensearch.core.xcontent.XContentParser;
Expand All @@ -50,20 +55,119 @@
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;

import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonMap;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasLength;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.in;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.nullValue;
import static org.junit.internal.matchers.ThrowableMessageMatcher.hasMessage;

public class XContentParserTests extends OpenSearchTestCase {
private static final Map<XContentType, Supplier<String>> GENERATORS = Map.of(
XContentType.JSON,
() -> randomAlphaOfLengthBetween(1, JsonXContent.DEFAULT_MAX_STRING_LEN),
XContentType.CBOR,
() -> randomAlphaOfLengthBetween(1, CborXContent.DEFAULT_MAX_STRING_LEN),
XContentType.SMILE,
() -> randomAlphaOfLengthBetween(1, SmileXContent.DEFAULT_MAX_STRING_LEN),
/* YAML parser limitation */
XContentType.YAML,
() -> randomRealisticUnicodeOfCodepointLengthBetween(1, 3140000)
);

private static final Map<XContentType, Supplier<String>> OFF_LIMIT_GENERATORS = Map.of(
XContentType.JSON,
() -> randomAlphaOfLength(JsonXContent.DEFAULT_MAX_STRING_LEN + 1),
XContentType.CBOR,
() -> randomAlphaOfLength(CborXContent.DEFAULT_MAX_STRING_LEN + 1),
XContentType.SMILE,
() -> randomAlphaOfLength(SmileXContent.DEFAULT_MAX_STRING_LEN + 1),
/* YAML parser limitation */
XContentType.YAML,
() -> randomRealisticUnicodeOfCodepointLength(3145730)
);

public void testStringOffLimit() throws IOException {
final XContentType xContentType = randomFrom(XContentType.values());

final String field = randomRealisticUnicodeOfCodepointLength(5);
final String value = OFF_LIMIT_GENERATORS.get(xContentType).get();

try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) {
builder.startObject();
if (randomBoolean()) {
builder.field(field, value);
} else {
builder.field(field).value(value);
}
builder.endObject();

try (XContentParser parser = createParser(xContentType.xContent(), BytesReference.bytes(builder))) {
assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
assertEquals(field, parser.currentName());
assertEquals(XContentParser.Token.VALUE_STRING, parser.nextToken());
if (xContentType != XContentType.YAML) {
assertThrows(StreamConstraintsException.class, () -> parser.text());
} else {
assertThrows(JacksonYAMLParseException.class, () -> parser.nextToken());
}
}
}
}

public void testString() throws IOException {
final XContentType xContentType = randomFrom(XContentType.values());

final String field = randomRealisticUnicodeOfCodepointLength(5);
final String value = GENERATORS.get(xContentType).get();

try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) {
builder.startObject();
if (randomBoolean()) {
builder.field(field, value);
} else {
builder.field(field).value(value);
}
builder.endObject();

final String text;
try (XContentParser parser = createParser(xContentType.xContent(), BytesReference.bytes(builder))) {
assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
assertEquals(field, parser.currentName());
assertEquals(XContentParser.Token.VALUE_STRING, parser.nextToken());

text = parser.text();

assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
assertNull(parser.nextToken());
}

assertThat(text, hasLength(value.length()));

switch (xContentType) {
case CBOR:
case SMILE:
assertThat(text, instanceOf(String.class));
break;
case JSON:
case YAML:
assertThat(text, instanceOf(String.class));
break;
default:
throw new AssertionError("unexpected x-content type [" + xContentType + "]");
}
}
}

public void testFloat() throws IOException {
final XContentType xContentType = randomFrom(XContentType.values());
Expand Down

0 comments on commit 14be00c

Please sign in to comment.