Skip to content

Commit

Permalink
Convert non-unicode input when reading w/ Jackson
Browse files Browse the repository at this point in the history 
This commit makes sure that Jackson-based message converters and
decoders can deal with non-unicode input. It does so by reading
non-unicode input messages with a InputStreamReader.

This commit also adds additional tests forthe canRead/canWrite methods
on both codecs and message converters.

Closes: spring-projectsgh-25247
  • Loading branch information
@poutsma @zx20110729
poutsma authored and zx20110729 committed Feb 18, 2022
1 parent b409fa3 commit 38b9d0f
Show file tree
Hide file tree
Showing 9 changed files with 117 additions and 81 deletions.
26 changes: 21 additions & 5 deletions spring-web/src/main/java/org/springframework/http/codec/json/AbstractJackson2Encoder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,18 @@ public abstract class AbstractJackson2Encoder extends Jackson2CodecSupport imple

private static final Map<MediaType, byte[]> STREAM_SEPARATORS;

private static final Map<Charset, JsonEncoding> ENCODINGS;

static {
STREAM_SEPARATORS = new HashMap<>(4);
STREAM_SEPARATORS.put(MediaType.APPLICATION_STREAM_JSON, NEWLINE_SEPARATOR);
STREAM_SEPARATORS.put(MediaType.parseMediaType("application/stream+x-jackson-smile"), new byte[0]);

ENCODINGS = new HashMap<>(JsonEncoding.values().length);
for (JsonEncoding encoding : JsonEncoding.values()) {
Charset charset = Charset.forName(encoding.getJavaName());
ENCODINGS.put(charset, encoding);
}
}


Expand Down Expand Up @@ -103,7 +111,16 @@ public void setStreamingMediaTypes(List<MediaType> mediaTypes) {
@Override
public boolean canEncode(ResolvableType elementType, @Nullable MimeType mimeType) {
Class<?> clazz = elementType.toClass();
return supportsMimeType(mimeType) && (Object.class == clazz ||
if (!supportsMimeType(mimeType)) {
return false;
}
if (mimeType != null && mimeType.getCharset() != null) {
Charset charset = mimeType.getCharset();
if (!ENCODINGS.containsKey(charset)) {
return false;
}
}
return (Object.class == clazz ||
(!String.class.isAssignableFrom(elementType.resolve(clazz)) && getObjectMapper().canSerialize(clazz)));
}

Expand Down Expand Up @@ -269,10 +286,9 @@ private byte[] streamSeparator(@Nullable MimeType mimeType) {
protected JsonEncoding getJsonEncoding(@Nullable MimeType mimeType) {
if (mimeType != null && mimeType.getCharset() != null) {
Charset charset = mimeType.getCharset();
for (JsonEncoding encoding : JsonEncoding.values()) {
if (charset.name().equals(encoding.getJavaName())) {
return encoding;
}
JsonEncoding result = ENCODINGS.get(charset);
if (result != null) {
return result;
}
}
return JsonEncoding.UTF8;
Expand Down
26 changes: 1 addition & 25 deletions spring-web/src/main/java/org/springframework/http/codec/json/Jackson2CodecSupport.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,13 @@

import java.lang.annotation.Annotation;
import java.lang.reflect.Type;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;

import com.fasterxml.jackson.annotation.JsonView;
import com.fasterxml.jackson.core.JsonEncoding;
import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.type.TypeFactory;
Expand Down Expand Up @@ -69,9 +64,6 @@ public abstract class Jackson2CodecSupport {
new MimeType("application", "json", StandardCharsets.UTF_8),
new MimeType("application", "*+json", StandardCharsets.UTF_8)));

private static final Map<String, JsonEncoding> ENCODINGS = jsonEncodings();



protected final Log logger = HttpLogging.forLogName(getClass());

Expand Down Expand Up @@ -104,17 +96,7 @@ protected List<MimeType> getMimeTypes() {


protected boolean supportsMimeType(@Nullable MimeType mimeType) {
if (mimeType == null) {
return true;
}
else if (this.mimeTypes.stream().noneMatch(m -> m.isCompatibleWith(mimeType))) {
return false;
}
else if (mimeType.getCharset() != null) {
Charset charset = mimeType.getCharset();
return ENCODINGS.containsKey(charset.name());
}
return true;
return (mimeType == null || this.mimeTypes.stream().anyMatch(m -> m.isCompatibleWith(mimeType)));
}

protected JavaType getJavaType(Type type, @Nullable Class<?> contextClass) {
Expand Down Expand Up @@ -143,10 +125,4 @@ protected MethodParameter getParameter(ResolvableType type) {
@Nullable
protected abstract <A extends Annotation> A getAnnotation(MethodParameter parameter, Class<A> annotType);

private static Map<String, JsonEncoding> jsonEncodings() {
return EnumSet.allOf(JsonEncoding.class).stream()
.collect(Collectors.toMap(JsonEncoding::getJavaName, Function.identity()));
}


}
72 changes: 41 additions & 31 deletions ...in/java/org/springframework/http/converter/json/AbstractJackson2HttpMessageConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
package org.springframework.http.converter.json;

import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.lang.reflect.Type;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
Expand All @@ -37,6 +39,7 @@
import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.databind.SerializationConfig;
import com.fasterxml.jackson.databind.SerializationFeature;
Expand Down Expand Up @@ -73,7 +76,7 @@
*/
public abstract class AbstractJackson2HttpMessageConverter extends AbstractGenericHttpMessageConverter<Object> {

private static final Map<String, JsonEncoding> ENCODINGS = jsonEncodings();
private static final Map<Charset, JsonEncoding> ENCODINGS = jsonEncodings();

/**
* The default charset used by the converter.
Expand Down Expand Up @@ -173,19 +176,17 @@ public boolean canRead(Type type, @Nullable Class<?> contextClass, @Nullable Med
return false;
}

@Override
protected boolean canRead(@Nullable MediaType mediaType) {
if (!super.canRead(mediaType)) {
return false;
}
return checkEncoding(mediaType);
}

@Override
public boolean canWrite(Class<?> clazz, @Nullable MediaType mediaType) {
if (!canWrite(mediaType)) {
return false;
}
if (mediaType != null && mediaType.getCharset() != null) {
Charset charset = mediaType.getCharset();
if (!ENCODINGS.containsKey(charset)) {
return false;
}
}
AtomicReference<Throwable> causeRef = new AtomicReference<>();
if (this.objectMapper.canSerialize(clazz, causeRef)) {
return true;
Expand All @@ -194,14 +195,6 @@ public boolean canWrite(Class<?> clazz, @Nullable MediaType mediaType) {
return false;
}

@Override
protected boolean canWrite(@Nullable MediaType mediaType) {
if (!super.canWrite(mediaType)) {
return false;
}
return checkEncoding(mediaType);
}

/**
* Determine whether to log the given exception coming from a
* {@link ObjectMapper#canDeserialize} / {@link ObjectMapper#canSerialize} check.
Expand Down Expand Up @@ -233,14 +226,6 @@ else if (logger.isDebugEnabled()) {
}
}

private boolean checkEncoding(@Nullable MediaType mediaType) {
if (mediaType != null && mediaType.getCharset() != null) {
Charset charset = mediaType.getCharset();
return ENCODINGS.containsKey(charset.name());
}
return true;
}

@Override
protected Object readInternal(Class<?> clazz, HttpInputMessage inputMessage)
throws IOException, HttpMessageNotReadableException {
Expand All @@ -258,15 +243,31 @@ public Object read(Type type, @Nullable Class<?> contextClass, HttpInputMessage
}

private Object readJavaType(JavaType javaType, HttpInputMessage inputMessage) throws IOException {
MediaType contentType = inputMessage.getHeaders().getContentType();
Charset charset = getCharset(contentType);

boolean isUnicode = ENCODINGS.containsKey(charset);
try {
if (inputMessage instanceof MappingJacksonInputMessage) {
Class<?> deserializationView = ((MappingJacksonInputMessage) inputMessage).getDeserializationView();
if (deserializationView != null) {
return this.objectMapper.readerWithView(deserializationView).forType(javaType).
readValue(inputMessage.getBody());
ObjectReader objectReader = this.objectMapper.readerWithView(deserializationView).forType(javaType);
if (isUnicode) {
return objectReader.readValue(inputMessage.getBody());
}
else {
Reader reader = new InputStreamReader(inputMessage.getBody(), charset);
return objectReader.readValue(reader);
}
}
}
return this.objectMapper.readValue(inputMessage.getBody(), javaType);
if (isUnicode) {
return this.objectMapper.readValue(inputMessage.getBody(), javaType);
}
else {
Reader reader = new InputStreamReader(inputMessage.getBody(), charset);
return this.objectMapper.readValue(reader, javaType);
}
}
catch (InvalidDefinitionException ex) {
throw new HttpMessageConversionException("Type definition error: " + ex.getType(), ex);
Expand All @@ -276,6 +277,15 @@ private Object readJavaType(JavaType javaType, HttpInputMessage inputMessage) th
}
}

private static Charset getCharset(@Nullable MediaType contentType) {
if (contentType != null && contentType.getCharset() != null) {
return contentType.getCharset();
}
else {
return StandardCharsets.UTF_8;
}
}

@Override
protected void writeInternal(Object object, @Nullable Type type, HttpOutputMessage outputMessage)
throws IOException, HttpMessageNotWritableException {
Expand Down Expand Up @@ -363,7 +373,7 @@ protected JavaType getJavaType(Type type, @Nullable Class<?> contextClass) {
protected JsonEncoding getJsonEncoding(@Nullable MediaType contentType) {
if (contentType != null && contentType.getCharset() != null) {
Charset charset = contentType.getCharset();
JsonEncoding encoding = ENCODINGS.get(charset.name());
JsonEncoding encoding = ENCODINGS.get(charset);
if (encoding != null) {
return encoding;
}
Expand All @@ -388,9 +398,9 @@ protected Long getContentLength(Object object, @Nullable MediaType contentType)
return super.getContentLength(object, contentType);
}

private static Map<String, JsonEncoding> jsonEncodings() {
private static Map<Charset, JsonEncoding> jsonEncodings() {
return EnumSet.allOf(JsonEncoding.class).stream()
.collect(Collectors.toMap(JsonEncoding::getJavaName, Function.identity()));
.collect(Collectors.toMap(encoding -> Charset.forName(encoding.getJavaName()), Function.identity()));
}

}
22 changes: 21 additions & 1 deletion spring-web/src/test/java/org/springframework/http/codec/json/Jackson2JsonDecoderTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ public void canDecode() {
assertFalse(decoder.canDecode(forClass(Pojo.class), APPLICATION_XML));
assertTrue(this.decoder.canDecode(forClass(Pojo.class),
new MediaType("application", "json", StandardCharsets.UTF_8)));
assertFalse(this.decoder.canDecode(forClass(Pojo.class),
assertTrue(this.decoder.canDecode(forClass(Pojo.class),
new MediaType("application", "json", StandardCharsets.ISO_8859_1)));

}
Expand Down Expand Up @@ -239,6 +239,26 @@ public void decodeNonUtf8Encoding() {
null);
}

@Test
@SuppressWarnings("unchecked")
public void decodeNonUnicode() {
Flux<DataBuffer> input = Flux.concat(
stringBuffer("{\"føø\":\"bår\"}", StandardCharsets.ISO_8859_1)
);

testDecode(input, ResolvableType.forType(new ParameterizedTypeReference<Map<String, String>>() {
}),
step -> step.assertNext(o -> {
assertTrue(o instanceof Map);
Map<String, String> map = (Map<String, String>) o;
assertEquals(1, map.size());
assertEquals("bår", map.get("føø"));
})
.verifyComplete(),
MediaType.parseMediaType("application/json; charset=iso-8859-1"),
null);
}

@Test
public void decodeMonoNonUtf8Encoding() {
Mono<DataBuffer> input = stringBuffer("{\"foo\":\"bar\"}", StandardCharsets.UTF_16);
Expand Down
6 changes: 0 additions & 6 deletions spring-web/src/test/java/org/springframework/http/codec/json/Jackson2SmileDecoderTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@

package org.springframework.http.codec.json;

import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;

Expand Down Expand Up @@ -65,11 +64,6 @@ public void canDecode() {

assertFalse(decoder.canDecode(forClass(String.class), null));
assertFalse(decoder.canDecode(forClass(Pojo.class), APPLICATION_JSON));

assertTrue(this.decoder.canDecode(ResolvableType.forClass(Pojo.class),
new MimeType("application", "x-jackson-smile", StandardCharsets.UTF_8)));
assertFalse(this.decoder.canDecode(ResolvableType.forClass(Pojo.class),
new MimeType("application", "x-jackson-smile", StandardCharsets.ISO_8859_1)));
}


Expand Down
6 changes: 0 additions & 6 deletions spring-web/src/test/java/org/springframework/http/codec/json/Jackson2SmileEncoderTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@

import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;

Expand Down Expand Up @@ -71,11 +70,6 @@ public void canEncode() {
assertTrue(this.encoder.canEncode(pojoType, STREAM_SMILE_MIME_TYPE));
assertTrue(this.encoder.canEncode(pojoType, null));

assertTrue(this.encoder.canEncode(ResolvableType.forClass(Pojo.class),
new MimeType("application", "x-jackson-smile", StandardCharsets.UTF_8)));
assertFalse(this.encoder.canEncode(ResolvableType.forClass(Pojo.class),
new MimeType("application", "x-jackson-smile", StandardCharsets.ISO_8859_1)));

// SPR-15464
assertTrue(this.encoder.canEncode(ResolvableType.NONE, null));
}
Expand Down
18 changes: 16 additions & 2 deletions ...ava/org/springframework/http/converter/json/MappingJackson2HttpMessageConverterTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import java.io.IOException;
import java.lang.reflect.Type;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
Expand Down Expand Up @@ -64,7 +65,7 @@ public void canRead() {
assertTrue(converter.canRead(MyBean.class, new MediaType("application", "json")));
assertTrue(converter.canRead(Map.class, new MediaType("application", "json")));
assertTrue(converter.canRead(MyBean.class, new MediaType("application", "json", StandardCharsets.UTF_8)));
assertFalse(converter.canRead(MyBean.class, new MediaType("application", "json", StandardCharsets.ISO_8859_1)));
assertTrue(converter.canRead(MyBean.class, new MediaType("application", "json", StandardCharsets.ISO_8859_1)));
}

@Test
Expand Down Expand Up @@ -439,7 +440,7 @@ public void writeSubTypeList() throws Exception {
@Test
public void readWithNoDefaultConstructor() throws Exception {
String body = "{\"property1\":\"foo\",\"property2\":\"bar\"}";
MockHttpInputMessage inputMessage = new MockHttpInputMessage(body.getBytes("UTF-8"));
MockHttpInputMessage inputMessage = new MockHttpInputMessage(body.getBytes(StandardCharsets.UTF_8));
inputMessage.getHeaders().setContentType(new MediaType("application", "json"));
try {
converter.read(BeanWithNoDefaultConstructor.class, inputMessage);
Expand All @@ -451,6 +452,19 @@ public void readWithNoDefaultConstructor() throws Exception {
fail();
}

@Test
@SuppressWarnings("unchecked")
public void readNonUnicode() throws Exception {
String body = "{\"føø\":\"bår\"}";
Charset charset = StandardCharsets.ISO_8859_1;
MockHttpInputMessage inputMessage = new MockHttpInputMessage(body.getBytes(charset));
inputMessage.getHeaders().setContentType(new MediaType("application", "json", charset));
HashMap<String, Object> result = (HashMap<String, Object>) this.converter.read(HashMap.class, inputMessage);

assertEquals(1, result.size());
assertEquals("bår", result.get("føø"));
}


interface MyInterface {

Expand Down
Loading

0 comments on commit 38b9d0f

Please sign in to comment.