fix: deal with unescaped char in string as prev

bytedance · Jun 28, 2024 · db8b064 · db8b064
1 parent 1334cf4
commit db8b064
Show file tree

Hide file tree

Showing 9 changed files with 29,860 additions and 29,654 deletions.
diff --git a/fuzz/compat_amd64_test.go b/fuzz/compat_amd64_test.go
diff --git a/fuzz/compat_other.go b/fuzz/compat_other.go
diff --git a/fuzz/other_fuzz_test.go b/fuzz/other_fuzz_test.go
@@ -51,7 +51,7 @@ func fuzzHtmlEscape(t *testing.T, data []byte){
 func fuzzStream(t *testing.T, data []byte) {
 	r := bytes.NewBuffer(data)
 	dc := decoder.NewStreamDecoder(r)
-	decoderEnableValidateString(dc)
+	dc.ValidateString()
 	r1 := bytes.NewBuffer(data)
 	dc1 := decoder.NewStreamDecoder(r1)
 

diff --git a/internal/native/avx2/parse_with_padding_subr.go b/internal/native/avx2/parse_with_padding_subr.go
diff --git a/internal/native/avx2/parse_with_padding_text_amd64.go b/internal/native/avx2/parse_with_padding_text_amd64.go
diff --git a/internal/native/sse/parse_with_padding_subr.go b/internal/native/sse/parse_with_padding_subr.go
diff --git a/internal/native/sse/parse_with_padding_text_amd64.go b/internal/native/sse/parse_with_padding_text_amd64.go
diff --git a/native/parse_with_padding.c b/native/parse_with_padding.c
@@ -588,7 +588,7 @@ static always_inline ssize_t handle_unicode(char** sp, char **dp) {
 
 // positive is length
 // negative is - error_code
-static always_inline long parse_string_inplace(uint8_t** cur, bool* has_esc, bool disable_surrogates_error) {
+static always_inline long parse_string_inplace(uint8_t** cur, bool* has_esc, uint64_t opts) {
     string_block block;
     uint8_t* start = *cur;
     v256u v;
@@ -606,7 +606,7 @@ static always_inline long parse_string_inplace(uint8_t** cur, bool* has_esc, boo
             break;
         }
 
-        if (unlikely(has_unescaped(&block))) {
+        if (unlikely((opts & F_VALIDATE_STRING) != 0 && has_unescaped(&block))) {
             *cur += trailing_zeros(block.esc);
             return -SONIC_CONTROL_CHAR;
         }
@@ -664,7 +664,7 @@ static always_inline long parse_string_inplace(uint8_t** cur, bool* has_esc, boo
         }
     }
 
-    if (unlikely(has_unescaped(&block))) {
+    if (unlikely((opts & F_VALIDATE_STRING) != 0 && has_unescaped(&block))) {
         *cur += trailing_zeros(block.esc);
         return -SONIC_CONTROL_CHAR;
     }
@@ -1115,7 +1115,7 @@ static always_inline error_code parse(GoParser* slf, reader* rdr, visitor* vis)
             neg = false;
             break;
         case '"':
-            slen = parse_string_inplace(cur, &has_esc, true);
+            slen = parse_string_inplace(cur, &has_esc, slf->opt);
             if (slen < 0) {
                 err =  (error_code)(-slen);
             }
@@ -1150,7 +1150,7 @@ static always_inline error_code parse(GoParser* slf, reader* rdr, visitor* vis)
     // parse key
     pos = offset_from(*cur);
 
-    slen = parse_string_inplace(cur, &has_esc, true);
+    slen = parse_string_inplace(cur, &has_esc, slf->opt);
     if (slen < 0) {
         err =  (error_code)(-slen);
         return err;
@@ -1206,7 +1206,7 @@ static always_inline error_code parse(GoParser* slf, reader* rdr, visitor* vis)
             neg = false;
             break;
         case '"':
-            slen = parse_string_inplace(cur, &has_esc, true);
+            slen = parse_string_inplace(cur, &has_esc, slf->opt);
             if (slen < 0) {
                 err =  (error_code)(-slen);
             }
@@ -1294,7 +1294,7 @@ static always_inline error_code parse(GoParser* slf, reader* rdr, visitor* vis)
             neg = false;
             break;
         case '"':
-            slen = parse_string_inplace(cur, &has_esc, true);
+            slen = parse_string_inplace(cur, &has_esc, slf->opt);
             if (slen < 0) {
                 err =  (error_code)(-slen);
             }

diff --git a/rfc_test.go b/rfc_test.go
@@ -0,0 +1,39 @@
+package sonic_test
+
+import (
+	"encoding/json"
+	"testing"
+	"strings"
+
+	"github.com/bytedance/sonic"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestUnescapedCharInString(t *testing.T) {
+	var data =  "\"" + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F" +  "\""
+
+	t.Run("Default", func(t *testing.T) {
+		var sonicv, jsonv string
+		sonice := sonic.Unmarshal([]byte(data), &sonicv)
+		assert.NoError(t, sonice)
+		assert.Equal(t, sonicv, data[1:len(data) - 1])
+
+		jsone := json.Unmarshal([]byte(data), &jsonv)
+		assert.True(t, strings.Contains(jsone.Error(), ("invalid char")))
+		assert.Equal(t, jsonv, "")
+	})
+
+	t.Run("ValidateString", func(t *testing.T) {
+		var sonicv, jsonv string
+		api := sonic.Config {
+			ValidateString: true,
+		}.Froze()
+		sonice := api.Unmarshal([]byte(data), &sonicv)
+		assert.True(t, strings.Contains(sonice.Error(), ("invalid char")))
+		assert.Equal(t, sonicv, "")
+
+		jsone := json.Unmarshal([]byte(data), &jsonv)
+		assert.True(t, strings.Contains(jsone.Error(), ("invalid char")))
+		assert.Equal(t, jsonv, "")
+	})
+}