sql: NFC-normalization on double-quoted identifiers

There was a lack of NFC-normalization on double-quoted
identifiers. This allowed for ambiguous table/db/column names.
This change adds normalization and prevents this case.

Release note: None

pkg/sql/lexbase/normalize.go

@@ -53,3 +53,10 @@ func NormalizeName(n string) string {
 	}
 	return norm.NFC.String(lower)
 }
+
+//NormalizeBytes normalizes to Unicode Normalization Form C (NFC).
+//This function is specifically for double quoted identifiers.
+
+func NormalizeBytes(n []byte) []byte {
+	return norm.NFC.Bytes(n)
+}

pkg/sql/normalization_test.go

@@ -0,0 +1,64 @@
+// Copyright 2021 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package sql_test
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/cockroachdb/cockroach/pkg/base"
+	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
+	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
+	"github.com/cockroachdb/cockroach/pkg/util/log"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNFCNormalization(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+	s, db, _ := serverutils.StartServer(t, base.TestServerArgs{Insecure: true})
+	defer s.Stopper().Stop(context.Background())
+	defer db.Close()
+
+	_, err := db.Exec("CREATE TABLE café (a INT)")
+	require.NoError(t, err)
+
+	_, err = db.Exec(`CREATE TABLE "Café" (a INT)`)
+	require.NoError(t, err)
+
+	_, err = db.Exec("CREATE TABLE Cafe\u0301 (a INT)")
+	require.Errorf(t, err, "The tables should be considered duplicates when normalized")
+	require.True(t, strings.Contains(err.Error(), "already exists"))
+
+	_, err = db.Exec("CREATE TABLE cafe\u0301 (a INT)")
+	require.Errorf(t, err, "The tables should be considered duplicates when normalized")
+	require.True(t, strings.Contains(err.Error(), "already exists"))
+
+	_, err = db.Exec("CREATE TABLE caf\u00E9 (a INT)")
+	require.Errorf(t, err, "The tables should be considered duplicates when normalized")
+	require.True(t, strings.Contains(err.Error(), "already exists"))
+
+	_, err = db.Exec("CREATE TABLE \"caf\u00E9\" (a INT)")
+	require.Errorf(t, err, "The tables should be considered duplicates when normalized")
+	require.True(t, strings.Contains(err.Error(), "already exists"))
+
+	_, err = db.Exec("CREATE TABLE \"cafe\u0301\" (a INT)")
+	require.Errorf(t, err, "The tables should be considered duplicates when normalized")
+	require.True(t, strings.Contains(err.Error(), "already exists"))
+
+	//Ensure normal strings are not normalized like double quoted strings
+	var b bool
+	err = db.QueryRow("SELECT 'caf\u00E9' = 'cafe\u0301'").Scan(&b)
+	require.NoError(t, err)
+	require.False(t, b)
+
+}

pkg/sql/scanner/scan.go

@@ -563,7 +563,7 @@ func (s *Scanner) scanIdent(lval ScanSymType) {
 	}
 	//fmt.Println("parsed: ", s.in[start:s.pos], isASCII, isLower)
 
-	if isLower {
+	if isLower && isASCII {
 		// Already lowercased - nothing to do.
 		lval.SetStr(s.in[start:s.pos])
 	} else if isASCII {
@@ -830,7 +830,6 @@ func (s *Scanner) scanString(lval ScanSymType, ch int, allowEscapes, requireUTF8
 	buf := s.buffer()
 	var runeTmp [utf8.UTFMax]byte
 	start := s.pos
-
 outer:
 	for {
 		switch s.next() {
@@ -918,7 +917,11 @@ outer:
 		return false
 	}
 
-	lval.SetStr(s.finishString(buf))
+	if ch == identQuote {
+		lval.SetStr(s.finishString(lexbase.NormalizeBytes(buf)))
+	} else {
+		lval.SetStr(s.finishString(buf))
+	}
 	return true
 }