Merge #66560 #67375 #67548 #67550 #67560

66560: sql: allowing add missing tables from json file r=RichardJCai a=mnovelodou Previously, we could only add missing tables on information_schema or pg_catalog from dump json file This was inadequate because sometimes we need to manually decide how columns data types are going to be To address this, this patch adds a flag that allows to specify the json file where to find the missing tables Release note: None 67375: roachtest: ensure privileges stay consistent after version upgrades r=rafiss a=RichardJCai roachtest: ensure privileges stay consistent after version upgrades Release note: None Resolves #65011 67548: roachtest: fix typeorm apt-get update r=rail a=RichardJCai Release note: None 67550: authors: add xinhaoz to AUTHORS r=xinhaoz a=xinhaoz Release note: None 67560: authors: add [email protected] to authors. r=rharding6373 a=rharding6373 Release note: None Co-authored-by: MiguelNovelo <[email protected]> Co-authored-by: richardjcai <[email protected]> Co-authored-by: Xin Hao Zhang <[email protected]> Co-authored-by: rharding6373 <[email protected]>
cockroachdb · Jul 13, 2021 · 388bbe8 · 388bbe8
6 parents 706f0d2 + 6a117a4 + e47cdfe + 5d9059d + e6201bd + 7936440
commit 388bbe8
Show file tree

Hide file tree

Showing 11 changed files with 359 additions and 27 deletions.
diff --git a/AUTHORS b/AUTHORS
@@ -300,6 +300,7 @@ Piotr Zurek <[email protected]>
 pocockn <[email protected]>
 Pooja Maniar <[email protected]> <[email protected]>
 Poornima Malepati <[email protected]> Poornima <[email protected]>
+Rachael Harding <[email protected]>
 Rachit Srivastava <[email protected]>
 Radu Berinde <[email protected]> RaduBerinde <[email protected]>
 Rafi Shamim <[email protected]> <[email protected]>
@@ -378,6 +379,7 @@ Wade Waldron <[email protected]>
 wenyong-h <[email protected]>
 Will Haack <[email protected]> <[email protected]>
 Xiang Li <[email protected]>
+Xin Hao Zhang <[email protected]> <[email protected]>
 Xinyu Zhou (Joe) <[email protected]>
 XisiHuang <[email protected]>
 xphoniex <[email protected]>

diff --git a/pkg/cmd/generate-metadata-tables/rdbms/metadata.go b/pkg/cmd/generate-metadata-tables/rdbms/metadata.go
@@ -16,6 +16,7 @@ package rdbms
 
 import (
 	"io"
+	"regexp"
 
 	"github.com/cockroachdb/cockroach/pkg/sql"
 )
@@ -34,20 +35,38 @@ type columnMetadata struct {
 	dataTypeOid  uint32
 }
 
+type excludePattern struct {
+	pattern *regexp.Regexp
+	except  map[string]struct{}
+}
+
 // ColumnMetadataList is a list of rows coming from rdbms describing a column.
-type ColumnMetadataList []*columnMetadata
+type ColumnMetadataList struct {
+	data       []*columnMetadata
+	exclusions []*excludePattern
+}
 
 // DBMetadataConnection structs can describe a schema like pg_catalog or
 // information_schema.
 type DBMetadataConnection interface {
 	io.Closer
-	DescribeSchema() (ColumnMetadataList, error)
+	DescribeSchema() (*ColumnMetadataList, error)
 	DatabaseVersion() (string, error)
 }
 
 // ForEachRow iterates over the rows gotten from DescribeSchema() call.
-func (l ColumnMetadataList) ForEachRow(addRow func(string, string, string, uint32)) {
-	for _, c := range l {
-		addRow(c.tableName, c.columnName, c.dataTypeName, c.dataTypeOid)
+func (l *ColumnMetadataList) ForEachRow(addRow func(string, string, string, uint32)) {
+	addRowIfAllowed := func(metadata *columnMetadata) {
+		for _, exclusion := range l.exclusions {
+			tableName := metadata.tableName
+			if _, ok := exclusion.except[tableName]; exclusion.pattern.MatchString(tableName) && !ok {
+				return
+			}
+		}
+
+		addRow(metadata.tableName, metadata.columnName, metadata.dataTypeName, metadata.dataTypeOid)
+	}
+	for _, metadata := range l.data {
+		addRowIfAllowed(metadata)
 	}
 }
diff --git a/pkg/cmd/generate-metadata-tables/rdbms/mysql.go b/pkg/cmd/generate-metadata-tables/rdbms/mysql.go
@@ -16,6 +16,7 @@ package rdbms
 import (
 	gosql "database/sql"
 	"fmt"
+	"regexp"
 	"strings"
 
 	// gosql implementation.
@@ -32,6 +33,13 @@ const mysqlDescribeSchema = `
 	ORDER BY table_name, column_name
 `
 
+var mysqlExclusions = []*excludePattern{
+	{
+		pattern: regexp.MustCompile(`innodb_.+`),
+		except:  make(map[string]struct{}),
+	},
+}
+
 type mysqlMetadataConnection struct {
 	*gosql.DB
 	catalog string
@@ -51,8 +59,8 @@ func (conn mysqlMetadataConnection) DatabaseVersion() (version string, err error
 	return version, err
 }
 
-func (conn mysqlMetadataConnection) DescribeSchema() (ColumnMetadataList, error) {
-	var metadata ColumnMetadataList
+func (conn mysqlMetadataConnection) DescribeSchema() (*ColumnMetadataList, error) {
+	metadata := &ColumnMetadataList{exclusions: mysqlExclusions}
 	rows, err := conn.Query(mysqlDescribeSchema, conn.catalog)
 	if err != nil {
 		return nil, err
@@ -66,7 +74,7 @@ func (conn mysqlMetadataConnection) DescribeSchema() (ColumnMetadataList, error)
 		}
 		row.tableName = strings.ToLower(row.tableName)
 		row.columnName = strings.ToLower(row.columnName)
-		metadata = append(metadata, row)
+		metadata.data = append(metadata.data, row)
 	}
 
 	return metadata, nil

diff --git a/pkg/cmd/generate-metadata-tables/rdbms/postgres.go b/pkg/cmd/generate-metadata-tables/rdbms/postgres.go
@@ -15,6 +15,7 @@ package rdbms
 
 import (
 	"fmt"
+	"regexp"
 
 	"github.com/cockroachdb/cockroach/pkg/sql"
 	"github.com/cockroachdb/cockroach/pkg/sql/types"
@@ -43,6 +44,22 @@ var unimplementedEquivalencies = map[oid.Oid]oid.Oid{
 	oid.T_pg_lsn:       oid.T_text,
 }
 
+var emptyStruct = struct{}{}
+
+var postgresExclusions = []*excludePattern{
+	{
+		pattern: regexp.MustCompile(`^pg_stat.+$`),
+		except: map[string]struct{}{
+			"pg_stat_database":           emptyStruct,
+			"pg_stat_database_conflicts": emptyStruct,
+		},
+	},
+	{
+		pattern: regexp.MustCompile(`^_pg_.+$`),
+		except:  make(map[string]struct{}),
+	},
+}
+
 type pgMetadataConnection struct {
 	*pgx.Conn
 	catalog string
@@ -61,7 +78,7 @@ func postgresConnect(address, user, catalog string) (DBMetadataConnection, error
 	return pgMetadataConnection{conn, catalog}, nil
 }
 
-func (conn pgMetadataConnection) DescribeSchema() (ColumnMetadataList, error) {
+func (conn pgMetadataConnection) DescribeSchema() (*ColumnMetadataList, error) {
 	var metadata []*columnMetadata
 	rows, err := conn.Query(sql.GetPGMetadataSQL, conn.catalog)
 	if err != nil {
@@ -84,7 +101,7 @@ func (conn pgMetadataConnection) DescribeSchema() (ColumnMetadataList, error) {
 		row.dataTypeOid = mappedTypeOid
 		metadata = append(metadata, row)
 	}
-	return metadata, nil
+	return &ColumnMetadataList{data: metadata, exclusions: postgresExclusions}, nil
 }
 
 func (conn pgMetadataConnection) DatabaseVersion() (pgVersion string, err error) {

diff --git a/pkg/cmd/roachtest/BUILD.bazel b/pkg/cmd/roachtest/BUILD.bazel
@@ -53,14 +53,12 @@ go_binary(
 
 go_test(
     name = "roachtest_test",
-    size = "small",
     srcs = [
         "cluster_test.go",
         "test_registry_test.go",
         "test_test.go",
     ],
     embed = [":roachtest_lib"],
-    tags = ["broken_in_bazel"],
     deps = [
         "//pkg/cmd/roachtest/cluster",
         "//pkg/cmd/roachtest/logger",

diff --git a/pkg/cmd/roachtest/tests/BUILD.bazel b/pkg/cmd/roachtest/tests/BUILD.bazel
@@ -78,6 +78,7 @@ go_library(
         "pgx_blocklist.go",
         "pop.go",
         "predecessor_version.go",
+        "privilege_version_upgrade.go",
         "psycopg.go",
         "psycopg_blocklist.go",
         "python_helpers.go",
@@ -150,6 +151,7 @@ go_library(
         "//pkg/server/serverpb",
         "//pkg/sql/pgwire/pgcode",
         "//pkg/sql/pgwire/pgerror",
+        "//pkg/sql/privilege",
         "//pkg/storage/cloud",
         "//pkg/storage/cloud/amazon",
         "//pkg/testutils",