diff --git a/docs/generated/sql/functions.md b/docs/generated/sql/functions.md index ca4b99c6b1a9..c53a0b9981da 100644 --- a/docs/generated/sql/functions.md +++ b/docs/generated/sql/functions.md @@ -2371,9 +2371,13 @@ SQL statement omitting multi-region related zone configuration fields. If the CONFIGURE ZONE statement can be inferred by the database’s or table’s zone configuration this will return NULL.

-crdb_internal.show_create_all_tables(dbName: string) → string

Returns a flat log of CREATE table statements followed by -ALTER table statements that add table constraints. The flat log can be used -to recreate a database.’

+crdb_internal.show_create_all_tables(database_name: string) → string

Returns rows of CREATE table statements followed by +ALTER table statements that add table constraints. The rows are ordered +by dependencies. All foreign keys are added after the creation of the table +in the alter statements. +It is not recommended to perform this operation on a database with many +tables. +The output can be used to recreate a database.’

 decode(text: string, format: string) → bytes

Decodes data using format (hex / escape / base64).

 diff --git a/pkg/sql/logictest/testdata/logic_test/show_create_all_tables b/pkg/sql/logictest/testdata/logic_test/show_create_all_tables index a5fe4e7ff8f2..007ac9b7fe4d 100644 --- a/pkg/sql/logictest/testdata/logic_test/show_create_all_tables +++ b/pkg/sql/logictest/testdata/logic_test/show_create_all_tables @@ -8,7 +8,6 @@ query T colnames SHOW CREATE ALL TABLES ---- create_statement -· statement ok CREATE TABLE d.parent ( @@ -47,35 +46,32 @@ SHOW CREATE ALL TABLES ---- create_statement CREATE TABLE public.parent ( - x INT8 NULL, - y INT8 NULL, - z INT8 NULL, - rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), - CONSTRAINT "primary" PRIMARY KEY (rowid ASC), - UNIQUE INDEX parent_x_y_z_key (x ASC, y ASC, z ASC), - UNIQUE INDEX parent_x_key (x ASC), - FAMILY f1 (x, y, z, rowid) + x INT8 NULL, + y INT8 NULL, + z INT8 NULL, + rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + UNIQUE INDEX parent_x_y_z_key (x ASC, y ASC, z ASC), + UNIQUE INDEX parent_x_key (x ASC), + FAMILY f1 (x, y, z, rowid) ); CREATE TABLE public.full_test ( - x INT8 NULL, - y INT8 NULL, - z INT8 NULL, - rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), - CONSTRAINT "primary" PRIMARY KEY (rowid ASC), - UNIQUE INDEX full_test_x_key (x ASC), - FAMILY f1 (x, y, z, rowid) + x INT8 NULL, + y INT8 NULL, + z INT8 NULL, + rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + UNIQUE INDEX full_test_x_key (x ASC), + FAMILY f1 (x, y, z, rowid) ); -CREATE SEQUENCE public.s MINVALUE 1 MAXVALUE 9223372036854775807 INCREMENT 1 START 1; CREATE VIEW public.vx ("?column?") AS SELECT 1; +CREATE SEQUENCE public.s MINVALUE 1 MAXVALUE 9223372036854775807 INCREMENT 1 START 1; ALTER TABLE public.full_test ADD CONSTRAINT fk_x_ref_parent FOREIGN KEY (x, y, z) REFERENCES public.parent(x, y, z) MATCH FULL ON DELETE CASCADE ON UPDATE CASCADE; ALTER TABLE public.full_test ADD CONSTRAINT test_fk FOREIGN KEY (x) REFERENCES public.parent(x) ON DELETE CASCADE; -- Validate foreign key constraints. These can fail if there was unvalidated data during the SHOW CREATE ALL TABLES ALTER TABLE public.full_test VALIDATE CONSTRAINT fk_x_ref_parent; ALTER TABLE public.full_test VALIDATE CONSTRAINT test_fk; - - - # testuser does not have CONNECT on database d and cannot see any tables. user testuser @@ -83,7 +79,6 @@ query T colnames SHOW CREATE ALL TABLES ---- create_statement -· user root @@ -99,35 +94,32 @@ SHOW CREATE ALL TABLES ---- create_statement CREATE TABLE public.parent ( - x INT8 NULL, - y INT8 NULL, - z INT8 NULL, - rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), - CONSTRAINT "primary" PRIMARY KEY (rowid ASC), - UNIQUE INDEX parent_x_y_z_key (x ASC, y ASC, z ASC), - UNIQUE INDEX parent_x_key (x ASC), - FAMILY f1 (x, y, z, rowid) + x INT8 NULL, + y INT8 NULL, + z INT8 NULL, + rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + UNIQUE INDEX parent_x_y_z_key (x ASC, y ASC, z ASC), + UNIQUE INDEX parent_x_key (x ASC), + FAMILY f1 (x, y, z, rowid) ); CREATE TABLE public.full_test ( - x INT8 NULL, - y INT8 NULL, - z INT8 NULL, - rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), - CONSTRAINT "primary" PRIMARY KEY (rowid ASC), - UNIQUE INDEX full_test_x_key (x ASC), - FAMILY f1 (x, y, z, rowid) + x INT8 NULL, + y INT8 NULL, + z INT8 NULL, + rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + UNIQUE INDEX full_test_x_key (x ASC), + FAMILY f1 (x, y, z, rowid) ); -CREATE SEQUENCE public.s MINVALUE 1 MAXVALUE 9223372036854775807 INCREMENT 1 START 1; CREATE VIEW public.vx ("?column?") AS SELECT 1; +CREATE SEQUENCE public.s MINVALUE 1 MAXVALUE 9223372036854775807 INCREMENT 1 START 1; ALTER TABLE public.full_test ADD CONSTRAINT fk_x_ref_parent FOREIGN KEY (x, y, z) REFERENCES public.parent(x, y, z) MATCH FULL ON DELETE CASCADE ON UPDATE CASCADE; ALTER TABLE public.full_test ADD CONSTRAINT test_fk FOREIGN KEY (x) REFERENCES public.parent(x) ON DELETE CASCADE; -- Validate foreign key constraints. These can fail if there was unvalidated data during the SHOW CREATE ALL TABLES ALTER TABLE public.full_test VALIDATE CONSTRAINT fk_x_ref_parent; ALTER TABLE public.full_test VALIDATE CONSTRAINT test_fk; - - - user root # Make sure temp tables don't show up in crdb_internal.show_create_all_tables. @@ -147,7 +139,6 @@ query T colnames SHOW CREATE ALL TABLES ---- create_statement -· # Test that a database with foreign keys has the right order. statement ok @@ -170,57 +161,57 @@ CREATE TABLE c (i int REFERENCES d); CREATE SEQUENCE s; CREATE TABLE s_tbl (id INT PRIMARY KEY DEFAULT nextval('s'), v INT, FAMILY f1 (id, v)); -# Table order should be B, A, E, G, F, D, C, sequence s, s_tbl. +# Table order should be B, A, G, F, E, D, C, sequence s, s_tbl. query T colnames SHOW CREATE ALL TABLES ---- create_statement CREATE TABLE public.b ( - i INT8 NOT NULL, - CONSTRAINT "primary" PRIMARY KEY (i ASC), - FAMILY "primary" (i) + i INT8 NOT NULL, + CONSTRAINT "primary" PRIMARY KEY (i ASC), + FAMILY "primary" (i) ); CREATE TABLE public.a ( - i INT8 NULL, - rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), - CONSTRAINT "primary" PRIMARY KEY (rowid ASC), - FAMILY "primary" (i, rowid) -); -CREATE TABLE public.e ( - i INT8 NOT NULL, - CONSTRAINT "primary" PRIMARY KEY (i ASC), - FAMILY "primary" (i) + i INT8 NULL, + rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + FAMILY "primary" (i, rowid) ); CREATE TABLE public.g ( - i INT8 NOT NULL, - CONSTRAINT "primary" PRIMARY KEY (i ASC), - FAMILY "primary" (i) + i INT8 NOT NULL, + CONSTRAINT "primary" PRIMARY KEY (i ASC), + FAMILY "primary" (i) ); CREATE TABLE public.f ( - i INT8 NOT NULL, - g INT8 NULL, - CONSTRAINT "primary" PRIMARY KEY (i ASC), - FAMILY f1 (i, g) + i INT8 NOT NULL, + g INT8 NULL, + CONSTRAINT "primary" PRIMARY KEY (i ASC), + FAMILY f1 (i, g) +); +CREATE TABLE public.e ( + i INT8 NOT NULL, + CONSTRAINT "primary" PRIMARY KEY (i ASC), + FAMILY "primary" (i) ); CREATE TABLE public.d ( - i INT8 NOT NULL, - e INT8 NULL, - f INT8 NULL, - CONSTRAINT "primary" PRIMARY KEY (i ASC), - FAMILY f1 (i, e, f) + i INT8 NOT NULL, + e INT8 NULL, + f INT8 NULL, + CONSTRAINT "primary" PRIMARY KEY (i ASC), + FAMILY f1 (i, e, f) ); CREATE TABLE public.c ( - i INT8 NULL, - rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), - CONSTRAINT "primary" PRIMARY KEY (rowid ASC), - FAMILY "primary" (i, rowid) + i INT8 NULL, + rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + FAMILY "primary" (i, rowid) ); CREATE SEQUENCE public.s MINVALUE 1 MAXVALUE 9223372036854775807 INCREMENT 1 START 1; CREATE TABLE public.s_tbl ( - id INT8 NOT NULL DEFAULT nextval('test_fk_order.public.s':::STRING::REGCLASS), - v INT8 NULL, - CONSTRAINT "primary" PRIMARY KEY (id ASC), - FAMILY f1 (id, v) + id INT8 NOT NULL DEFAULT nextval('test_fk_order.public.s':::STRING::REGCLASS), + v INT8 NULL, + CONSTRAINT "primary" PRIMARY KEY (id ASC), + FAMILY f1 (id, v) ); ALTER TABLE public.a ADD CONSTRAINT fk_i_ref_b FOREIGN KEY (i) REFERENCES public.b(i); ALTER TABLE public.f ADD CONSTRAINT fk_g_ref_g FOREIGN KEY (g) REFERENCES public.g(i); @@ -234,9 +225,6 @@ ALTER TABLE public.d VALIDATE CONSTRAINT fk_e_ref_e; ALTER TABLE public.d VALIDATE CONSTRAINT fk_f_ref_f; ALTER TABLE public.c VALIDATE CONSTRAINT fk_i_ref_d; - - - # Test that a cycle between two tables is handled correctly. statement ok CREATE DATABASE test_cycle; @@ -278,9 +266,6 @@ ALTER TABLE public.loop_a ADD CONSTRAINT b_id_delete_constraint FOREIGN KEY (b_i ALTER TABLE public.loop_b VALIDATE CONSTRAINT fk_a_id_ref_loop_a; ALTER TABLE public.loop_a VALIDATE CONSTRAINT b_id_delete_constraint; - - - # Test that a primary key with a non-default name works. statement ok CREATE DATABASE test_primary_key; @@ -295,13 +280,11 @@ SHOW CREATE ALL TABLES ---- create_statement CREATE TABLE public.t ( - i INT8 NOT NULL, - CONSTRAINT pk_name PRIMARY KEY (i ASC), - FAMILY "primary" (i) + i INT8 NOT NULL, + CONSTRAINT pk_name PRIMARY KEY (i ASC), + FAMILY "primary" (i) ); - - # Test that computed columns are shown correctly. statement ok CREATE DATABASE test_computed_column; @@ -317,15 +300,12 @@ SHOW CREATE ALL TABLES ---- create_statement CREATE TABLE public.t ( - a INT8 NOT NULL, - b INT8 NULL AS (a + 1:::INT8) STORED, - CONSTRAINT "primary" PRIMARY KEY (a ASC), - FAMILY f1 (a, b) + a INT8 NOT NULL, + b INT8 NULL AS (a + 1:::INT8) STORED, + CONSTRAINT "primary" PRIMARY KEY (a ASC), + FAMILY f1 (a, b) ); - - - # Test showing a table with a semicolon in the table, index, and # column names properly escapes. statement ok @@ -339,16 +319,13 @@ SHOW CREATE ALL TABLES ---- create_statement CREATE TABLE public.";" ( - ";" INT8 NULL, - rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), - CONSTRAINT "primary" PRIMARY KEY (rowid ASC), - INDEX ";_;_idx" (";" ASC), - FAMILY "primary" (";", rowid) + ";" INT8 NULL, + rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + INDEX ";_;_idx" (";" ASC), + FAMILY "primary" (";", rowid) ); - - - # Ensure quotes in comments are properly escaped, also that the object names # are properly escaped in the output of the COMMENT statements. statement ok @@ -372,8 +349,6 @@ COMMENT ON TABLE public."t t" IS e'has \' quotes'; COMMENT ON COLUMN public."t t"."x'" IS e'i \' just \' love \' quotes'; COMMENT ON INDEX public."t t"@primary IS e'has \' more \' quotes'; - - # Ensure schemas are shown correctly. statement ok CREATE DATABASE test_schema; @@ -388,21 +363,18 @@ SHOW CREATE ALL TABLES ---- create_statement CREATE TABLE sc1.t ( - x INT8 NULL, - rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), - CONSTRAINT "primary" PRIMARY KEY (rowid ASC), - FAMILY "primary" (x, rowid) + x INT8 NULL, + rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + FAMILY "primary" (x, rowid) ); CREATE TABLE sc2.t ( - x INT8 NULL, - rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), - CONSTRAINT "primary" PRIMARY KEY (rowid ASC), - FAMILY "primary" (x, rowid) + x INT8 NULL, + rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + FAMILY "primary" (x, rowid) ); - - - # Ensure sequences are shown correctly. statement ok CREATE DATABASE test_sequence; diff --git a/pkg/sql/logictest/testdata/logic_test/show_create_all_tables_builtin b/pkg/sql/logictest/testdata/logic_test/show_create_all_tables_builtin index b36f033f7486..092d55b6e5e7 100644 --- a/pkg/sql/logictest/testdata/logic_test/show_create_all_tables_builtin +++ b/pkg/sql/logictest/testdata/logic_test/show_create_all_tables_builtin @@ -1,7 +1,5 @@ -query T +statement error pq: crdb_internal.show_create_all_tables: database "d" does not exist SELECT crdb_internal.show_create_all_tables('d') ----- -· statement ok CREATE DATABASE d @@ -9,7 +7,6 @@ CREATE DATABASE d query T SELECT crdb_internal.show_create_all_tables('d') ---- -· statement ok CREATE TABLE d.parent ( @@ -47,40 +44,38 @@ query T SELECT crdb_internal.show_create_all_tables('d') ---- CREATE TABLE public.parent ( - x INT8 NULL, - y INT8 NULL, - z INT8 NULL, - rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), - CONSTRAINT "primary" PRIMARY KEY (rowid ASC), - UNIQUE INDEX parent_x_y_z_key (x ASC, y ASC, z ASC), - UNIQUE INDEX parent_x_key (x ASC), - FAMILY f1 (x, y, z, rowid) + x INT8 NULL, + y INT8 NULL, + z INT8 NULL, + rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + UNIQUE INDEX parent_x_y_z_key (x ASC, y ASC, z ASC), + UNIQUE INDEX parent_x_key (x ASC), + FAMILY f1 (x, y, z, rowid) ); CREATE TABLE public.full_test ( - x INT8 NULL, - y INT8 NULL, - z INT8 NULL, - rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), - CONSTRAINT "primary" PRIMARY KEY (rowid ASC), - UNIQUE INDEX full_test_x_key (x ASC), - FAMILY f1 (x, y, z, rowid) + x INT8 NULL, + y INT8 NULL, + z INT8 NULL, + rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + UNIQUE INDEX full_test_x_key (x ASC), + FAMILY f1 (x, y, z, rowid) ); -CREATE SEQUENCE public.s MINVALUE 1 MAXVALUE 9223372036854775807 INCREMENT 1 START 1; CREATE VIEW public.vx ("?column?") AS SELECT 1; +CREATE SEQUENCE public.s MINVALUE 1 MAXVALUE 9223372036854775807 INCREMENT 1 START 1; ALTER TABLE public.full_test ADD CONSTRAINT fk_x_ref_parent FOREIGN KEY (x, y, z) REFERENCES public.parent(x, y, z) MATCH FULL ON DELETE CASCADE ON UPDATE CASCADE; ALTER TABLE public.full_test ADD CONSTRAINT test_fk FOREIGN KEY (x) REFERENCES public.parent(x) ON DELETE CASCADE; -- Validate foreign key constraints. These can fail if there was unvalidated data during the SHOW CREATE ALL TABLES ALTER TABLE public.full_test VALIDATE CONSTRAINT fk_x_ref_parent; ALTER TABLE public.full_test VALIDATE CONSTRAINT test_fk; - # testuser does not have CONNECT on database d and cannot see any tables. user testuser query T SELECT crdb_internal.show_create_all_tables('d') ---- -· user root @@ -95,33 +90,32 @@ query T SELECT crdb_internal.show_create_all_tables('d') ---- CREATE TABLE public.parent ( - x INT8 NULL, - y INT8 NULL, - z INT8 NULL, - rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), - CONSTRAINT "primary" PRIMARY KEY (rowid ASC), - UNIQUE INDEX parent_x_y_z_key (x ASC, y ASC, z ASC), - UNIQUE INDEX parent_x_key (x ASC), - FAMILY f1 (x, y, z, rowid) + x INT8 NULL, + y INT8 NULL, + z INT8 NULL, + rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + UNIQUE INDEX parent_x_y_z_key (x ASC, y ASC, z ASC), + UNIQUE INDEX parent_x_key (x ASC), + FAMILY f1 (x, y, z, rowid) ); CREATE TABLE public.full_test ( - x INT8 NULL, - y INT8 NULL, - z INT8 NULL, - rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), - CONSTRAINT "primary" PRIMARY KEY (rowid ASC), - UNIQUE INDEX full_test_x_key (x ASC), - FAMILY f1 (x, y, z, rowid) + x INT8 NULL, + y INT8 NULL, + z INT8 NULL, + rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + UNIQUE INDEX full_test_x_key (x ASC), + FAMILY f1 (x, y, z, rowid) ); -CREATE SEQUENCE public.s MINVALUE 1 MAXVALUE 9223372036854775807 INCREMENT 1 START 1; CREATE VIEW public.vx ("?column?") AS SELECT 1; +CREATE SEQUENCE public.s MINVALUE 1 MAXVALUE 9223372036854775807 INCREMENT 1 START 1; ALTER TABLE public.full_test ADD CONSTRAINT fk_x_ref_parent FOREIGN KEY (x, y, z) REFERENCES public.parent(x, y, z) MATCH FULL ON DELETE CASCADE ON UPDATE CASCADE; ALTER TABLE public.full_test ADD CONSTRAINT test_fk FOREIGN KEY (x) REFERENCES public.parent(x) ON DELETE CASCADE; -- Validate foreign key constraints. These can fail if there was unvalidated data during the SHOW CREATE ALL TABLES ALTER TABLE public.full_test VALIDATE CONSTRAINT fk_x_ref_parent; ALTER TABLE public.full_test VALIDATE CONSTRAINT test_fk; - user root # Make sure temp tables don't show up in crdb_internal.show_create_all_tables. @@ -140,7 +134,6 @@ CREATE TEMPORARY TABLE t() query T SELECT crdb_internal.show_create_all_tables('temp_test') ---- -· # Test that a database with foreign keys has the right order. statement ok @@ -168,51 +161,51 @@ query T SELECT crdb_internal.show_create_all_tables('test_fk_order') ---- CREATE TABLE public.b ( - i INT8 NOT NULL, - CONSTRAINT "primary" PRIMARY KEY (i ASC), - FAMILY "primary" (i) + i INT8 NOT NULL, + CONSTRAINT "primary" PRIMARY KEY (i ASC), + FAMILY "primary" (i) ); CREATE TABLE public.a ( - i INT8 NULL, - rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), - CONSTRAINT "primary" PRIMARY KEY (rowid ASC), - FAMILY "primary" (i, rowid) -); -CREATE TABLE public.e ( - i INT8 NOT NULL, - CONSTRAINT "primary" PRIMARY KEY (i ASC), - FAMILY "primary" (i) + i INT8 NULL, + rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + FAMILY "primary" (i, rowid) ); CREATE TABLE public.g ( - i INT8 NOT NULL, - CONSTRAINT "primary" PRIMARY KEY (i ASC), - FAMILY "primary" (i) + i INT8 NOT NULL, + CONSTRAINT "primary" PRIMARY KEY (i ASC), + FAMILY "primary" (i) ); CREATE TABLE public.f ( - i INT8 NOT NULL, - g INT8 NULL, - CONSTRAINT "primary" PRIMARY KEY (i ASC), - FAMILY f1 (i, g) + i INT8 NOT NULL, + g INT8 NULL, + CONSTRAINT "primary" PRIMARY KEY (i ASC), + FAMILY f1 (i, g) +); +CREATE TABLE public.e ( + i INT8 NOT NULL, + CONSTRAINT "primary" PRIMARY KEY (i ASC), + FAMILY "primary" (i) ); CREATE TABLE public.d ( - i INT8 NOT NULL, - e INT8 NULL, - f INT8 NULL, - CONSTRAINT "primary" PRIMARY KEY (i ASC), - FAMILY f1 (i, e, f) + i INT8 NOT NULL, + e INT8 NULL, + f INT8 NULL, + CONSTRAINT "primary" PRIMARY KEY (i ASC), + FAMILY f1 (i, e, f) ); CREATE TABLE public.c ( - i INT8 NULL, - rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), - CONSTRAINT "primary" PRIMARY KEY (rowid ASC), - FAMILY "primary" (i, rowid) + i INT8 NULL, + rowid INT8 NOT VISIBLE NOT NULL DEFAULT unique_rowid(), + CONSTRAINT "primary" PRIMARY KEY (rowid ASC), + FAMILY "primary" (i, rowid) ); CREATE SEQUENCE public.s MINVALUE 1 MAXVALUE 9223372036854775807 INCREMENT 1 START 1; CREATE TABLE public.s_tbl ( - id INT8 NOT NULL DEFAULT nextval('test_fk_order.public.s':::STRING::REGCLASS), - v INT8 NULL, - CONSTRAINT "primary" PRIMARY KEY (id ASC), - FAMILY f1 (id, v) + id INT8 NOT NULL DEFAULT nextval('test_fk_order.public.s':::STRING::REGCLASS), + v INT8 NULL, + CONSTRAINT "primary" PRIMARY KEY (id ASC), + FAMILY f1 (id, v) ); ALTER TABLE public.a ADD CONSTRAINT fk_i_ref_b FOREIGN KEY (i) REFERENCES public.b(i); ALTER TABLE public.f ADD CONSTRAINT fk_g_ref_g FOREIGN KEY (g) REFERENCES public.g(i); @@ -226,7 +219,6 @@ ALTER TABLE public.d VALIDATE CONSTRAINT fk_e_ref_e; ALTER TABLE public.d VALIDATE CONSTRAINT fk_f_ref_f; ALTER TABLE public.c VALIDATE CONSTRAINT fk_i_ref_d; - # Test that a cycle between two tables is handled correctly. statement ok CREATE DATABASE test_cycle; @@ -267,7 +259,6 @@ ALTER TABLE public.loop_a ADD CONSTRAINT b_id_delete_constraint FOREIGN KEY (b_i ALTER TABLE public.loop_b VALIDATE CONSTRAINT fk_a_id_ref_loop_a; ALTER TABLE public.loop_a VALIDATE CONSTRAINT b_id_delete_constraint; - # Test that a primary key with a non-default name works. statement ok CREATE DATABASE test_primary_key; @@ -304,7 +295,6 @@ CREATE TABLE public.t ( FAMILY f1 (a, b) ); - # Test showing a table with a semicolon in the table, index, and # column names properly escapes. statement ok @@ -323,7 +313,6 @@ CREATE TABLE public.";" ( FAMILY "primary" (";", rowid) ); - # Ensure quotes in comments are properly escaped, also that the object names # are properly escaped in the output of the COMMENT statements. statement ok @@ -370,7 +359,6 @@ CREATE TABLE sc2.t ( FAMILY "primary" (x, rowid) ); - # Ensure sequences are shown correctly. statement ok CREATE DATABASE test_sequence; diff --git a/pkg/sql/opt/norm/project_set_funcs.go b/pkg/sql/opt/norm/project_set_funcs.go index 733185423a5c..4dec3e777083 100644 --- a/pkg/sql/opt/norm/project_set_funcs.go +++ b/pkg/sql/opt/norm/project_set_funcs.go @@ -141,7 +141,7 @@ func (c *CustomFuncs) ConstructValuesFromZips(zip memo.ZipExpr) memo.RelExpr { val := c.f.ConstructConstVal(vals[0], vals[0].ResolvedType()) addValToOutRows(val, j, i) } - generator.Close() + generator.Close(c.f.evalCtx.Context) default: panic(errors.AssertionFailedf("invalid parameter type")) diff --git a/pkg/sql/rowexec/project_set.go b/pkg/sql/rowexec/project_set.go index 293d9181ae2e..53e61eef3f1d 100644 --- a/pkg/sql/rowexec/project_set.go +++ b/pkg/sql/rowexec/project_set.go @@ -92,7 +92,13 @@ func newProjectSetProcessor( processorID, output, nil, /* memMonitor */ - execinfra.ProcStateOpts{InputsToDrain: []execinfra.RowSource{ps.input}}, + execinfra.ProcStateOpts{ + InputsToDrain: []execinfra.RowSource{ps.input}, + TrailingMetaCallback: func() []execinfrapb.ProducerMetadata { + ps.close() + return nil + }, + }, ); err != nil { return nil, err } @@ -278,6 +284,21 @@ func (ps *projectSetProcessor) toEncDatum(d tree.Datum, colIdx int) rowenc.EncDa return rowenc.DatumToEncDatum(ctyp, d) } +func (ps *projectSetProcessor) close() { + if ps.InternalClose() { + for _, gen := range ps.gens { + if gen != nil { + gen.Close(ps.Ctx) + } + } + } +} + +// ConsumerClosed is part of the RowSource interface. +func (ps *projectSetProcessor) ConsumerClosed() { + ps.close() +} + // ChildCount is part of the execinfra.OpNode interface. func (ps *projectSetProcessor) ChildCount(verbose bool) int { if _, ok := ps.input.(execinfra.OpNode); ok { diff --git a/pkg/sql/sem/builtins/builtins.go b/pkg/sql/sem/builtins/builtins.go index e9007aaafa8e..97f1b2e0058b 100644 --- a/pkg/sql/sem/builtins/builtins.go +++ b/pkg/sql/sem/builtins/builtins.go @@ -5052,21 +5052,6 @@ table's zone configuration this will return NULL.`, tree.VolatilityStable, ), ), - - "crdb_internal.show_create_all_tables": makeBuiltin( - tree.FunctionProperties{}, - tree.Overload{ - Types: tree.ArgTypes{ - {"dbName", types.String}, - }, - ReturnType: tree.FixedReturnType(types.String), - Fn: showCreateAllTablesBuiltin, - Info: `Returns a flat log of CREATE table statements followed by -ALTER table statements that add table constraints. The flat log can be used -to recreate a database.'`, - Volatility: tree.VolatilityStable, - }, - ), } var lengthImpls = func(incBitOverload bool) builtinDefinition { diff --git a/pkg/sql/sem/builtins/generator_builtins.go b/pkg/sql/sem/builtins/generator_builtins.go index c3bfd379abdc..79c6fd6023b7 100644 --- a/pkg/sql/sem/builtins/generator_builtins.go +++ b/pkg/sql/sem/builtins/generator_builtins.go @@ -32,6 +32,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/duration" "github.com/cockroachdb/cockroach/pkg/util/errorutil" "github.com/cockroachdb/cockroach/pkg/util/json" + "github.com/cockroachdb/cockroach/pkg/util/mon" + "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/cockroachdb/cockroach/pkg/util/tracing" "github.com/cockroachdb/errors" pbtypes "github.com/gogo/protobuf/types" @@ -84,7 +86,7 @@ type aclexplodeGenerator struct{} func (aclexplodeGenerator) ResolvedType() *types.T { return aclexplodeGeneratorType } func (aclexplodeGenerator) Start(_ context.Context, _ *kv.Txn) error { return nil } -func (aclexplodeGenerator) Close() {} +func (aclexplodeGenerator) Close(_ context.Context) {} func (aclexplodeGenerator) Next(_ context.Context) (bool, error) { return false, nil } func (aclexplodeGenerator) Values() (tree.Datums, error) { return nil, nil } @@ -348,7 +350,6 @@ var generators = map[string]builtinDefinition{ tree.VolatilityVolatile, ), ), - "crdb_internal.payloads_for_trace": makeBuiltin( tree.FunctionProperties{ Class: tree.GeneratorClass, @@ -364,6 +365,27 @@ var generators = map[string]builtinDefinition{ tree.VolatilityVolatile, ), ), + "crdb_internal.show_create_all_tables": makeBuiltin( + tree.FunctionProperties{ + Class: tree.GeneratorClass, + }, + makeGeneratorOverload( + tree.ArgTypes{ + {"database_name", types.String}, + }, + showCreateAllTablesGeneratorType, + makeShowCreateAllTablesGenerator, + `Returns rows of CREATE table statements followed by +ALTER table statements that add table constraints. The rows are ordered +by dependencies. All foreign keys are added after the creation of the table +in the alter statements. +It is not recommended to perform this operation on a database with many +tables. +The output can be used to recreate a database.' +`, + tree.VolatilityVolatile, + ), + ), } func makeGeneratorOverload( @@ -420,7 +442,7 @@ func makeRegexpSplitToTableGeneratorFactory(hasFlags bool) tree.GeneratorFactory func (*regexpSplitToTableGenerator) ResolvedType() *types.T { return types.String } // Close implements the tree.ValueGenerator interface. -func (*regexpSplitToTableGenerator) Close() {} +func (*regexpSplitToTableGenerator) Close(_ context.Context) {} // Start implements the tree.ValueGenerator interface. func (g *regexpSplitToTableGenerator) Start(_ context.Context, _ *kv.Txn) error { @@ -457,7 +479,7 @@ func makeKeywordsGenerator(_ *tree.EvalContext, _ tree.Datums) (tree.ValueGenera func (*keywordsValueGenerator) ResolvedType() *types.T { return keywordsValueGeneratorType } // Close implements the tree.ValueGenerator interface. -func (*keywordsValueGenerator) Close() {} +func (*keywordsValueGenerator) Close(_ context.Context) {} // Start implements the tree.ValueGenerator interface. func (k *keywordsValueGenerator) Start(_ context.Context, _ *kv.Txn) error { @@ -609,7 +631,7 @@ func (s *seriesValueGenerator) Start(_ context.Context, _ *kv.Txn) error { } // Close implements the tree.ValueGenerator interface. -func (s *seriesValueGenerator) Close() {} +func (s *seriesValueGenerator) Close(_ context.Context) {} // Next implements the tree.ValueGenerator interface. func (s *seriesValueGenerator) Next(_ context.Context) (bool, error) { @@ -660,7 +682,7 @@ func (s *multipleArrayValueGenerator) Start(_ context.Context, _ *kv.Txn) error } // Close implements the tree.ValueGenerator interface. -func (s *multipleArrayValueGenerator) Close() {} +func (s *multipleArrayValueGenerator) Close(_ context.Context) {} // Next implements the tree.ValueGenerator interface. func (s *multipleArrayValueGenerator) Next(_ context.Context) (bool, error) { @@ -709,7 +731,7 @@ func (s *arrayValueGenerator) Start(_ context.Context, _ *kv.Txn) error { } // Close implements the tree.ValueGenerator interface. -func (s *arrayValueGenerator) Close() {} +func (s *arrayValueGenerator) Close(_ context.Context) {} // Next implements the tree.ValueGenerator interface. func (s *arrayValueGenerator) Next(_ context.Context) (bool, error) { @@ -758,7 +780,7 @@ func (s *expandArrayValueGenerator) Start(_ context.Context, _ *kv.Txn) error { } // Close implements the tree.ValueGenerator interface. -func (s *expandArrayValueGenerator) Close() {} +func (s *expandArrayValueGenerator) Close(_ context.Context) {} // Next implements the tree.ValueGenerator interface. func (s *expandArrayValueGenerator) Next(_ context.Context) (bool, error) { @@ -831,7 +853,7 @@ func (s *subscriptsValueGenerator) Start(_ context.Context, _ *kv.Txn) error { } // Close implements the tree.ValueGenerator interface. -func (s *subscriptsValueGenerator) Close() {} +func (s *subscriptsValueGenerator) Close(_ context.Context) {} // Next implements the tree.ValueGenerator interface. func (s *subscriptsValueGenerator) Next(_ context.Context) (bool, error) { @@ -876,7 +898,7 @@ func (s *unaryValueGenerator) Start(_ context.Context, _ *kv.Txn) error { } // Close implements the tree.ValueGenerator interface. -func (s *unaryValueGenerator) Close() {} +func (s *unaryValueGenerator) Close(_ context.Context) {} // Next implements the tree.ValueGenerator interface. func (s *unaryValueGenerator) Next(_ context.Context) (bool, error) { @@ -981,7 +1003,7 @@ func (g *jsonArrayGenerator) Start(_ context.Context, _ *kv.Txn) error { } // Close implements the tree.ValueGenerator interface. -func (g *jsonArrayGenerator) Close() {} +func (g *jsonArrayGenerator) Close(_ context.Context) {} // Next implements the tree.ValueGenerator interface. func (g *jsonArrayGenerator) Next(_ context.Context) (bool, error) { @@ -1050,7 +1072,7 @@ func (g *jsonObjectKeysGenerator) ResolvedType() *types.T { func (g *jsonObjectKeysGenerator) Start(_ context.Context, _ *kv.Txn) error { return nil } // Close implements the tree.ValueGenerator interface. -func (g *jsonObjectKeysGenerator) Close() {} +func (g *jsonObjectKeysGenerator) Close(_ context.Context) {} // Next implements the tree.ValueGenerator interface. func (g *jsonObjectKeysGenerator) Next(_ context.Context) (bool, error) { @@ -1146,7 +1168,7 @@ func (g *jsonEachGenerator) Start(_ context.Context, _ *kv.Txn) error { } // Close implements the tree.ValueGenerator interface. -func (g *jsonEachGenerator) Close() {} +func (g *jsonEachGenerator) Close(_ context.Context) {} // Next implements the tree.ValueGenerator interface. func (g *jsonEachGenerator) Next(_ context.Context) (bool, error) { @@ -1278,7 +1300,7 @@ func (c *checkConsistencyGenerator) Values() (tree.Datums, error) { } // Close is part of the tree.ValueGenerator interface. -func (c *checkConsistencyGenerator) Close() {} +func (c *checkConsistencyGenerator) Close(_ context.Context) {} // rangeKeyIteratorChunkSize is the number of K/V pairs that the // rangeKeyIterator requests at a time. If this changes, make sure @@ -1400,7 +1422,7 @@ func (rk *rangeKeyIterator) Values() (tree.Datums, error) { } // Close implements the tree.ValueGenerator interface. -func (rk *rangeKeyIterator) Close() {} +func (rk *rangeKeyIterator) Close(_ context.Context) {} var payloadsForSpanGeneratorLabels = []string{"payload_type", "payload_jsonb"} @@ -1530,7 +1552,7 @@ func (p *payloadsForSpanGenerator) Values() (tree.Datums, error) { } // Close implements the tree.ValueGenerator interface. -func (p *payloadsForSpanGenerator) Close() {} +func (p *payloadsForSpanGenerator) Close(_ context.Context) {} var payloadsForTraceGeneratorLabels = []string{"span_id", "payload_type", "payload_jsonb"} @@ -1607,10 +1629,184 @@ func (p *payloadsForTraceGenerator) Values() (tree.Datums, error) { } // Close implements the tree.ValueGenerator interface. -func (p *payloadsForTraceGenerator) Close() { +func (p *payloadsForTraceGenerator) Close(_ context.Context) { err := p.it.Close() if err != nil { // TODO(angelapwen, yuzefovich): The iterator's error should be surfaced here. return } } + +var showCreateAllTablesGeneratorType = types.String + +// Phase is used to determine if CREATE statements or ALTER statements +// are being generated for showCreateAllTables. +type Phase int + +const ( + create Phase = iota + alterAddFks + alterValidateFks +) + +// showCreateAllTablesGenerator supports the execution of +// crdb_internal.show_create_all_tables(dbName). +type showCreateAllTablesGenerator struct { + ie sqlutil.InternalExecutor + txn *kv.Txn + timestamp string + ids []int64 + dbName string + acc mon.BoundAccount + + // The following variables are updated during + // calls to Next() and change throughout the lifecycle of + // showCreateAllTablesGenerator. + curr tree.Datum + idx int + shouldValidate bool + alterArr tree.Datums + alterArrIdx int + phase Phase +} + +// ResolvedType implements the tree.ValueGenerator interface. +func (s *showCreateAllTablesGenerator) ResolvedType() *types.T { + return showCreateAllTablesGeneratorType +} + +// Start implements the tree.ValueGenerator interface. +func (s *showCreateAllTablesGenerator) Start(ctx context.Context, txn *kv.Txn) error { + // Note: All the table ids are accumulated in ram before the generator + // starts generating values. + // This is reasonable under the assumption that: + // This uses approximately the same amount of memory as required when + // generating the vtable crdb_internal.show_create_statements. If generating + // and reading from the vtable succeeds which we do to retrieve the ids, then + // it is reasonable to use the same amount of memory to hold the ids in + // ram during the lifecycle of showCreateAllTablesGenerator. + // + // We also account for the memory in the BoundAccount memory monitor in + // showCreateAllTablesGenerator. + ids, err := getTopologicallySortedTableIDs( + ctx, s.ie, txn, s.dbName, s.timestamp, &s.acc, + ) + if err != nil { + return err + } + + s.ids = ids + + s.txn = txn + s.idx = -1 + s.phase = create + return nil +} + +func (s *showCreateAllTablesGenerator) Next(ctx context.Context) (bool, error) { + switch s.phase { + case create: + s.idx++ + if s.idx >= len(s.ids) { + // Were done generating the create statements, start generating alters. + s.phase = alterAddFks + s.idx = -1 + return s.Next(ctx) + } + + createStmt, err := getCreateStatement( + ctx, s.ie, s.txn, s.ids[s.idx], s.timestamp, s.dbName, + ) + if err != nil { + return false, err + } + createStmtStr := string(tree.MustBeDString(createStmt)) + s.curr = tree.NewDString(createStmtStr + ";") + case alterAddFks, alterValidateFks: + // We have existing alter statements to generate for the current + // table id. + s.alterArrIdx++ + if s.alterArrIdx < len(s.alterArr) { + alterStmtStr := string(tree.MustBeDString(s.alterArr[s.alterArrIdx])) + s.curr = tree.NewDString(alterStmtStr + ";") + + // At least one FK was added, we must validate the FK. + s.shouldValidate = true + return true, nil + } + // We need to generate the alter statements for the next table. + s.idx++ + if s.idx >= len(s.ids) { + if s.phase == alterAddFks { + // Were done generating the alter fk statements, + // start generating alter validate fk statements. + s.phase = alterValidateFks + s.idx = -1 + + if s.shouldValidate { + // Add a warning about the possibility of foreign key + // validation failing. + s.curr = tree.NewDString(foreignKeyValidationWarning) + return true, nil + } + return s.Next(ctx) + } + // We're done if were on phase alterValidateFks and we + // finish going through all the table ids. + return false, nil + } + + statementType := alterAddFKStatements + if s.phase == alterValidateFks { + statementType = alterValidateFKStatements + } + alterStmt, err := getAlterStatements( + ctx, s.ie, s.txn, s.ids[s.idx], s.timestamp, s.dbName, statementType, + ) + if err != nil { + return false, err + } + if alterStmt == nil { + // There can be no ALTER statements for a given id, in this case + // we go next. + return s.Next(ctx) + } + s.alterArr = tree.MustBeDArray(alterStmt).Array + s.alterArrIdx = -1 + return s.Next(ctx) + } + + return true, nil +} + +// Values implements the tree.ValueGenerator interface. +func (s *showCreateAllTablesGenerator) Values() (tree.Datums, error) { + return tree.Datums{s.curr}, nil +} + +// Close implements the tree.ValueGenerator interface. +func (s *showCreateAllTablesGenerator) Close(ctx context.Context) { + s.acc.Close(ctx) +} + +// makeShowCreateAllTablesGenerator creates a generator to support the +// crdb_internal.show_create_all_tables(dbName) builtin. +// We use the timestamp of when the generator is created as the +// timestamp to pass to AS OF SYSTEM TIME for looking up the create table +// and alter table statements. +func makeShowCreateAllTablesGenerator( + ctx *tree.EvalContext, args tree.Datums, +) (tree.ValueGenerator, error) { + dbName := string(tree.MustBeDString(args[0])) + tsI, err := tree.MakeDTimestamp(timeutil.Now(), time.Microsecond) + if err != nil { + return nil, err + } + ts := tsI.String() + return &showCreateAllTablesGenerator{ + timestamp: ts, + dbName: dbName, + ie: ctx.InternalExecutor.(sqlutil.InternalExecutor), + acc: ctx.Mon.MakeBoundAccount(), + }, nil +} diff --git a/pkg/sql/sem/builtins/geo_builtins.go b/pkg/sql/sem/builtins/geo_builtins.go index bd465d246552..ba541bafc84d 100644 --- a/pkg/sql/sem/builtins/geo_builtins.go +++ b/pkg/sql/sem/builtins/geo_builtins.go @@ -409,7 +409,7 @@ func (m *minimumBoundRadiusGen) Values() (tree.Datums, error) { tree.NewDFloat(tree.DFloat(m.radius))}, nil } -func (m *minimumBoundRadiusGen) Close() {} +func (m *minimumBoundRadiusGen) Close(_ context.Context) {} func makeSubdividedGeometriesGeneratorFactory(expectMaxVerticesArg bool) tree.GeneratorFactory { return func( @@ -441,7 +441,7 @@ type subdividedGeometriesGen struct { func (s *subdividedGeometriesGen) ResolvedType() *types.T { return types.Geometry } -func (s *subdividedGeometriesGen) Close() {} +func (s *subdividedGeometriesGen) Close(_ context.Context) {} func (s *subdividedGeometriesGen) Start(_ context.Context, _ *kv.Txn) error { s.curr = -1 diff --git a/pkg/sql/sem/builtins/show_create_all_tables_builtin.go b/pkg/sql/sem/builtins/show_create_all_tables_builtin.go index cacde442c115..5d1b5cb32c5b 100644 --- a/pkg/sql/sem/builtins/show_create_all_tables_builtin.go +++ b/pkg/sql/sem/builtins/show_create_all_tables_builtin.go @@ -11,300 +11,296 @@ package builtins import ( + "context" "fmt" "sort" - "strings" - "time" + "unsafe" + "github.com/cockroachdb/cockroach/pkg/kv" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sessiondata" "github.com/cockroachdb/cockroach/pkg/sql/sqlutil" - "github.com/cockroachdb/cockroach/pkg/util/timeutil" + "github.com/cockroachdb/cockroach/pkg/util/mon" "github.com/cockroachdb/errors" ) -type tableMetadata struct { - ID int64 - name *tree.TableName - createStmt string - dependsOn []int64 - alter []string - validate []string -} +const sizeOfInt64 = int64(unsafe.Sizeof(int64(0))) -type tableWithSchema struct { - schema string - table string -} +// mapEntryOverhead is a guess on how much space (in bytes) +// each item added to a map takes. +// More explanation in cockroach/pkg/sql/rowexec/aggregator.go variable +// hashAggregatorSizeOfBucketsItem +const mapEntryOverhead = 64 + +// alterAddFKStatements represents the column name for alter_statements in +// crdb_internal.create_statements. +const alterAddFKStatements = "alter_statements" -// showCreateAllTablesBuiltin presents CREATE TABLE statements followed by -// ALTER TABLE statements for a database. The CREATE TABLE / ALTER TABLE -// statements are sorted such that the tables are ordered topologically to -// account for dependencies and references. The output can be used to -// recreate a database. -// The reason for ALTER TABLE statements coming after all CREATE TABLE -// statements is that we want to add constraints such as foreign keys -// after all the referenced tables are created. -func showCreateAllTablesBuiltin(evalCtx *tree.EvalContext, arg tree.Datums) (tree.Datum, error) { - mds, err := getMetadataForTablesInDB(evalCtx, arg) +// alterValidateFKStatements represents the column name for validate_statements in +// crdb_internal.create_statements. +const alterValidateFKStatements = "validate_statements" + +// foreignKeyValidationWarning is a warning letting the user know that +// the validate foreign key constraints may fail. +const foreignKeyValidationWarning = "-- Validate foreign key constraints. These can fail if there was unvalidated data during the SHOW CREATE ALL TABLES" + +// getTopologicallySortedTableIDs returns the set of table ids sorted +// first by table id, then topologically ordered such that dependencies are +// ordered before tables that depend on them. (ie, sequences will appear before +// the table that uses the sequence). +// The tables are sorted by table id first to guarantee stable ordering. +func getTopologicallySortedTableIDs( + ctx context.Context, + ie sqlutil.InternalExecutor, + txn *kv.Txn, + dbName string, + ts string, + acc *mon.BoundAccount, +) ([]int64, error) { + ids, err := getTableIDs(ctx, ie, txn, ts, dbName, acc) if err != nil { return nil, err } - if len(mds) == 0 { - return tree.NewDString(""), nil + if len(ids) == 0 { + return nil, nil } - byID := make(map[int64]tableMetadata) - for _, md := range mds { - byID[md.ID] = md + sizeOfMap := int64(0) + // dependsOnIDs maps an id of a table to the ids it depends on. + // We perform the topological sort on dependsOnIDs instead of on the + // byID map to reduce memory usage. + dependsOnIDs := make(map[int64][]int64) + for _, tid := range ids { + query := fmt.Sprintf(` + SELECT dependson_id + FROM %s.crdb_internal.backward_dependencies + AS OF SYSTEM TIME %s + WHERE descriptor_id = $1 + `, dbName, ts) + it, err := ie.QueryIteratorEx( + ctx, + "crdb_internal.show_create_all_tables", + txn, + sessiondata.NoSessionDataOverride, + query, + tid, + ) + if err != nil { + return nil, err + } + + var refs []int64 + var ok bool + for ok, err = it.Next(ctx); ok; ok, err = it.Next(ctx) { + id := tree.MustBeDInt(it.Cur()[0]) + refs = append(refs, int64(id)) + } + if err != nil { + return nil, err + } + + // Account for memory of map. + sizeOfKeyValue := int64(unsafe.Sizeof(tid)) + int64(len(refs))*sizeOfInt64 + sizeOfMap += sizeOfKeyValue + mapEntryOverhead + if err = acc.Grow(ctx, sizeOfKeyValue+mapEntryOverhead); err != nil { + return nil, err + } + + dependsOnIDs[tid] = refs } - // First sort by name to guarantee stable output. - sort.Slice(mds, func(i, j int) bool { - return mds[i].name.String() < mds[j].name.String() + // First sort by ids to guarantee stable output. + sort.Slice(ids, func(i, j int) bool { + return ids[i] < ids[j] }) // Collect transitive dependencies in topological order into collected. // The topological order is essential here since it captures dependencies // for views and sequences creation, hence simple alphabetical sort won't // be enough. - var collected []int64 - seen := make(map[int64]bool) - for _, md := range mds { - collect(md.ID, byID, seen, &collected) - } - // collectOrder maps a table ID to its collection index. This is needed - // instead of just using range over collected because collected may contain - // table IDs not present in the dump spec. It is simpler to sort mds correctly - // to skip over these referenced-but-not-dumped tables. - collectOrder := make(map[int64]int) - for i, id := range collected { - collectOrder[id] = i - } - - // Second sort dumped tables by dependency order. - sort.SliceStable(mds, func(i, j int) bool { - return collectOrder[mds[i].ID] < collectOrder[mds[j].ID] - }) - - var out []string - for _, md := range mds { - out = append(out, md.createStmt+";\n") - } + var topologicallyOrderedIDs []int64 - hasRefs := false - for _, md := range mds { - for _, alter := range md.alter { - if !hasRefs { - hasRefs = true - } - out = append(out, fmt.Sprintf("%s;\n", alter)) - } - } - if hasRefs { - const alterValidateMessage = `-- Validate foreign key constraints. These can fail if there was unvalidated data during the SHOW CREATE ALL TABLES` - out = append(out, alterValidateMessage+"\n") - for _, md := range mds { - for _, validate := range md.validate { - out = append(out, fmt.Sprintf("%s;\n", validate)) - - } - } - } - - result := tree.NewDString(strings.Join(out, "")) - return result, nil -} - -// getMetadataForTablesInDB finds all the table names in a given database and -// populates the tableMetadata information for all the tables. -func getMetadataForTablesInDB(evalCtx *tree.EvalContext, arg tree.Datums) ([]tableMetadata, error) { - tsI, err := tree.MakeDTimestamp(timeutil.Now(), time.Microsecond) - if err != nil { - return nil, err - } - ts := tsI.String() - dbName := string(tree.MustBeDString(arg[0])) - tableMDs, err := getTableNames(evalCtx, dbName, ts) - if err != nil { + // The sort relies on creating a new array for the ids. + if err = acc.Grow(ctx, int64(len(ids))*sizeOfInt64); err != nil { return nil, err } - - mds := make([]tableMetadata, len(tableMDs)) - for i, dumpTable := range tableMDs { - tableMD, err := getTableMetadata(evalCtx, dbName, dumpTable, ts) - if err != nil { + seen := make(map[int64]struct{}) + for _, id := range ids { + if err := topologicalSort( + ctx, id, dependsOnIDs, seen, &topologicallyOrderedIDs, acc, + ); err != nil { return nil, err } - mds[i] = tableMD } - return mds, nil + // Clear memory used by the seen map. + sizeOfSeen := len(seen) + seen = nil + acc.Shrink(ctx, int64(sizeOfSeen)*(sizeOfInt64+mapEntryOverhead)) + + // The lengths should match. This is also important for memory accounting, + // the two arrays should have the same length. + if len(ids) != len(topologicallyOrderedIDs) { + return nil, errors.AssertionFailedf("show_create_all_tables_builtin failed. "+ + "len(ids):% d not equal to len(topologicallySortedIDs): %d", + len(ids), len(topologicallyOrderedIDs)) + } + + // Shrink the memory we used for the original ids array. + acc.Shrink(ctx, int64(len(ids))*sizeOfInt64) + acc.Shrink(ctx, sizeOfMap) + return topologicallyOrderedIDs, nil } -// getTableMetadata populates the metadata for a given table by querying -// crdb_internal.create_statements. -func getTableMetadata( - evalCtx *tree.EvalContext, dbName string, table tableWithSchema, ts string, -) (tableMetadata, error) { - tn := tree.MakeTableNameWithSchema(tree.Name(dbName), tree.Name(table.schema), tree.Name(table.table)) - // Fetch table ID. +// getTableIDs returns the set of table ids from +// crdb_internal.show_create_all_tables for a specified database. +func getTableIDs( + ctx context.Context, + ie sqlutil.InternalExecutor, + txn *kv.Txn, + ts string, + dbName string, + acc *mon.BoundAccount, +) ([]int64, error) { query := fmt.Sprintf(` - SELECT - schema_name, - descriptor_id, - create_nofks, - alter_statements, - validate_statements + SELECT descriptor_id FROM %s.crdb_internal.create_statements AS OF SYSTEM TIME %s - WHERE database_name = $1 - AND schema_name = $2 - AND descriptor_name = $3 - `, dbName, ts) - ie := evalCtx.InternalExecutor.(sqlutil.InternalExecutor) - vals, err := ie.QueryRowEx( - evalCtx.Context, + WHERE database_name = $1 + AND is_virtual = FALSE + AND is_temporary = FALSE + `, dbName, ts) + it, err := ie.QueryIteratorEx( + ctx, "crdb_internal.show_create_all_tables", - evalCtx.Txn, + txn, sessiondata.NoSessionDataOverride, query, dbName, - table.schema, - table.table, ) if err != nil { - return tableMetadata{}, err + return nil, err } - if len(vals) == 0 { - return tableMetadata{}, nil + var tableIDs []int64 + + var ok bool + for ok, err = it.Next(ctx); ok; ok, err = it.Next(ctx) { + tid := tree.MustBeDInt(it.Cur()[0]) + + tableIDs = append(tableIDs, int64(tid)) + if err = acc.Grow(ctx, int64(unsafe.Sizeof(tid))); err != nil { + return nil, err + } + } + if err != nil { + return tableIDs, err } - // Check the schema to disallow dumping temp tables, views and sequences. This - // will only be triggered if a user explicitly specifies a temp construct as - // one of the arguments to the `cockroach dump` command. When no table names - // are specified on the CLI, we ignore temp tables at the stage where we read - // all table names in getTableNames. - schemaName := string(tree.MustBeDString(vals[0])) - if strings.HasPrefix(schemaName, sessiondata.PgTempSchemaName) { - return tableMetadata{}, errors.Newf("cannot dump temp table %s", tn.String()) + return tableIDs, nil +} + +// topologicalSort sorts transitive dependencies in topological order into +// collected. +// The topological order is essential here since it captures dependencies +// for views and sequences creation, hence simple alphabetical sort won't +// be enough. +func topologicalSort( + ctx context.Context, + tid int64, + dependsOnIDs map[int64][]int64, + seen map[int64]struct{}, + collected *[]int64, + acc *mon.BoundAccount, +) error { + // has this table already been collected previously? + // We need this check because a table could be traversed to multiple times + // if it is referenced. + // For example, if a table references itself, without this check + // collect would infinitely recurse. + if _, isPresent := seen[tid]; isPresent { + return nil + } + + // Account for memory of map. + // The key value entry into the map is only the memory of an int64 since + // the value stuct{}{} uses no memory. + if err := acc.Grow(ctx, sizeOfInt64+mapEntryOverhead); err != nil { + return err + } + seen[tid] = struct{}{} + for _, dep := range dependsOnIDs[tid] { + if err := topologicalSort(ctx, dep, dependsOnIDs, seen, collected, acc); err != nil { + return err + } } - id := int64(tree.MustBeDInt(vals[1])) - createStatement := string(tree.MustBeDString(vals[2])) - alterStatements := extractArray(vals[3]) - validateStatements := extractArray(vals[4]) + if err := acc.Grow(ctx, int64(unsafe.Sizeof(tid))); err != nil { + return err + } + *collected = append(*collected, tid) - // Get dependencies. - query = fmt.Sprintf(` - SELECT dependson_id - FROM %s.crdb_internal.backward_dependencies + return nil +} + +// getCreateStatement gets the create statement to recreate a table (ignoring fks) +// for a given table id in a database. +func getCreateStatement( + ctx context.Context, ie sqlutil.InternalExecutor, txn *kv.Txn, id int64, ts string, dbName string, +) (tree.Datum, error) { + query := fmt.Sprintf(` + SELECT + create_nofks + FROM %s.crdb_internal.create_statements AS OF SYSTEM TIME %s WHERE descriptor_id = $1 - `, dbName, ts) - it, err := ie.QueryIteratorEx( - evalCtx.Context, + `, dbName, ts) + row, err := ie.QueryRowEx( + ctx, "crdb_internal.show_create_all_tables", - evalCtx.Txn, + txn, sessiondata.NoSessionDataOverride, query, id, ) - if err != nil { - return tableMetadata{}, err - } - var refs []int64 - var ok bool - for ok, err = it.Next(evalCtx.Context); ok; ok, err = it.Next(evalCtx.Context) { - id := tree.MustBeDInt(it.Cur()[0]) - refs = append(refs, int64(id)) - } if err != nil { - return tableMetadata{}, err - } - md := tableMetadata{ - ID: id, - name: &tn, - createStmt: createStatement, - dependsOn: refs, - alter: alterStatements, - validate: validateStatements, - } - - return md, nil -} - -// extractArray ensures that a tree.Datum is a DArray and converts -// the DArray into a list of strings. -func extractArray(val tree.Datum) []string { - arr := tree.MustBeDArray(val) - res := make([]string, len(arr.Array)) - for i, v := range arr.Array { - res[i] = string(*v.(*tree.DString)) + return nil, err } - return res + return row[0], nil } -// getTableNames retrieves all tables names in the given database. Following -// pg_dump, we ignore all descriptors which are part of the temp schema. This -// includes tables, views and sequences. -func getTableNames(evalCtx *tree.EvalContext, dbName string, ts string) ([]tableWithSchema, error) { +// getAlterStatements gets the set of alter statements that add and validate +// foreign keys for a given table id in a database. +func getAlterStatements( + ctx context.Context, + ie sqlutil.InternalExecutor, + txn *kv.Txn, + id int64, + ts string, + dbName string, + statementType string, +) (tree.Datum, error) { query := fmt.Sprintf(` - SELECT schema_name, descriptor_name - FROM "".crdb_internal.create_statements + SELECT + %s + FROM %s.crdb_internal.create_statements AS OF SYSTEM TIME %s - WHERE database_name = $1 AND schema_name NOT LIKE $2 - `, ts) - ie := evalCtx.InternalExecutor.(sqlutil.InternalExecutor) - it, err := ie.QueryIteratorEx( - evalCtx.Ctx(), + WHERE descriptor_id = $1 + `, statementType, dbName, ts) + row, err := ie.QueryRowEx( + ctx, "crdb_internal.show_create_all_tables", - evalCtx.Txn, + txn, sessiondata.NoSessionDataOverride, query, - dbName, - sessiondata.PgTempSchemaName+"%", + id, ) - if err != nil { - return nil, err - } - - var tableNames []tableWithSchema - var ok bool - for ok, err = it.Next(evalCtx.Context); ok; ok, err = it.Next(evalCtx.Context) { - schema := string(tree.MustBeDString(it.Cur()[0])) - table := string(tree.MustBeDString(it.Cur()[1])) - - tableNames = append(tableNames, tableWithSchema{table: table, schema: schema}) - } if err != nil { - return tableNames, err + return nil, err } - return tableNames, nil -} - -// collect maps a table id to it's tableMetadata and ensures tables are only -// mapped once. -func collect(tid int64, byID map[int64]tableMetadata, seen map[int64]bool, collected *[]int64) { - // has this table already been collected previously? - // We need this check because a table could be multiple times - // if it is referenced. - // For example, if a table references itself, without this check - // collect would infinitely recurse. - if seen[tid] { - return - } - // no: mark it as seen. - seen[tid] = true - for _, dep := range byID[tid].dependsOn { - // depth-first collection of dependencies - collect(dep, byID, seen, collected) - } - // Only add it after its dependencies. - *collected = append(*collected, tid) + return row[0], nil } diff --git a/pkg/sql/sem/tree/eval.go b/pkg/sql/sem/tree/eval.go index f0fd91b393cb..e2987cff7343 100644 --- a/pkg/sql/sem/tree/eval.go +++ b/pkg/sql/sem/tree/eval.go @@ -5408,7 +5408,7 @@ func (c *CallbackValueGenerator) Values() (Datums, error) { } // Close is part of the ValueGenerator interface. -func (c *CallbackValueGenerator) Close() {} +func (c *CallbackValueGenerator) Close(_ context.Context) {} // Sqrt returns the square root of x. func Sqrt(x float64) (*DFloat, error) { diff --git a/pkg/sql/sem/tree/generators.go b/pkg/sql/sem/tree/generators.go index bc7fc9fbd5de..5ca4e23348db 100644 --- a/pkg/sql/sem/tree/generators.go +++ b/pkg/sql/sem/tree/generators.go @@ -59,7 +59,7 @@ type ValueGenerator interface { // Close must be called after Start() before disposing of the // ValueGenerator. It does not need to be called if Start() has not // been called yet. It must not be called in-between restarts. - Close() + Close(ctx context.Context) } // GeneratorFactory is the type of constructor functions for