From 6d0d912713299d38eafad93b36cd550bb82457cc Mon Sep 17 00:00:00 2001 From: Marius Posta Date: Thu, 7 Apr 2022 19:57:23 +0000 Subject: [PATCH 1/2] scstage: rename package import For no good reason, scgraph was imported as scgraph2. This commit fixes this. Release note: None --- .../schemachanger/scplan/internal/scstage/stage.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/sql/schemachanger/scplan/internal/scstage/stage.go b/pkg/sql/schemachanger/scplan/internal/scstage/stage.go index 2c51cb9e1a0d..fb3ec79fb838 100644 --- a/pkg/sql/schemachanger/scplan/internal/scstage/stage.go +++ b/pkg/sql/schemachanger/scplan/internal/scstage/stage.go @@ -15,7 +15,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scop" "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scpb" - scgraph2 "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scplan/internal/scgraph" + "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scplan/internal/scgraph" "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/screl" "github.com/cockroachdb/errors" ) @@ -75,7 +75,7 @@ func (s Stage) String() string { } // ValidateStages checks that the plan is valid. -func ValidateStages(ts scpb.TargetState, stages []Stage, g *scgraph2.Graph) error { +func ValidateStages(ts scpb.TargetState, stages []Stage, g *scgraph.Graph) error { if len(stages) == 0 { return nil } @@ -141,9 +141,9 @@ func validateAdjacentStagesStates(previous, next Stage) error { return nil } -func validateStageSubgraph(ts scpb.TargetState, stage Stage, g *scgraph2.Graph) error { +func validateStageSubgraph(ts scpb.TargetState, stage Stage, g *scgraph.Graph) error { // Transform the ops in a non-repeating sequence of their original op edges. - var queue []*scgraph2.OpEdge + var queue []*scgraph.OpEdge for _, op := range stage.EdgeOps { oe := g.GetOpEdgeFromOp(op) if oe == nil { @@ -169,8 +169,8 @@ func validateStageSubgraph(ts scpb.TargetState, stage Stage, g *scgraph2.Graph) current[i] = n } { - edgesTo := make(map[*screl.Node][]scgraph2.Edge, g.Order()) - _ = g.ForEachEdge(func(e scgraph2.Edge) error { + edgesTo := make(map[*screl.Node][]scgraph.Edge, g.Order()) + _ = g.ForEachEdge(func(e scgraph.Edge) error { edgesTo[e.To()] = append(edgesTo[e.To()], e) return nil }) @@ -211,7 +211,7 @@ func validateStageSubgraph(ts scpb.TargetState, stage Stage, g *scgraph2.Graph) // Prevent making progress on this target if there are unmet dependencies. var hasUnmetDeps bool - if err := g.ForEachDepEdgeTo(oe.To(), func(de *scgraph2.DepEdge) error { + if err := g.ForEachDepEdgeTo(oe.To(), func(de *scgraph.DepEdge) error { hasUnmetDeps = hasUnmetDeps || !fulfilled[de.From()] return nil }); err != nil { From 00a868c08a2a5c71883913768084165c5cd31186 Mon Sep 17 00:00:00 2001 From: Marius Posta Date: Fri, 8 Apr 2022 15:50:21 +0000 Subject: [PATCH 2/2] sql: improve EXPLAIN (DDL) output Previously, the EXPLAIN (DDL, STAGES) and EXPLAIN (DDL, DEPS) statements were printing graphviz URLs to help engineers during the development of the declarative schema changer. Now that the declarative schema changer is going live, the audience for EXPLAIN (DDL) changes to become the users. The graphviz URLs were, at any rate, not very useful for schema changes involving many objects, due to limitations in the rendering engine. This commit gets rid of the STAGES and DEPS flags to provide a unified plan which is similar to the data-driven test output in the scplan package. The VERBOSE flag can now be used to render a superset of the information in the default rendering. Touches #77158. Release note (sql change): EXPLAIN (DDL), when invoked on statements supported by the declarative schema changer, prints a plan of what the schema changer will do. This can be useful for the user to anticipate the complexity of a schema change (anything involving Backfill or Validation operations might be slow to run). This can be useful for troubleshooting. EXPLAIN (DDL, VERBOSE) produces a more detailed plan. --- pkg/sql/explain_ddl.go | 18 +- .../testdata/logic_test/new_schema_changer | 619 +++++++++++++++++- pkg/sql/opt/optbuilder/explain.go | 7 +- pkg/sql/schemachanger/schemachanger_test.go | 1 - pkg/sql/schemachanger/scplan/BUILD.bazel | 9 +- .../scplan/internal/rules/registry.go | 9 +- .../scplan/internal/scgraph/graph.go | 35 +- .../scplan/internal/scgraphviz/graphviz.go | 27 - pkg/sql/schemachanger/scplan/plan.go | 16 - pkg/sql/schemachanger/scplan/plan_explain.go | 340 ++++++++++ pkg/sql/sem/tree/explain.go | 4 - pkg/sql/sqltelemetry/planning.go | 7 +- 12 files changed, 1005 insertions(+), 87 deletions(-) create mode 100644 pkg/sql/schemachanger/scplan/plan_explain.go diff --git a/pkg/sql/explain_ddl.go b/pkg/sql/explain_ddl.go index 2f7973a11175..2d7da94b3cf4 100644 --- a/pkg/sql/explain_ddl.go +++ b/pkg/sql/explain_ddl.go @@ -72,18 +72,18 @@ func (n *explainDDLNode) startExec(params runParams) error { if err != nil { return errors.WithAssertionFailure(err) } - var vizURL string - if n.options.Flags[tree.ExplainFlagDeps] { - if vizURL, err = sc.DependenciesURL(); err != nil { - return errors.WithAssertionFailure(err) - } + var info string + if n.options.Flags[tree.ExplainFlagVerbose] { + info, err = sc.ExplainVerbose() } else { - if vizURL, err = sc.StagesURL(); err != nil { - return errors.WithAssertionFailure(err) - } + info, err = sc.ExplainCompact() + } + + if err != nil { + return errors.WithAssertionFailure(err) } n.values = tree.Datums{ - tree.NewDString(vizURL), + tree.NewDString(info), } return nil } diff --git a/pkg/sql/logictest/testdata/logic_test/new_schema_changer b/pkg/sql/logictest/testdata/logic_test/new_schema_changer index 6f7d9ccb3830..a4e229f2d60b 100644 --- a/pkg/sql/logictest/testdata/logic_test/new_schema_changer +++ b/pkg/sql/logictest/testdata/logic_test/new_schema_changer @@ -14,11 +14,282 @@ EXPLAIN (DDL) ALTER TABLE foo ADD COLUMN j INT statement ok SET use_declarative_schema_changer = 'unsafe' -statement ok -EXPLAIN (DDL) ALTER TABLE foo ADD COLUMN j INT - -statement ok -EXPLAIN (DDL, DEPS) ALTER TABLE foo ADD COLUMN j INT +query T +EXPLAIN (DDL, VERBOSE) ALTER TABLE foo ADD COLUMN j INT +---- +• Schema change plan for ALTER TABLE ‹test›.public.‹foo› ADD COLUMN ‹j› INT8; +│ +├── • PreCommitPhase +│ │ +│ └── • Stage 1 of 1 in PreCommitPhase +│ │ +│ ├── • 3 elements transitioning toward PUBLIC +│ │ │ +│ │ ├── • Column:{DescID: 106, ColumnID: 2} +│ │ │ ABSENT → DELETE_ONLY +│ │ │ +│ │ ├── • ColumnType:{DescID: 106, ColumnFamilyID: 0, ColumnID: 2} +│ │ │ │ ABSENT → PUBLIC +│ │ │ │ +│ │ │ └── • SameStagePrecedence dependency from DELETE_ONLY Column:{DescID: 106, ColumnID: 2} +│ │ │ rule: "column type set right after column existence" +│ │ │ +│ │ └── • PrimaryIndex:{DescID: 106, IndexID: 2} +│ │ │ ABSENT → DELETE_ONLY +│ │ │ +│ │ └── • Precedence dependency from DELETE_ONLY Column:{DescID: 106, ColumnID: 2} +│ │ rule: "column existence precedes index existence" +│ │ +│ └── • 6 Mutation operations +│ │ +│ ├── • MakeAddedColumnDeleteOnly +│ │ Column: +│ │ ColumnID: 2 +│ │ TableID: 106 +│ │ +│ ├── • LogEvent +│ │ Authorization: +│ │ UserName: root +│ │ Element: +│ │ Column: +│ │ columnId: 2 +│ │ tableId: 106 +│ │ Statement: ALTER TABLE ‹test›.public.‹foo› ADD COLUMN ‹j› INT8 +│ │ StatementTag: ALTER TABLE +│ │ TargetMetadata: +│ │ SourceElementID: 1 +│ │ SubWorkID: 1 +│ │ TargetStatus: 2 +│ │ +│ ├── • SetAddedColumnType +│ │ ColumnType: +│ │ ColumnID: 2 +│ │ IsNullable: true +│ │ TableID: 106 +│ │ TypeT: +│ │ Type: +│ │ family: IntFamily +│ │ oid: 20 +│ │ width: 64 +│ │ +│ ├── • MakeAddedIndexDeleteOnly +│ │ Index: +│ │ IndexID: 2 +│ │ IsUnique: true +│ │ KeyColumnDirections: +│ │ - 0 +│ │ KeyColumnIDs: +│ │ - 1 +│ │ SourceIndexID: 1 +│ │ StoringColumnIDs: +│ │ - 2 +│ │ TableID: 106 +│ │ +│ ├── • SetJobStateOnDescriptor +│ │ DescriptorID: 106 +│ │ Initialize: true +│ │ +│ └── • CreateSchemaChangerJob +│ Authorization: +│ UserName: root +│ DescriptorIDs: +│ - 106 +│ JobID: 1 +│ Statements: +│ - statement: ALTER TABLE foo ADD COLUMN j INT8 +│ redactedstatement: ALTER TABLE ‹test›.public.‹foo› ADD COLUMN ‹j› INT8 +│ statementtag: ALTER TABLE +│ +├── • PostCommitPhase +│ │ +│ ├── • Stage 1 of 4 in PostCommitPhase +│ │ │ +│ │ ├── • 2 elements transitioning toward PUBLIC +│ │ │ │ +│ │ │ ├── • Column:{DescID: 106, ColumnID: 2} +│ │ │ │ │ DELETE_ONLY → WRITE_ONLY +│ │ │ │ │ +│ │ │ │ └── • Precedence dependency from WRITE_ONLY PrimaryIndex:{DescID: 106, IndexID: 2} +│ │ │ │ rule: "column depends on primary index" +│ │ │ │ +│ │ │ └── • PrimaryIndex:{DescID: 106, IndexID: 2} +│ │ │ DELETE_ONLY → WRITE_ONLY +│ │ │ +│ │ └── • 4 Mutation operations +│ │ │ +│ │ ├── • MakeAddedIndexDeleteAndWriteOnly +│ │ │ IndexID: 2 +│ │ │ TableID: 106 +│ │ │ +│ │ ├── • MakeAddedColumnDeleteAndWriteOnly +│ │ │ ColumnID: 2 +│ │ │ TableID: 106 +│ │ │ +│ │ ├── • SetJobStateOnDescriptor +│ │ │ DescriptorID: 106 +│ │ │ +│ │ └── • UpdateSchemaChangerJob +│ │ JobID: 1 +│ │ +│ ├── • Stage 2 of 4 in PostCommitPhase +│ │ │ +│ │ ├── • 1 element transitioning toward PUBLIC +│ │ │ │ +│ │ │ └── • PrimaryIndex:{DescID: 106, IndexID: 2} +│ │ │ WRITE_ONLY → BACKFILLED +│ │ │ +│ │ └── • 1 Backfill operation +│ │ │ +│ │ └── • BackfillIndex +│ │ IndexID: 2 +│ │ SourceIndexID: 1 +│ │ TableID: 106 +│ │ +│ ├── • Stage 3 of 4 in PostCommitPhase +│ │ │ +│ │ ├── • 1 element transitioning toward PUBLIC +│ │ │ │ +│ │ │ └── • PrimaryIndex:{DescID: 106, IndexID: 2} +│ │ │ BACKFILLED → VALIDATED +│ │ │ +│ │ └── • 1 Validation operation +│ │ │ +│ │ └── • ValidateUniqueIndex +│ │ IndexID: 2 +│ │ TableID: 106 +│ │ +│ └── • Stage 4 of 4 in PostCommitPhase +│ │ +│ ├── • 2 elements transitioning toward ABSENT +│ │ │ +│ │ ├── • PrimaryIndex:{DescID: 106, IndexID: 1} +│ │ │ PUBLIC → VALIDATED +│ │ │ +│ │ └── • IndexName:{DescID: 106, Name: foo_pkey, IndexID: 1} +│ │ │ PUBLIC → ABSENT +│ │ │ +│ │ └── • Precedence dependency from VALIDATED PrimaryIndex:{DescID: 106, IndexID: 1} +│ │ rule: "dependents removed after index no longer public" +│ │ +│ ├── • 4 elements transitioning toward PUBLIC +│ │ │ +│ │ ├── • Column:{DescID: 106, ColumnID: 2} +│ │ │ │ WRITE_ONLY → PUBLIC +│ │ │ │ +│ │ │ ├── • SameStagePrecedence dependency from PUBLIC ColumnName:{DescID: 106, Name: j, ColumnID: 2} +│ │ │ │ rule: "column named right before column becomes public" +│ │ │ │ +│ │ │ └── • Precedence dependency from PUBLIC PrimaryIndex:{DescID: 106, IndexID: 2} +│ │ │ rule: "column depends on primary index" +│ │ │ +│ │ ├── • ColumnName:{DescID: 106, Name: j, ColumnID: 2} +│ │ │ │ ABSENT → PUBLIC +│ │ │ │ +│ │ │ └── • Precedence dependency from DELETE_ONLY Column:{DescID: 106, ColumnID: 2} +│ │ │ rule: "column existence precedes column dependents" +│ │ │ +│ │ ├── • PrimaryIndex:{DescID: 106, IndexID: 2} +│ │ │ │ VALIDATED → PUBLIC +│ │ │ │ +│ │ │ ├── • SameStagePrecedence dependency from VALIDATED PrimaryIndex:{DescID: 106, IndexID: 1} +│ │ │ │ rule: "primary index swap" +│ │ │ │ +│ │ │ └── • SameStagePrecedence dependency from PUBLIC IndexName:{DescID: 106, Name: foo_pkey, IndexID: 2} +│ │ │ rule: "index named right before index becomes public" +│ │ │ +│ │ └── • IndexName:{DescID: 106, Name: foo_pkey, IndexID: 2} +│ │ │ ABSENT → PUBLIC +│ │ │ +│ │ └── • Precedence dependency from DELETE_ONLY PrimaryIndex:{DescID: 106, IndexID: 2} +│ │ rule: "index existence precedes index dependents" +│ │ +│ └── • 8 Mutation operations +│ │ +│ ├── • MakeDroppedPrimaryIndexDeleteAndWriteOnly +│ │ IndexID: 1 +│ │ TableID: 106 +│ │ +│ ├── • SetIndexName +│ │ IndexID: 1 +│ │ Name: crdb_internal_index_1_name_placeholder +│ │ TableID: 106 +│ │ +│ ├── • SetIndexName +│ │ IndexID: 2 +│ │ Name: foo_pkey +│ │ TableID: 106 +│ │ +│ ├── • MakeAddedPrimaryIndexPublic +│ │ IndexID: 2 +│ │ TableID: 106 +│ │ +│ ├── • SetColumnName +│ │ ColumnID: 2 +│ │ Name: j +│ │ TableID: 106 +│ │ +│ ├── • MakeColumnPublic +│ │ ColumnID: 2 +│ │ TableID: 106 +│ │ +│ ├── • SetJobStateOnDescriptor +│ │ DescriptorID: 106 +│ │ +│ └── • UpdateSchemaChangerJob +│ IsNonCancelable: true +│ JobID: 1 +│ +└── • PostCommitNonRevertiblePhase + │ + ├── • Stage 1 of 2 in PostCommitNonRevertiblePhase + │ │ + │ ├── • 1 element transitioning toward ABSENT + │ │ │ + │ │ └── • PrimaryIndex:{DescID: 106, IndexID: 1} + │ │ VALIDATED → DELETE_ONLY + │ │ + │ └── • 3 Mutation operations + │ │ + │ ├── • MakeDroppedIndexDeleteOnly + │ │ IndexID: 1 + │ │ TableID: 106 + │ │ + │ ├── • SetJobStateOnDescriptor + │ │ DescriptorID: 106 + │ │ + │ └── • UpdateSchemaChangerJob + │ IsNonCancelable: true + │ JobID: 1 + │ + └── • Stage 2 of 2 in PostCommitNonRevertiblePhase + │ + ├── • 1 element transitioning toward ABSENT + │ │ + │ └── • PrimaryIndex:{DescID: 106, IndexID: 1} + │ │ DELETE_ONLY → ABSENT + │ │ + │ └── • Precedence dependency from ABSENT IndexName:{DescID: 106, Name: foo_pkey, IndexID: 1} + │ rule: "dependents removed before index" + │ + └── • 4 Mutation operations + │ + ├── • CreateGcJobForIndex + │ IndexID: 1 + │ StatementForDropJob: + │ Statement: ALTER TABLE test.public.foo ADD COLUMN j INT8 + │ TableID: 106 + │ + ├── • MakeIndexAbsent + │ IndexID: 1 + │ TableID: 106 + │ + ├── • RemoveJobStateFromDescriptor + │ DescriptorID: 106 + │ JobID: 1 + │ + └── • UpdateSchemaChangerJob + IsNonCancelable: true + JobID: 1 statement ok ALTER TABLE foo ADD COLUMN j INT @@ -207,8 +478,199 @@ CREATE TABLE blog_posts2 (id INT PRIMARY KEY, val int DEFAULT nextval('sq1'), ti statement error pq: cannot drop sequence sq1 because other objects depend on it EXPLAIN (DDL) DROP SEQUENCE sq1; -statement ok -EXPLAIN (DDL) DROP SEQUENCE sq1 CASCADE; +query T +EXPLAIN (DDL, VERBOSE) DROP SEQUENCE sq1 CASCADE; +---- +• Schema change plan for DROP SEQUENCE ‹test›.public.‹sq1› CASCADE; +│ +├── • StatementPhase +│ │ +│ └── • Stage 1 of 1 in StatementPhase +│ │ +│ ├── • 1 element transitioning toward ABSENT +│ │ │ +│ │ └── • Sequence:{DescID: 113} +│ │ PUBLIC → TXN_DROPPED +│ │ +│ └── • 1 Mutation operation +│ │ +│ └── • MarkDescriptorAsDroppedSynthetically +│ DescID: 113 +│ +├── • PreCommitPhase +│ │ +│ └── • Stage 1 of 1 in PreCommitPhase +│ │ +│ ├── • 9 elements transitioning toward ABSENT +│ │ │ +│ │ ├── • Namespace:{DescID: 113, Name: sq1, ReferencedDescID: 104} +│ │ │ PUBLIC → ABSENT +│ │ │ +│ │ ├── • Owner:{DescID: 113} +│ │ │ │ PUBLIC → ABSENT +│ │ │ │ +│ │ │ └── • skip PUBLIC → ABSENT operations +│ │ │ rule: "skip element removal ops on descriptor drop" +│ │ │ +│ │ ├── • UserPrivileges:{DescID: 113, Name: admin} +│ │ │ │ PUBLIC → ABSENT +│ │ │ │ +│ │ │ └── • skip PUBLIC → ABSENT operations +│ │ │ rule: "skip element removal ops on descriptor drop" +│ │ │ +│ │ ├── • UserPrivileges:{DescID: 113, Name: root} +│ │ │ │ PUBLIC → ABSENT +│ │ │ │ +│ │ │ └── • skip PUBLIC → ABSENT operations +│ │ │ rule: "skip element removal ops on descriptor drop" +│ │ │ +│ │ ├── • Sequence:{DescID: 113} +│ │ │ │ TXN_DROPPED → DROPPED +│ │ │ │ +│ │ │ ├── • Precedence dependency from ABSENT Namespace:{DescID: 113, Name: sq1, ReferencedDescID: 104} +│ │ │ │ rule: "dependent element removal before descriptor drop" +│ │ │ │ +│ │ │ ├── • Precedence dependency from ABSENT Owner:{DescID: 113} +│ │ │ │ rule: "dependent element removal before descriptor drop" +│ │ │ │ +│ │ │ ├── • Precedence dependency from ABSENT UserPrivileges:{DescID: 113, Name: admin} +│ │ │ │ rule: "dependent element removal before descriptor drop" +│ │ │ │ +│ │ │ ├── • Precedence dependency from ABSENT UserPrivileges:{DescID: 113, Name: root} +│ │ │ │ rule: "dependent element removal before descriptor drop" +│ │ │ │ +│ │ │ ├── • Precedence dependency from ABSENT ObjectParent:{DescID: 113, ReferencedDescID: 105} +│ │ │ │ rule: "dependent element removal before descriptor drop" +│ │ │ │ +│ │ │ ├── • Precedence dependency from ABSENT TableComment:{DescID: 113} +│ │ │ │ rule: "dependent element removal before descriptor drop" +│ │ │ │ +│ │ │ ├── • Precedence dependency from ABSENT ColumnDefaultExpression:{DescID: 114, ColumnID: 2} +│ │ │ │ rule: "column DEFAULT removed before dropping dependent types and sequences" +│ │ │ │ +│ │ │ └── • Precedence dependency from ABSENT ColumnDefaultExpression:{DescID: 115, ColumnID: 2} +│ │ │ rule: "column DEFAULT removed before dropping dependent types and sequences" +│ │ │ +│ │ ├── • ObjectParent:{DescID: 113, ReferencedDescID: 105} +│ │ │ PUBLIC → ABSENT +│ │ │ +│ │ ├── • TableComment:{DescID: 113} +│ │ │ │ PUBLIC → ABSENT +│ │ │ │ +│ │ │ └── • skip PUBLIC → ABSENT operations +│ │ │ rule: "skip table comment removal ops on descriptor drop" +│ │ │ +│ │ ├── • ColumnDefaultExpression:{DescID: 114, ColumnID: 2} +│ │ │ PUBLIC → ABSENT +│ │ │ +│ │ └── • ColumnDefaultExpression:{DescID: 115, ColumnID: 2} +│ │ PUBLIC → ABSENT +│ │ +│ └── • 11 Mutation operations +│ │ +│ ├── • DrainDescriptorName +│ │ Namespace: +│ │ DatabaseID: 104 +│ │ DescriptorID: 113 +│ │ Name: sq1 +│ │ SchemaID: 105 +│ │ +│ ├── • RemoveColumnDefaultExpression +│ │ ColumnID: 2 +│ │ TableID: 114 +│ │ +│ ├── • UpdateBackReferencesInSequences +│ │ BackReferencedColumnID: 2 +│ │ BackReferencedTableID: 114 +│ │ SequenceIDs: +│ │ - 113 +│ │ +│ ├── • RemoveColumnDefaultExpression +│ │ ColumnID: 2 +│ │ TableID: 115 +│ │ +│ ├── • UpdateBackReferencesInSequences +│ │ BackReferencedColumnID: 2 +│ │ BackReferencedTableID: 115 +│ │ SequenceIDs: +│ │ - 113 +│ │ +│ ├── • MarkDescriptorAsDropped +│ │ DescID: 113 +│ │ +│ ├── • RemoveAllTableComments +│ │ TableID: 113 +│ │ +│ ├── • SetJobStateOnDescriptor +│ │ DescriptorID: 113 +│ │ Initialize: true +│ │ +│ ├── • SetJobStateOnDescriptor +│ │ DescriptorID: 114 +│ │ Initialize: true +│ │ +│ ├── • SetJobStateOnDescriptor +│ │ DescriptorID: 115 +│ │ Initialize: true +│ │ +│ └── • CreateSchemaChangerJob +│ Authorization: +│ UserName: root +│ DescriptorIDs: +│ - 113 +│ - 114 +│ - 115 +│ JobID: 1 +│ NonCancelable: true +│ Statements: +│ - statement: DROP SEQUENCE sq1 CASCADE +│ redactedstatement: DROP SEQUENCE ‹test›.public.‹sq1› CASCADE +│ statementtag: DROP SEQUENCE +│ +└── • PostCommitNonRevertiblePhase + │ + └── • Stage 1 of 1 in PostCommitNonRevertiblePhase + │ + ├── • 1 element transitioning toward ABSENT + │ │ + │ └── • Sequence:{DescID: 113} + │ DROPPED → ABSENT + │ + └── • 6 Mutation operations + │ + ├── • LogEvent + │ Authorization: + │ UserName: root + │ Element: + │ Sequence: + │ sequenceId: 113 + │ Statement: DROP SEQUENCE ‹test›.public.‹sq1› CASCADE + │ StatementTag: DROP SEQUENCE + │ TargetMetadata: + │ SourceElementID: 1 + │ SubWorkID: 1 + │ TargetStatus: 1 + │ + ├── • CreateGcJobForTable + │ StatementForDropJob: + │ Statement: DROP SEQUENCE test.public.sq1 CASCADE + │ TableID: 113 + │ + ├── • RemoveJobStateFromDescriptor + │ DescriptorID: 113 + │ JobID: 1 + │ + ├── • RemoveJobStateFromDescriptor + │ DescriptorID: 114 + │ JobID: 1 + │ + ├── • RemoveJobStateFromDescriptor + │ DescriptorID: 115 + │ JobID: 1 + │ + └── • UpdateSchemaChangerJob + IsNonCancelable: true + JobID: 1 # Success with cascade statement ok @@ -388,8 +850,147 @@ CREATE VIEW v3Dep AS (SELECT name, n1 FROM v1Dep, v2Dep); statement ok CREATE VIEW v4Dep AS (SELECT n2, n1 FROM v2Dep); -statement ok -explain (DDL, DEPS) DROP VIEW v1Dep CASCADE; +query T +EXPLAIN (DDL) DROP VIEW v1Dep CASCADE; +---- +Schema change plan for DROP VIEW ‹test›.public.‹v1dep› CASCADE; + ├── StatementPhase + │ └── Stage 1 of 1 in StatementPhase + │ ├── 4 elements transitioning toward ABSENT + │ │ ├── PUBLIC → TXN_DROPPED View:{DescID: 147} + │ │ ├── PUBLIC → TXN_DROPPED View:{DescID: 148} + │ │ ├── PUBLIC → TXN_DROPPED View:{DescID: 149} + │ │ └── PUBLIC → TXN_DROPPED View:{DescID: 150} + │ └── 4 Mutation operations + │ ├── MarkDescriptorAsDroppedSynthetically {"DescID":147} + │ ├── MarkDescriptorAsDroppedSynthetically {"DescID":148} + │ ├── MarkDescriptorAsDroppedSynthetically {"DescID":149} + │ └── MarkDescriptorAsDroppedSynthetically {"DescID":150} + ├── PreCommitPhase + │ └── Stage 1 of 1 in PreCommitPhase + │ ├── 56 elements transitioning toward ABSENT + │ │ ├── PUBLIC → ABSENT Namespace:{DescID: 147, Name: v1dep, ReferencedDescID: 104} + │ │ ├── PUBLIC → ABSENT Owner:{DescID: 147} + │ │ ├── PUBLIC → ABSENT UserPrivileges:{DescID: 147, Name: admin} + │ │ ├── PUBLIC → ABSENT UserPrivileges:{DescID: 147, Name: root} + │ │ ├── TXN_DROPPED → DROPPED View:{DescID: 147} + │ │ ├── PUBLIC → ABSENT ObjectParent:{DescID: 147, ReferencedDescID: 105} + │ │ ├── PUBLIC → ABSENT TableComment:{DescID: 147} + │ │ ├── PUBLIC → WRITE_ONLY Column:{DescID: 147, ColumnID: 1} + │ │ ├── PUBLIC → ABSENT ColumnName:{DescID: 147, Name: name, ColumnID: 1} + │ │ ├── PUBLIC → ABSENT ColumnType:{DescID: 147, ColumnFamilyID: 0, ColumnID: 1} + │ │ ├── PUBLIC → ABSENT ColumnComment:{DescID: 147, ColumnID: 1} + │ │ ├── PUBLIC → ABSENT Namespace:{DescID: 148, Name: v2dep, ReferencedDescID: 104} + │ │ ├── PUBLIC → ABSENT Owner:{DescID: 148} + │ │ ├── PUBLIC → ABSENT UserPrivileges:{DescID: 148, Name: admin} + │ │ ├── PUBLIC → ABSENT UserPrivileges:{DescID: 148, Name: root} + │ │ ├── TXN_DROPPED → DROPPED View:{DescID: 148} + │ │ ├── PUBLIC → ABSENT ObjectParent:{DescID: 148, ReferencedDescID: 105} + │ │ ├── PUBLIC → ABSENT TableComment:{DescID: 148} + │ │ ├── PUBLIC → WRITE_ONLY Column:{DescID: 148, ColumnID: 1} + │ │ ├── PUBLIC → ABSENT ColumnName:{DescID: 148, Name: n1, ColumnID: 1} + │ │ ├── PUBLIC → ABSENT ColumnType:{DescID: 148, ColumnFamilyID: 0, ColumnID: 1} + │ │ ├── PUBLIC → ABSENT ColumnComment:{DescID: 148, ColumnID: 1} + │ │ ├── PUBLIC → WRITE_ONLY Column:{DescID: 148, ColumnID: 2} + │ │ ├── PUBLIC → ABSENT ColumnName:{DescID: 148, Name: n2, ColumnID: 2} + │ │ ├── PUBLIC → ABSENT ColumnType:{DescID: 148, ColumnFamilyID: 0, ColumnID: 2} + │ │ ├── PUBLIC → ABSENT ColumnComment:{DescID: 148, ColumnID: 2} + │ │ ├── PUBLIC → ABSENT Namespace:{DescID: 149, Name: v3dep, ReferencedDescID: 104} + │ │ ├── PUBLIC → ABSENT Owner:{DescID: 149} + │ │ ├── PUBLIC → ABSENT UserPrivileges:{DescID: 149, Name: admin} + │ │ ├── PUBLIC → ABSENT UserPrivileges:{DescID: 149, Name: root} + │ │ ├── TXN_DROPPED → DROPPED View:{DescID: 149} + │ │ ├── PUBLIC → ABSENT ObjectParent:{DescID: 149, ReferencedDescID: 105} + │ │ ├── PUBLIC → ABSENT TableComment:{DescID: 149} + │ │ ├── PUBLIC → WRITE_ONLY Column:{DescID: 149, ColumnID: 1} + │ │ ├── PUBLIC → ABSENT ColumnName:{DescID: 149, Name: name, ColumnID: 1} + │ │ ├── PUBLIC → ABSENT ColumnType:{DescID: 149, ColumnFamilyID: 0, ColumnID: 1} + │ │ ├── PUBLIC → ABSENT ColumnComment:{DescID: 149, ColumnID: 1} + │ │ ├── PUBLIC → WRITE_ONLY Column:{DescID: 149, ColumnID: 2} + │ │ ├── PUBLIC → ABSENT ColumnName:{DescID: 149, Name: n1, ColumnID: 2} + │ │ ├── PUBLIC → ABSENT ColumnType:{DescID: 149, ColumnFamilyID: 0, ColumnID: 2} + │ │ ├── PUBLIC → ABSENT ColumnComment:{DescID: 149, ColumnID: 2} + │ │ ├── PUBLIC → ABSENT Namespace:{DescID: 150, Name: v4dep, ReferencedDescID: 104} + │ │ ├── PUBLIC → ABSENT Owner:{DescID: 150} + │ │ ├── PUBLIC → ABSENT UserPrivileges:{DescID: 150, Name: admin} + │ │ ├── PUBLIC → ABSENT UserPrivileges:{DescID: 150, Name: root} + │ │ ├── TXN_DROPPED → DROPPED View:{DescID: 150} + │ │ ├── PUBLIC → ABSENT ObjectParent:{DescID: 150, ReferencedDescID: 105} + │ │ ├── PUBLIC → ABSENT TableComment:{DescID: 150} + │ │ ├── PUBLIC → WRITE_ONLY Column:{DescID: 150, ColumnID: 1} + │ │ ├── PUBLIC → ABSENT ColumnName:{DescID: 150, Name: n2, ColumnID: 1} + │ │ ├── PUBLIC → ABSENT ColumnType:{DescID: 150, ColumnFamilyID: 0, ColumnID: 1} + │ │ ├── PUBLIC → ABSENT ColumnComment:{DescID: 150, ColumnID: 1} + │ │ ├── PUBLIC → WRITE_ONLY Column:{DescID: 150, ColumnID: 2} + │ │ ├── PUBLIC → ABSENT ColumnName:{DescID: 150, Name: n1, ColumnID: 2} + │ │ ├── PUBLIC → ABSENT ColumnType:{DescID: 150, ColumnFamilyID: 0, ColumnID: 2} + │ │ └── PUBLIC → ABSENT ColumnComment:{DescID: 150, ColumnID: 2} + │ └── 29 Mutation operations + │ ├── DrainDescriptorName {"Namespace":{"DatabaseID":104,"DescriptorID":147,"Name":"v1dep","SchemaID":105}} + │ ├── RemoveDroppedColumnType {"ColumnID":1,"TableID":147} + │ ├── DrainDescriptorName {"Namespace":{"DatabaseID":104,"DescriptorID":148,"Name":"v2dep","SchemaID":105}} + │ ├── RemoveDroppedColumnType {"ColumnID":1,"TableID":148} + │ ├── RemoveDroppedColumnType {"ColumnID":2,"TableID":148} + │ ├── DrainDescriptorName {"Namespace":{"DatabaseID":104,"DescriptorID":149,"Name":"v3dep","SchemaID":105}} + │ ├── RemoveDroppedColumnType {"ColumnID":1,"TableID":149} + │ ├── RemoveDroppedColumnType {"ColumnID":2,"TableID":149} + │ ├── DrainDescriptorName {"Namespace":{"DatabaseID":104,"DescriptorID":150,"Name":"v4dep","SchemaID":105}} + │ ├── RemoveDroppedColumnType {"ColumnID":1,"TableID":150} + │ ├── RemoveDroppedColumnType {"ColumnID":2,"TableID":150} + │ ├── MarkDescriptorAsDropped {"DescID":149} + │ ├── RemoveViewBackReferencesInRelations {"BackReferencedViewID":149} + │ ├── RemoveAllTableComments {"TableID":149} + │ ├── MarkDescriptorAsDropped {"DescID":150} + │ ├── RemoveViewBackReferencesInRelations {"BackReferencedViewID":150} + │ ├── RemoveAllTableComments {"TableID":150} + │ ├── MarkDescriptorAsDropped {"DescID":148} + │ ├── RemoveViewBackReferencesInRelations {"BackReferencedViewID":148} + │ ├── RemoveAllTableComments {"TableID":148} + │ ├── MarkDescriptorAsDropped {"DescID":147} + │ ├── RemoveViewBackReferencesInRelations {"BackReferencedViewID":147} + │ ├── RemoveAllTableComments {"TableID":147} + │ ├── SetJobStateOnDescriptor {"DescriptorID":146,"Initialize":true} + │ ├── SetJobStateOnDescriptor {"DescriptorID":147,"Initialize":true} + │ ├── SetJobStateOnDescriptor {"DescriptorID":148,"Initialize":true} + │ ├── SetJobStateOnDescriptor {"DescriptorID":149,"Initialize":true} + │ ├── SetJobStateOnDescriptor {"DescriptorID":150,"Initialize":true} + │ └── CreateSchemaChangerJob {"NonCancelable":true} + └── PostCommitNonRevertiblePhase + └── Stage 1 of 1 in PostCommitNonRevertiblePhase + ├── 11 elements transitioning toward ABSENT + │ ├── DROPPED → ABSENT View:{DescID: 147} + │ ├── WRITE_ONLY → ABSENT Column:{DescID: 147, ColumnID: 1} + │ ├── DROPPED → ABSENT View:{DescID: 148} + │ ├── WRITE_ONLY → ABSENT Column:{DescID: 148, ColumnID: 1} + │ ├── WRITE_ONLY → ABSENT Column:{DescID: 148, ColumnID: 2} + │ ├── DROPPED → ABSENT View:{DescID: 149} + │ ├── WRITE_ONLY → ABSENT Column:{DescID: 149, ColumnID: 1} + │ ├── WRITE_ONLY → ABSENT Column:{DescID: 149, ColumnID: 2} + │ ├── DROPPED → ABSENT View:{DescID: 150} + │ ├── WRITE_ONLY → ABSENT Column:{DescID: 150, ColumnID: 1} + │ └── WRITE_ONLY → ABSENT Column:{DescID: 150, ColumnID: 2} + └── 21 Mutation operations + ├── LogEvent {"Statement":"DROP VIEW ‹test›...","StatementTag":"DROP VIEW","TargetStatus":1} + ├── DeleteDescriptor {"DescriptorID":147} + ├── MakeColumnAbsent {"ColumnID":1,"TableID":147} + ├── LogEvent {"Statement":"DROP VIEW ‹test›...","StatementTag":"DROP VIEW","TargetStatus":1} + ├── DeleteDescriptor {"DescriptorID":148} + ├── MakeColumnAbsent {"ColumnID":1,"TableID":148} + ├── MakeColumnAbsent {"ColumnID":2,"TableID":148} + ├── LogEvent {"Statement":"DROP VIEW ‹test›...","StatementTag":"DROP VIEW","TargetStatus":1} + ├── DeleteDescriptor {"DescriptorID":149} + ├── MakeColumnAbsent {"ColumnID":1,"TableID":149} + ├── MakeColumnAbsent {"ColumnID":2,"TableID":149} + ├── LogEvent {"Statement":"DROP VIEW ‹test›...","StatementTag":"DROP VIEW","TargetStatus":1} + ├── DeleteDescriptor {"DescriptorID":150} + ├── MakeColumnAbsent {"ColumnID":1,"TableID":150} + ├── MakeColumnAbsent {"ColumnID":2,"TableID":150} + ├── RemoveJobStateFromDescriptor {"DescriptorID":146} + ├── RemoveJobStateFromDescriptor {"DescriptorID":147} + ├── RemoveJobStateFromDescriptor {"DescriptorID":148} + ├── RemoveJobStateFromDescriptor {"DescriptorID":149} + ├── RemoveJobStateFromDescriptor {"DescriptorID":150} + └── UpdateSchemaChangerJob {"IsNonCancelable":true} statement error pq: cannot drop relation "v1dep" because view "v3dep" depends on it DROP VIEW v1Dep RESTRICT; diff --git a/pkg/sql/opt/optbuilder/explain.go b/pkg/sql/opt/optbuilder/explain.go index dbbb8ccbfb12..33d697b87f47 100644 --- a/pkg/sql/opt/optbuilder/explain.go +++ b/pkg/sql/opt/optbuilder/explain.go @@ -48,12 +48,9 @@ func (b *Builder) buildExplain(explain *tree.Explain, inScope *scope) (outScope case tree.ExplainVec: telemetry.Inc(sqltelemetry.ExplainVecUseCounter) + case tree.ExplainDDL: - if explain.Flags[tree.ExplainFlagDeps] { - telemetry.Inc(sqltelemetry.ExplainDDLDeps) - } else { - telemetry.Inc(sqltelemetry.ExplainDDLStages) - } + telemetry.Inc(sqltelemetry.ExplainDDL) case tree.ExplainGist: telemetry.Inc(sqltelemetry.ExplainGist) diff --git a/pkg/sql/schemachanger/schemachanger_test.go b/pkg/sql/schemachanger/schemachanger_test.go index f6d22db29bf8..738a469b2a0d 100644 --- a/pkg/sql/schemachanger/schemachanger_test.go +++ b/pkg/sql/schemachanger/schemachanger_test.go @@ -890,7 +890,6 @@ func TestNewSchemaChangerVersionGating(t *testing.T) { results := tdb.QueryStr(t, "EXPLAIN (DDL) DROP TABLE db.t;") require.Equal(t, len(results), 1) require.Equal(t, len(results[0]), 1) - require.Contains(t, results[0][0], "https://cockroachdb.github.io/scplan/viz.html") }) t.Run("new_schema_changer_version_disabled", func(t *testing.T) { diff --git a/pkg/sql/schemachanger/scplan/BUILD.bazel b/pkg/sql/schemachanger/scplan/BUILD.bazel index 03e11119ae30..524b292bcddc 100644 --- a/pkg/sql/schemachanger/scplan/BUILD.bazel +++ b/pkg/sql/schemachanger/scplan/BUILD.bazel @@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "scplan", - srcs = ["plan.go"], + srcs = [ + "plan.go", + "plan_explain.go", + ], importpath = "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scplan", visibility = ["//visibility:public"], deps = [ @@ -14,9 +17,13 @@ go_library( "//pkg/sql/schemachanger/scplan/internal/scgraph", "//pkg/sql/schemachanger/scplan/internal/scgraphviz", "//pkg/sql/schemachanger/scplan/internal/scstage", + "//pkg/sql/schemachanger/screl", + "//pkg/util", "//pkg/util/log", "//pkg/util/timeutil", + "//pkg/util/treeprinter", "@com_github_cockroachdb_errors//:errors", + "@in_gopkg_yaml_v2//:yaml_v2", ], ) diff --git a/pkg/sql/schemachanger/scplan/internal/rules/registry.go b/pkg/sql/schemachanger/scplan/internal/rules/registry.go index 71c7de4d15ec..5ba36526f235 100644 --- a/pkg/sql/schemachanger/scplan/internal/rules/registry.go +++ b/pkg/sql/schemachanger/scplan/internal/rules/registry.go @@ -53,13 +53,14 @@ func ApplyDepRules(g *scgraph.Graph) error { // to the registered rules. func ApplyOpRules(g *scgraph.Graph) (*scgraph.Graph, error) { db := g.Database() - m := make(map[*screl.Node]struct{}) + m := make(map[*screl.Node][]string) for _, rule := range registry.opRules { var added int start := timeutil.Now() err := rule.q.Iterate(db, func(r rel.Result) error { added++ - m[r.Var(rule.from).(*screl.Node)] = struct{}{} + n := r.Var(rule.from).(*screl.Node) + m[n] = append(m[n], rule.name) return nil }) if err != nil { @@ -74,9 +75,9 @@ func ApplyOpRules(g *scgraph.Graph) (*scgraph.Graph, error) { } // Mark any op edges from these nodes as no-op. ret := g.ShallowClone() - for from := range m { + for from, rules := range m { if opEdge, ok := g.GetOpEdgeFrom(from); ok { - ret.MarkAsNoOp(opEdge) + ret.MarkAsNoOp(opEdge, rules...) } } return ret, nil diff --git a/pkg/sql/schemachanger/scplan/internal/scgraph/graph.go b/pkg/sql/schemachanger/scplan/internal/scgraph/graph.go index da144c6ea7c6..2037f90f4f1e 100644 --- a/pkg/sql/schemachanger/scplan/internal/scgraph/graph.go +++ b/pkg/sql/schemachanger/scplan/internal/scgraph/graph.go @@ -11,6 +11,8 @@ package scgraph import ( + "sort" + "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/rel" "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scop" "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scpb" @@ -48,7 +50,7 @@ type Graph struct { // noOpOpEdges that are marked optimized out, and will not generate // any operations. - noOpOpEdges map[*OpEdge]bool + noOpOpEdges map[*OpEdge]map[string]struct{} edges []Edge @@ -75,7 +77,7 @@ func New(cs scpb.CurrentState) (*Graph, error) { g := Graph{ targetIdxMap: map[*scpb.Target]int{}, opEdgesFrom: map[*screl.Node]*OpEdge{}, - noOpOpEdges: map[*OpEdge]bool{}, + noOpOpEdges: map[*OpEdge]map[string]struct{}{}, opToOpEdge: map[scop.Op]*OpEdge{}, entities: db, } @@ -112,7 +114,7 @@ func (g *Graph) ShallowClone() *Graph { opToOpEdge: g.opToOpEdge, edges: g.edges, entities: g.entities, - noOpOpEdges: make(map[*OpEdge]bool), + noOpOpEdges: make(map[*OpEdge]map[string]struct{}), } // Any decorations for mutations will be copied. for edge, noop := range g.noOpOpEdges { @@ -239,13 +241,34 @@ func (g *Graph) AddDepEdge( // MarkAsNoOp marks an edge as no-op, so that no operations are emitted from // this edge during planning. -func (g *Graph) MarkAsNoOp(edge *OpEdge) { - g.noOpOpEdges[edge] = true +func (g *Graph) MarkAsNoOp(edge *OpEdge, rule ...string) { + m := make(map[string]struct{}) + for _, r := range rule { + m[r] = struct{}{} + } + g.noOpOpEdges[edge] = m } // IsNoOp checks if an edge is marked as an edge that should emit no operations. func (g *Graph) IsNoOp(edge *OpEdge) bool { - return len(edge.op) == 0 || g.noOpOpEdges[edge] + if len(edge.op) == 0 { + return true + } + _, isNoOp := g.noOpOpEdges[edge] + return isNoOp +} + +// NoOpRules returns the rules which caused the edge to not emit any operations. +func (g *Graph) NoOpRules(edge *OpEdge) (rules []string) { + if !g.IsNoOp(edge) { + return nil + } + m := g.noOpOpEdges[edge] + for rule := range m { + rules = append(rules, rule) + } + sort.Strings(rules) + return rules } // Order returns the number of nodes in this graph. diff --git a/pkg/sql/schemachanger/scplan/internal/scgraphviz/graphviz.go b/pkg/sql/schemachanger/scplan/internal/scgraphviz/graphviz.go index 486d80e70378..0bc62b5542c9 100644 --- a/pkg/sql/schemachanger/scplan/internal/scgraphviz/graphviz.go +++ b/pkg/sql/schemachanger/scplan/internal/scgraphviz/graphviz.go @@ -70,33 +70,6 @@ func buildURL(gv string) (string, error) { }).String(), nil } -// DecorateErrorWithPlanDetails adds plan graphviz URLs as error details. -func DecorateErrorWithPlanDetails( - err error, cs scpb.CurrentState, g *scgraph.Graph, stages []scstage.Stage, -) error { - if err == nil { - return nil - } - - if len(stages) > 0 { - stagesURL, stagesErr := StagesURL(cs, g, stages) - if stagesErr != nil { - return errors.CombineErrors(err, stagesErr) - } - err = errors.WithDetailf(err, "stages: %s", stagesURL) - } - - if g != nil { - dependenciesURL, dependenciesErr := DependenciesURL(cs, g) - if dependenciesErr != nil { - return errors.CombineErrors(err, dependenciesErr) - } - err = errors.WithDetailf(err, "dependencies: %s", dependenciesURL) - } - - return errors.WithAssertionFailure(err) -} - // DrawStages returns a graphviz string of the stages of the Plan. func DrawStages(cs scpb.CurrentState, g *scgraph.Graph, stages []scstage.Stage) (string, error) { if len(stages) == 0 { diff --git a/pkg/sql/schemachanger/scplan/plan.go b/pkg/sql/schemachanger/scplan/plan.go index 10875c6a85d7..7ee467877332 100644 --- a/pkg/sql/schemachanger/scplan/plan.go +++ b/pkg/sql/schemachanger/scplan/plan.go @@ -19,7 +19,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scplan/internal/opgen" "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scplan/internal/rules" "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scplan/internal/scgraph" - "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scplan/internal/scgraphviz" "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scplan/internal/scstage" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/timeutil" @@ -73,21 +72,6 @@ func (p Plan) StagesForCurrentPhase() []scstage.Stage { return p.Stages } -// DecorateErrorWithPlanDetails adds plan graphviz URLs as error details. -func (p Plan) DecorateErrorWithPlanDetails(err error) error { - return scgraphviz.DecorateErrorWithPlanDetails(err, p.CurrentState, p.Graph, p.Stages) -} - -// DependenciesURL returns a URL to render the dependency graph in the Plan. -func (p Plan) DependenciesURL() (string, error) { - return scgraphviz.DependenciesURL(p.CurrentState, p.Graph) -} - -// StagesURL returns a URL to render the stages in the Plan. -func (p Plan) StagesURL() (string, error) { - return scgraphviz.StagesURL(p.CurrentState, p.Graph, p.Stages) -} - // MakePlan generates a Plan for a particular phase of a schema change, given // the initial state for a set of targets. // Returns an error when planning fails. It is up to the caller to wrap this diff --git a/pkg/sql/schemachanger/scplan/plan_explain.go b/pkg/sql/schemachanger/scplan/plan_explain.go new file mode 100644 index 000000000000..3160ffc1d67f --- /dev/null +++ b/pkg/sql/schemachanger/scplan/plan_explain.go @@ -0,0 +1,340 @@ +// Copyright 2022 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package scplan + +import ( + gojson "encoding/json" + "fmt" + "reflect" + "strings" + "unicode/utf8" + + "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scop" + "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scpb" + "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scplan/internal/scgraph" + "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scplan/internal/scgraphviz" + "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scplan/internal/scstage" + "github.com/cockroachdb/cockroach/pkg/sql/schemachanger/screl" + "github.com/cockroachdb/cockroach/pkg/util" + "github.com/cockroachdb/cockroach/pkg/util/treeprinter" + "github.com/cockroachdb/errors" + "gopkg.in/yaml.v2" +) + +// DecorateErrorWithPlanDetails adds plan graphviz URLs as error details. +func (p Plan) DecorateErrorWithPlanDetails(err error) error { + if err == nil { + return nil + } + + if len(p.Stages) > 0 { + explain, explainErr := p.ExplainVerbose() + if explainErr != nil { + explainErr = errors.Wrapf(explainErr, "error when generating EXPLAIN plan") + err = errors.CombineErrors(err, explainErr) + } else { + err = errors.WithDetailf(err, "%s", explain) + } + + stagesURL, stagesErr := p.StagesURL() + if stagesErr != nil { + stagesErr = errors.Wrapf(stagesErr, "error when generating EXPLAIN graphviz URL") + err = errors.CombineErrors(err, stagesErr) + } else { + err = errors.WithDetailf(err, "stages graphviz: %s", stagesURL) + } + } + + if p.Graph != nil { + dependenciesURL, dependenciesErr := p.DependenciesURL() + if dependenciesErr != nil { + dependenciesErr = errors.Wrapf(dependenciesErr, "error when generating dependencies graphviz URL") + err = errors.CombineErrors(err, dependenciesErr) + } else { + err = errors.WithDetailf(err, "dependencies graphviz: %s", dependenciesURL) + } + } + + return errors.WithAssertionFailure(err) +} + +// DependenciesURL returns a URL to render the dependency graph in the Plan. +func (p Plan) DependenciesURL() (string, error) { + return scgraphviz.DependenciesURL(p.CurrentState, p.Graph) +} + +// StagesURL returns a URL to render the stages in the Plan. +func (p Plan) StagesURL() (string, error) { + return scgraphviz.StagesURL(p.CurrentState, p.Graph, p.Stages) +} + +// ExplainCompact returns a human-readable plan rendering for +// EXPLAIN (DDL) statements. +func (p Plan) ExplainCompact() (string, error) { + return p.explain(treeprinter.DefaultStyle) +} + +// ExplainVerbose returns a human-readable plan rendering for +// EXPLAIN (DDL, VERBOSE) statements. +func (p Plan) ExplainVerbose() (string, error) { + return p.explain(treeprinter.BulletStyle) +} + +func (p Plan) explain(style treeprinter.Style) (string, error) { + // Generate root node. + tp := treeprinter.NewWithStyle(style) + var sb strings.Builder + { + sb.WriteString("Schema change plan for ") + if p.InRollback { + sb.WriteString("rolling back ") + } + for _, stmt := range p.Statements { + sb.WriteString(strings.TrimSuffix(stmt.RedactedStatement, ";")) + sb.WriteString("; ") + } + } + root := tp.Child(sb.String()) + var pn treeprinter.Node + for i, s := range p.Stages { + // Generate stage node, grouped by phase. + if i == 0 || s.Phase != p.Stages[i-1].Phase { + pn = root.Childf("%s", s.Phase) + } + sn := pn.Childf("Stage %d of %d in %s", s.Ordinal, s.StagesInPhase, s.Phase) + // Generate status transition nodes, grouped by target type. + if err := p.explainTargets(s, sn, style); err != nil { + return "", err + } + // Generate operations nodes. + if err := p.explainOps(s, sn, style); err != nil { + return "", err + } + } + return tp.String(), nil +} + +func (p Plan) explainTargets(s scstage.Stage, sn treeprinter.Node, style treeprinter.Style) error { + var targetTypeMap util.FastIntMap + depEdgeByElement := make(map[scpb.Element][]*scgraph.DepEdge) + noOpByElement := make(map[scpb.Element][]*scgraph.OpEdge) + var beforeMaxLen, afterMaxLen int + // Collect non-empty target status groupings for this stage. + for j, before := range s.Before { + t := &p.TargetState.Targets[j] + after := s.After[j] + if before == after { + continue + } + // Update status string max lengths for width-aligned formatting. + if l := utf8.RuneCountInString(before.String()); l > beforeMaxLen { + beforeMaxLen = l + } + if l := utf8.RuneCountInString(after.String()); l > afterMaxLen { + afterMaxLen = l + } + // Update grouping size. + k := int(scpb.AsTargetStatus(t.TargetStatus)) + numTransitions, found := targetTypeMap.Get(k) + if !found { + targetTypeMap.Set(k, 1) + } else { + targetTypeMap.Set(k, numTransitions+1) + } + // Collect rules affecting this element's status transitions. + if style == treeprinter.BulletStyle { + n, nodeFound := p.Graph.GetNode(t, before) + if !nodeFound { + return errors.Errorf("could not find node [[%s, %s], %s] in graph", + screl.ElementString(t.Element()), t.TargetStatus, before) + } + for n.CurrentStatus != after { + oe, edgeFound := p.Graph.GetOpEdgeFrom(n) + if !edgeFound { + return errors.Errorf("could not find op edge from %s in graph", screl.NodeString(n)) + } + n = oe.To() + if p.Graph.IsNoOp(oe) { + noOpByElement[t.Element()] = append(noOpByElement[t.Element()], oe) + } + if err := p.Graph.ForEachDepEdgeTo(n, func(de *scgraph.DepEdge) error { + depEdgeByElement[t.Element()] = append(depEdgeByElement[t.Element()], de) + return nil + }); err != nil { + return err + } + } + } + } + // Generate format string for printing element status transition. + fmtCompactTransition := fmt.Sprintf("%%-%ds → %%-%ds %%s", beforeMaxLen, afterMaxLen) + // Go over each target grouping. + targetTypeMap.ForEach(func(key, numTransitions int) { + ts := scpb.TargetStatus(key) + plural := "s" + if numTransitions == 1 { + plural = "" + } + tn := sn.Childf("%d element%s transitioning toward %s", + numTransitions, plural, ts.Status()) + for j, before := range s.Before { + t := &p.TargetState.Targets[j] + after := s.After[j] + if t.TargetStatus != ts.Status() || before == after { + continue + } + // Add element node and child rule nodes. + var en treeprinter.Node + if style == treeprinter.BulletStyle { + en = tn.Child(screl.ElementString(t.Element())) + en.AddLine(fmt.Sprintf("%s → %s", before, after)) + } else { + en = tn.Childf(fmtCompactTransition, before, after, screl.ElementString(t.Element())) + } + depEdges := depEdgeByElement[t.Element()] + for _, de := range depEdges { + rn := en.Childf("%s dependency from %s %s", + de.Kind(), de.From().CurrentStatus, screl.ElementString(de.From().Element())) + rn.AddLine(fmt.Sprintf("rule: %q", de.Name())) + } + noOpEdges := noOpByElement[t.Element()] + for _, oe := range noOpEdges { + noOpRules := p.Graph.NoOpRules(oe) + if len(noOpRules) == 0 { + continue + } + nn := en.Childf("skip %s → %s operations", + oe.From().CurrentStatus, oe.To().CurrentStatus) + for _, rule := range noOpRules { + nn.AddLine(fmt.Sprintf("rule: %q", rule)) + } + } + } + }) + return nil +} + +func (p Plan) explainOps(s scstage.Stage, sn treeprinter.Node, style treeprinter.Style) error { + ops := s.Ops() + if len(ops) == 0 { + return nil + } + plural := "s" + if len(ops) == 1 { + plural = "" + } + on := sn.Childf("%d %s operation%s", len(ops), strings.TrimSuffix(s.Type().String(), "Type"), plural) + for _, op := range ops { + if setJobStateOp, ok := op.(*scop.SetJobStateOnDescriptor); ok { + clone := *setJobStateOp + clone.State = scpb.DescriptorState{} + op = &clone + } + opName := strings.TrimPrefix(fmt.Sprintf("%T", op), "*scop.") + if style == treeprinter.BulletStyle { + n := on.Child(opName) + opBody, err := explainOpBodyVerbose(op) + if err != nil { + return err + } + for _, line := range strings.Split(opBody, "\n") { + n.AddLine(line) + } + } else { + opBody, err := explainOpBodyCompact(op) + if err != nil { + return err + } + if len(opBody) == 0 { + on.Child(opName) + } else { + on.Childf("%s %s", opName, opBody) + } + } + } + return nil +} + +func explainOpBodyVerbose(op scop.Op) (string, error) { + opMap, err := scgraphviz.ToMap(op) + if err != nil { + return "", err + } + yml, err := yaml.Marshal(opMap) + if err != nil { + return "", err + } + return strings.TrimSuffix(string(yml), "\n"), nil +} + +func explainOpBodyCompact(op scop.Op) (string, error) { + m := map[string]interface{}{} + // Round-trip the op through the json marshaller to make it easier to use + // reflection afterwards. + { + opMap, err := scgraphviz.ToMap(op) + if err != nil { + return "", err + } + jb, err := gojson.Marshal(opMap) + if err != nil { + return "", err + } + if err := gojson.Unmarshal(jb, &m); err != nil { + return "", err + } + } + // Trim the map to get rid of long strings, nested lists and objects, etc. + tm := opMapTrim(m) + if len(tm) == 0 { + // We've been trimming too aggressively. + // Add nested objects back, but trim them first. + for k, v := range m { + vm, ok := v.(map[string]interface{}) + if !ok { + continue + } + tvm := opMapTrim(vm) + if len(tvm) == 0 { + continue + } + tm[k] = tvm + } + if len(tm) == 0 { + return "", nil + } + } + jb, err := gojson.Marshal(tm) + return string(jb), err +} + +func opMapTrim(in map[string]interface{}) map[string]interface{} { + const jobIDKey = "JobID" + out := make(map[string]interface{}) + for k, v := range in { + vv := reflect.ValueOf(v) + switch vv.Type().Kind() { + case reflect.Bool: + out[k] = vv.Bool() + case reflect.Float64: + if k != jobIDKey || vv.Float() != 1 { + out[k] = vv.Float() + } + case reflect.String: + if str := vv.String(); utf8.RuneCountInString(str) > 16 { + out[k] = fmt.Sprintf("%.16s...", str) + } else { + out[k] = str + } + } + } + return out +} diff --git a/pkg/sql/sem/tree/explain.go b/pkg/sql/sem/tree/explain.go index c16b8706c5b9..d4b0d76b1a51 100644 --- a/pkg/sql/sem/tree/explain.go +++ b/pkg/sql/sem/tree/explain.go @@ -113,8 +113,6 @@ const ( ExplainFlagEnv ExplainFlagCatalog ExplainFlagJSON - ExplainFlagStages - ExplainFlagDeps ExplainFlagMemo ExplainFlagShape numExplainFlags = iota @@ -126,8 +124,6 @@ var explainFlagStrings = [...]string{ ExplainFlagEnv: "ENV", ExplainFlagCatalog: "CATALOG", ExplainFlagJSON: "JSON", - ExplainFlagStages: "STAGES", - ExplainFlagDeps: "DEPS", ExplainFlagMemo: "MEMO", ExplainFlagShape: "SHAPE", } diff --git a/pkg/sql/sqltelemetry/planning.go b/pkg/sql/sqltelemetry/planning.go index ce42efda6824..149a46f82bff 100644 --- a/pkg/sql/sqltelemetry/planning.go +++ b/pkg/sql/sqltelemetry/planning.go @@ -106,11 +106,8 @@ var ExplainOptUseCounter = telemetry.GetCounterOnce("sql.plan.explain-opt") // ExplainVecUseCounter is to be incremented whenever EXPLAIN (VEC) is run. var ExplainVecUseCounter = telemetry.GetCounterOnce("sql.plan.explain-vec") -// ExplainDDLStages is to be incremented whenever EXPLAIN (DDL, STAGES) is run. -var ExplainDDLStages = telemetry.GetCounterOnce("sql.plan.explain-ddl-stages") - -// ExplainDDLDeps is to be incremented whenever EXPLAIN (DDL, DEPS) is run. -var ExplainDDLDeps = telemetry.GetCounterOnce("sql.plan.explain-ddl-deps") +// ExplainDDL is to be incremented whenever EXPLAIN (DDL) is run. +var ExplainDDL = telemetry.GetCounterOnce("sql.plan.explain-ddl") // ExplainOptVerboseUseCounter is to be incremented whenever // EXPLAIN (OPT, VERBOSE) is run.