From f5dc9ef13e23e52c94798ef81f2665480d097e2c Mon Sep 17 00:00:00 2001 From: Daniel Harrison Date: Thu, 30 Jan 2020 12:57:43 -0800 Subject: [PATCH 1/2] clusterversion: split ClusterVersion out of pkg/settings/cluster Dep reasons. Release note: None --- .../cluster_version.go | 2 +- .../cluster_version.pb.go | 40 +++++++++---------- .../cluster_version.proto | 2 +- .../cockroach_versions.go | 2 +- .../cockroach_versions_test.go | 2 +- .../keyed_versions.go | 2 +- .../cluster => clusterversion}/testutils.go | 2 +- .../versionkey_string.go | 2 +- pkg/kv/txn_interceptor_committer.go | 3 +- pkg/rpc/heartbeat.go | 3 +- pkg/settings/cluster/settings.go | 37 ++++++++--------- pkg/sql/alter_table.go | 3 +- pkg/sql/alter_user.go | 5 ++- pkg/sql/backfill.go | 3 +- pkg/sql/create_index.go | 3 +- pkg/sql/create_table.go | 9 +++-- pkg/sql/descriptor.go | 3 +- pkg/sql/drop_index.go | 3 +- pkg/sql/plan.go | 3 +- pkg/sql/rowexec/backfiller.go | 3 +- pkg/sql/split.go | 5 ++- pkg/sql/sqlbase/metadata.go | 7 ++-- pkg/sql/sqlbase/namespace.go | 3 +- pkg/sql/sqlbase/structured.go | 3 +- pkg/sql/unsplit.go | 9 +++-- pkg/storage/engine/mvcc.go | 4 +- 26 files changed, 90 insertions(+), 73 deletions(-) rename pkg/{settings/cluster => clusterversion}/cluster_version.go (97%) rename pkg/{settings/cluster => clusterversion}/cluster_version.pb.go (81%) rename pkg/{settings/cluster => clusterversion}/cluster_version.proto (96%) rename pkg/{settings/cluster => clusterversion}/cockroach_versions.go (99%) rename pkg/{settings/cluster => clusterversion}/cockroach_versions_test.go (96%) rename pkg/{settings/cluster => clusterversion}/keyed_versions.go (99%) rename pkg/{settings/cluster => clusterversion}/testutils.go (96%) rename pkg/{settings/cluster => clusterversion}/versionkey_string.go (98%) diff --git a/pkg/settings/cluster/cluster_version.go b/pkg/clusterversion/cluster_version.go similarity index 97% rename from pkg/settings/cluster/cluster_version.go rename to pkg/clusterversion/cluster_version.go index a726f42f9709..31836a0c39ca 100644 --- a/pkg/settings/cluster/cluster_version.go +++ b/pkg/clusterversion/cluster_version.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package cluster +package clusterversion import "github.com/cockroachdb/cockroach/pkg/roachpb" diff --git a/pkg/settings/cluster/cluster_version.pb.go b/pkg/clusterversion/cluster_version.pb.go similarity index 81% rename from pkg/settings/cluster/cluster_version.pb.go rename to pkg/clusterversion/cluster_version.pb.go index 1cdb2658d2fe..db5acb056b8c 100644 --- a/pkg/settings/cluster/cluster_version.pb.go +++ b/pkg/clusterversion/cluster_version.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: settings/cluster/cluster_version.proto +// source: clusterversion/cluster_version.proto -package cluster +package clusterversion import proto "github.com/gogo/protobuf/proto" import fmt "fmt" @@ -33,7 +33,7 @@ type ClusterVersion struct { func (m *ClusterVersion) Reset() { *m = ClusterVersion{} } func (*ClusterVersion) ProtoMessage() {} func (*ClusterVersion) Descriptor() ([]byte, []int) { - return fileDescriptor_cluster_version_6bb8528e3c56dcd8, []int{0} + return fileDescriptor_cluster_version_3f03e67a33fb82fb, []int{0} } func (m *ClusterVersion) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -306,23 +306,23 @@ var ( ) func init() { - proto.RegisterFile("settings/cluster/cluster_version.proto", fileDescriptor_cluster_version_6bb8528e3c56dcd8) + proto.RegisterFile("clusterversion/cluster_version.proto", fileDescriptor_cluster_version_3f03e67a33fb82fb) } -var fileDescriptor_cluster_version_6bb8528e3c56dcd8 = []byte{ - // 216 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x2b, 0x4e, 0x2d, 0x29, - 0xc9, 0xcc, 0x4b, 0x2f, 0xd6, 0x4f, 0xce, 0x29, 0x2d, 0x2e, 0x49, 0x2d, 0x82, 0xd1, 0xf1, 0x65, - 0xa9, 0x45, 0xc5, 0x99, 0xf9, 0x79, 0x7a, 0x05, 0x45, 0xf9, 0x25, 0xf9, 0x42, 0x7c, 0xc9, 0xf9, - 0xc9, 0xd9, 0x45, 0xf9, 0x89, 0xc9, 0x19, 0x7a, 0x49, 0x89, 0xc5, 0xa9, 0x52, 0x62, 0x60, 0x76, - 0x41, 0x92, 0x7e, 0x6e, 0x6a, 0x49, 0x62, 0x4a, 0x62, 0x49, 0x22, 0x44, 0x9d, 0x94, 0x48, 0x7a, - 0x7e, 0x7a, 0x3e, 0x98, 0xa9, 0x0f, 0x62, 0x41, 0x44, 0x95, 0x32, 0xb9, 0xf8, 0x9c, 0x21, 0xc6, - 0x86, 0x41, 0x4c, 0x15, 0xf2, 0xe6, 0xe2, 0x4b, 0x4c, 0x2e, 0xc9, 0x2c, 0x4b, 0x85, 0xd9, 0x23, - 0xc1, 0xa4, 0xc0, 0xa8, 0xc1, 0x6d, 0x24, 0xa5, 0x87, 0xb0, 0x08, 0x6a, 0x85, 0x1e, 0x54, 0x8f, - 0x13, 0xc7, 0x89, 0x7b, 0xf2, 0x0c, 0x17, 0xee, 0xc9, 0x33, 0x06, 0xf1, 0x42, 0xf4, 0x42, 0x25, - 0xac, 0x58, 0x66, 0x2c, 0x90, 0x67, 0xf0, 0x62, 0xe1, 0x60, 0x14, 0x60, 0x72, 0xd2, 0x3c, 0xf1, - 0x50, 0x8e, 0xe1, 0xc4, 0x23, 0x39, 0xc6, 0x0b, 0x8f, 0xe4, 0x18, 0x6f, 0x3c, 0x92, 0x63, 0x7c, - 0xf0, 0x48, 0x8e, 0x71, 0xc2, 0x63, 0x39, 0x86, 0x0b, 0x8f, 0xe5, 0x18, 0x6e, 0x3c, 0x96, 0x63, - 0x88, 0x62, 0x87, 0xfa, 0x30, 0x89, 0x0d, 0xec, 0x38, 0x63, 0x40, 0x00, 0x00, 0x00, 0xff, 0xff, - 0x45, 0xdc, 0x17, 0x61, 0x04, 0x01, 0x00, 0x00, +var fileDescriptor_cluster_version_3f03e67a33fb82fb = []byte{ + // 210 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x49, 0xce, 0x29, 0x2d, + 0x2e, 0x49, 0x2d, 0x2a, 0x4b, 0x2d, 0x2a, 0xce, 0xcc, 0xcf, 0xd3, 0x87, 0x72, 0xe3, 0xa1, 0x7c, + 0xbd, 0x82, 0xa2, 0xfc, 0x92, 0x7c, 0x21, 0xbe, 0xe4, 0xfc, 0xe4, 0xec, 0xa2, 0xfc, 0xc4, 0xe4, + 0x0c, 0xbd, 0xa4, 0xc4, 0xe2, 0x54, 0x29, 0x31, 0x30, 0xbb, 0x20, 0x49, 0x3f, 0x37, 0xb5, 0x24, + 0x31, 0x25, 0xb1, 0x24, 0x11, 0xa2, 0x4e, 0x4a, 0x24, 0x3d, 0x3f, 0x3d, 0x1f, 0xcc, 0xd4, 0x07, + 0xb1, 0x20, 0xa2, 0x4a, 0x99, 0x5c, 0x7c, 0xce, 0x10, 0x63, 0xc3, 0x20, 0xa6, 0x0a, 0x79, 0x73, + 0xf1, 0x25, 0x26, 0x97, 0x64, 0x96, 0xa5, 0xc2, 0xec, 0x91, 0x60, 0x52, 0x60, 0xd4, 0xe0, 0x36, + 0x92, 0xd2, 0x43, 0x58, 0x04, 0xb5, 0x42, 0x0f, 0xaa, 0xc7, 0x89, 0xe3, 0xc4, 0x3d, 0x79, 0x86, + 0x0b, 0xf7, 0xe4, 0x19, 0x83, 0x78, 0x21, 0x7a, 0xa1, 0x12, 0x56, 0x2c, 0x33, 0x16, 0xc8, 0x33, + 0x78, 0xb1, 0x70, 0x30, 0x0a, 0x30, 0x39, 0x19, 0x9c, 0x78, 0x28, 0xc7, 0x70, 0xe2, 0x91, 0x1c, + 0xe3, 0x85, 0x47, 0x72, 0x8c, 0x37, 0x1e, 0xc9, 0x31, 0x3e, 0x78, 0x24, 0xc7, 0x38, 0xe1, 0xb1, + 0x1c, 0xc3, 0x85, 0xc7, 0x72, 0x0c, 0x37, 0x1e, 0xcb, 0x31, 0x44, 0xf1, 0xa1, 0x7a, 0x38, 0x89, + 0x0d, 0xec, 0x46, 0x63, 0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0xc3, 0x5f, 0x63, 0x79, 0x09, 0x01, + 0x00, 0x00, } diff --git a/pkg/settings/cluster/cluster_version.proto b/pkg/clusterversion/cluster_version.proto similarity index 96% rename from pkg/settings/cluster/cluster_version.proto rename to pkg/clusterversion/cluster_version.proto index 9353a01cd8aa..d238be0aff4e 100644 --- a/pkg/settings/cluster/cluster_version.proto +++ b/pkg/clusterversion/cluster_version.proto @@ -10,7 +10,7 @@ syntax = "proto3"; package cockroach.base; -option go_package = "cluster"; +option go_package = "clusterversion"; import "roachpb/metadata.proto"; import "gogoproto/gogo.proto"; diff --git a/pkg/settings/cluster/cockroach_versions.go b/pkg/clusterversion/cockroach_versions.go similarity index 99% rename from pkg/settings/cluster/cockroach_versions.go rename to pkg/clusterversion/cockroach_versions.go index a8809e1eb893..7a327d8fb691 100644 --- a/pkg/settings/cluster/cockroach_versions.go +++ b/pkg/clusterversion/cockroach_versions.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package cluster +package clusterversion import "github.com/cockroachdb/cockroach/pkg/roachpb" diff --git a/pkg/settings/cluster/cockroach_versions_test.go b/pkg/clusterversion/cockroach_versions_test.go similarity index 96% rename from pkg/settings/cluster/cockroach_versions_test.go rename to pkg/clusterversion/cockroach_versions_test.go index 3aa7c6fd0882..9ec4409cb53f 100644 --- a/pkg/settings/cluster/cockroach_versions_test.go +++ b/pkg/clusterversion/cockroach_versions_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package cluster +package clusterversion import ( "testing" diff --git a/pkg/settings/cluster/keyed_versions.go b/pkg/clusterversion/keyed_versions.go similarity index 99% rename from pkg/settings/cluster/keyed_versions.go rename to pkg/clusterversion/keyed_versions.go index 586844f8353b..c037db9dfeae 100644 --- a/pkg/settings/cluster/keyed_versions.go +++ b/pkg/clusterversion/keyed_versions.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package cluster +package clusterversion import ( "context" diff --git a/pkg/settings/cluster/testutils.go b/pkg/clusterversion/testutils.go similarity index 96% rename from pkg/settings/cluster/testutils.go rename to pkg/clusterversion/testutils.go index a9b248de1e17..e493c4502503 100644 --- a/pkg/settings/cluster/testutils.go +++ b/pkg/clusterversion/testutils.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package cluster +package clusterversion // TestingClusterVersion is a ClusterVersion that tests can use when they don't // want to go through a Settings object. diff --git a/pkg/settings/cluster/versionkey_string.go b/pkg/clusterversion/versionkey_string.go similarity index 98% rename from pkg/settings/cluster/versionkey_string.go rename to pkg/clusterversion/versionkey_string.go index 67f179a3f984..5f61a2ecb012 100644 --- a/pkg/settings/cluster/versionkey_string.go +++ b/pkg/clusterversion/versionkey_string.go @@ -1,6 +1,6 @@ // Code generated by "stringer -type=VersionKey"; DO NOT EDIT. -package cluster +package clusterversion import "strconv" diff --git a/pkg/kv/txn_interceptor_committer.go b/pkg/kv/txn_interceptor_committer.go index 53e18bcc877c..122235828ffb 100644 --- a/pkg/kv/txn_interceptor_committer.go +++ b/pkg/kv/txn_interceptor_committer.go @@ -14,6 +14,7 @@ import ( "context" "sync" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/settings/cluster" @@ -278,7 +279,7 @@ func (tc *txnCommitter) sendLockedWithElidedEndTxn( func (tc *txnCommitter) canCommitInParallelWithWrites( ctx context.Context, ba roachpb.BatchRequest, et *roachpb.EndTxnRequest, ) bool { - if !cluster.Version.IsActive(ctx, tc.st, cluster.VersionParallelCommits) { + if !cluster.Version.IsActive(ctx, tc.st, clusterversion.VersionParallelCommits) { return false } if !parallelCommitsEnabled.Get(&tc.st.SV) { diff --git a/pkg/rpc/heartbeat.go b/pkg/rpc/heartbeat.go index 7cd66487c105..772bc70e215c 100644 --- a/pkg/rpc/heartbeat.go +++ b/pkg/rpc/heartbeat.go @@ -16,6 +16,7 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/util/hlc" @@ -79,7 +80,7 @@ func checkClusterName(clusterName string, peerName string) error { func checkVersion(ctx context.Context, st *cluster.Settings, peerVersion roachpb.Version) error { activeVersion := cluster.Version.ActiveVersionOrEmpty(ctx, st) - if activeVersion == (cluster.ClusterVersion{}) { + if activeVersion == (clusterversion.ClusterVersion{}) { // Cluster version has not yet been determined. return nil } diff --git a/pkg/settings/cluster/settings.go b/pkg/settings/cluster/settings.go index e6dc72727522..793f4bb428bd 100644 --- a/pkg/settings/cluster/settings.go +++ b/pkg/settings/cluster/settings.go @@ -14,6 +14,7 @@ import ( "context" "sync/atomic" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/util/envutil" @@ -57,7 +58,7 @@ type Settings struct { beforeClusterVersionChangeMu struct { syncutil.Mutex // Callback to be called when the cluster version is about to be updated. - cb func(ctx context.Context, newVersion ClusterVersion) + cb func(ctx context.Context, newVersion clusterversion.ClusterVersion) } } @@ -166,7 +167,7 @@ var preserveDowngradeVersion = func() *settings.StringSetting { // MakeTestingClusterSettings returns a Settings object that has had its version // initialized to BinaryServerVersion. func MakeTestingClusterSettings() *Settings { - return MakeTestingClusterSettingsWithVersion(BinaryServerVersion, BinaryServerVersion) + return MakeTestingClusterSettingsWithVersion(clusterversion.BinaryServerVersion, clusterversion.BinaryServerVersion) } // MakeTestingClusterSettingsWithVersion returns a Settings object that has had @@ -239,7 +240,7 @@ func (cv clusterVersionSetting) BinaryMinSupportedVersion(st *Settings) roachpb. func (cv clusterVersionSetting) Initialize( ctx context.Context, version roachpb.Version, st *Settings, ) error { - if ver := cv.ActiveVersionOrEmpty(ctx, st); ver != (ClusterVersion{}) { + if ver := cv.ActiveVersionOrEmpty(ctx, st); ver != (clusterversion.ClusterVersion{}) { // Allow initializing a second time as long as it's setting the version to // what it was already set. This is useful in tests that use // MakeTestingClusterSettings() which initializes the version, and the @@ -255,7 +256,7 @@ func (cv clusterVersionSetting) Initialize( } // Return the serialized form of the new version. - newV := ClusterVersion{Version: version} + newV := clusterversion.ClusterVersion{Version: version} encoded, err := protoutil.Marshal(&newV) if err != nil { return err @@ -268,9 +269,9 @@ func (cv clusterVersionSetting) Initialize( // cluster version the caller may assume is in effect. // // ActiveVersion fatals if the version has not been initialized. -func (cv *clusterVersionSetting) ActiveVersion(ctx context.Context, st *Settings) ClusterVersion { +func (cv *clusterVersionSetting) ActiveVersion(ctx context.Context, st *Settings) clusterversion.ClusterVersion { ver := cv.ActiveVersionOrEmpty(ctx, st) - if ver == (ClusterVersion{}) { + if ver == (clusterversion.ClusterVersion{}) { log.Fatalf(ctx, "version not initialized") } return ver @@ -280,12 +281,12 @@ func (cv *clusterVersionSetting) ActiveVersion(ctx context.Context, st *Settings // the active version was not initialized. func (cv *clusterVersionSetting) ActiveVersionOrEmpty( ctx context.Context, st *Settings, -) ClusterVersion { +) clusterversion.ClusterVersion { encoded := cv.GetInternal(&st.SV) if encoded == nil { - return ClusterVersion{} + return clusterversion.ClusterVersion{} } - var curVer ClusterVersion + var curVer clusterversion.ClusterVersion if err := protoutil.Unmarshal(encoded.([]byte), &curVer); err != nil { log.Fatal(ctx, err) } @@ -318,7 +319,7 @@ func (cv *clusterVersionSetting) ActiveVersionOrEmpty( // latest active version, node2 is aware that the sending node has, and it will // too, eventually. func (cv *clusterVersionSetting) IsActive( - ctx context.Context, st *Settings, versionKey VersionKey, + ctx context.Context, st *Settings, versionKey clusterversion.VersionKey, ) bool { return cv.ActiveVersion(ctx, st).IsActive(versionKey) } @@ -327,7 +328,7 @@ func (cv *clusterVersionSetting) IsActive( func (cv clusterVersionSetting) BeforeChange( ctx context.Context, encodedVal []byte, sv *settings.Values, ) { - var clusterVersion ClusterVersion + var clusterVersion clusterversion.ClusterVersion if err := protoutil.Unmarshal(encodedVal, &clusterVersion); err != nil { log.Fatalf(ctx, "failed to unmarshall version: %s", err) } @@ -347,7 +348,7 @@ func (cv clusterVersionSetting) BeforeChange( // // The callback can be set at most once. func (cv clusterVersionSetting) SetBeforeChange( - ctx context.Context, st *Settings, cb func(ctx context.Context, newVersion ClusterVersion), + ctx context.Context, st *Settings, cb func(ctx context.Context, newVersion clusterversion.ClusterVersion), ) { st.beforeClusterVersionChangeMu.Lock() defer st.beforeClusterVersionChangeMu.Unlock() @@ -359,7 +360,7 @@ func (cv clusterVersionSetting) SetBeforeChange( // Decode is part of the StateMachineSettingImpl interface. func (cv clusterVersionSetting) Decode(val []byte) (interface{}, error) { - var clusterVersion ClusterVersion + var clusterVersion clusterversion.ClusterVersion if err := protoutil.Unmarshal(val, &clusterVersion); err != nil { return "", err } @@ -372,7 +373,7 @@ func (cv clusterVersionSetting) DecodeToString(val []byte) (string, error) { if err != nil { return "", err } - return clusterVersion.(ClusterVersion).Version.String(), nil + return clusterVersion.(clusterversion.ClusterVersion).Version.String(), nil } // ValidateLogical is part of the StateMachineSettingImpl interface. @@ -387,7 +388,7 @@ func (cv clusterVersionSetting) ValidateLogical( return nil, err } - var oldV ClusterVersion + var oldV clusterversion.ClusterVersion if err := protoutil.Unmarshal(curRawProto, &oldV); err != nil { return nil, err } @@ -407,7 +408,7 @@ func (cv clusterVersionSetting) ValidateLogical( } // Return the serialized form of the new version. - newV := ClusterVersion{Version: newVersion} + newV := clusterversion.ClusterVersion{Version: newVersion} return protoutil.Marshal(&newV) } @@ -425,7 +426,7 @@ func (cv clusterVersionSetting) ValidateGossipUpdate( } }() - var ver ClusterVersion + var ver clusterversion.ClusterVersion if err := protoutil.Unmarshal(rawProto, &ver); err != nil { return err } @@ -454,5 +455,5 @@ func (cv clusterVersionSetting) validateSupportedVersionInner( // SettingsListDefault is part of the StateMachineSettingImpl interface. func (cv clusterVersionSetting) SettingsListDefault() string { - return BinaryServerVersion.String() + return clusterversion.BinaryServerVersion.String() } diff --git a/pkg/sql/alter_table.go b/pkg/sql/alter_table.go index 6c7d252eaef6..caef4e9660a8 100644 --- a/pkg/sql/alter_table.go +++ b/pkg/sql/alter_table.go @@ -16,6 +16,7 @@ import ( gojson "encoding/json" "fmt" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/server/telemetry" "github.com/cockroachdb/cockroach/pkg/settings/cluster" @@ -314,7 +315,7 @@ func (n *alterTableNode) startExec(params runParams) error { case *tree.AlterTableAlterPrimaryKey: // Make sure that all nodes in the cluster are able to perform primary key changes before proceeding. version := cluster.Version.ActiveVersionOrEmpty(params.ctx, params.p.ExecCfg().Settings) - if !version.IsActive(cluster.VersionPrimaryKeyChanges) { + if !version.IsActive(clusterversion.VersionPrimaryKeyChanges) { return pgerror.Newf(pgcode.FeatureNotSupported, "all nodes are not the correct version for primary key changes") } diff --git a/pkg/sql/alter_user.go b/pkg/sql/alter_user.go index bcd577eee048..9055a420780f 100644 --- a/pkg/sql/alter_user.go +++ b/pkg/sql/alter_user.go @@ -13,6 +13,7 @@ package sql import ( "context" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/security" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" @@ -68,10 +69,10 @@ func (n *alterUserSetPasswordNode) startExec(params runParams) error { // TODO(knz): Remove in 20.2. if normalizedUsername == security.RootUser && len(hashedPassword) > 0 && - !cluster.Version.IsActive(params.ctx, params.EvalContext().Settings, cluster.VersionRootPassword) { + !cluster.Version.IsActive(params.ctx, params.EvalContext().Settings, clusterversion.VersionRootPassword) { return pgerror.Newf(pgcode.ObjectNotInPrerequisiteState, `setting a root password requires all nodes to be upgraded to %s`, - cluster.VersionByKey(cluster.VersionRootPassword), + clusterversion.VersionByKey(clusterversion.VersionRootPassword), ) } diff --git a/pkg/sql/backfill.go b/pkg/sql/backfill.go index 56e941abd5e0..f0832c68b85c 100644 --- a/pkg/sql/backfill.go +++ b/pkg/sql/backfill.go @@ -16,6 +16,7 @@ import ( "sort" "time" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/jobs" "github.com/cockroachdb/cockroach/pkg/jobs/jobspb" @@ -712,7 +713,7 @@ func (sc *SchemaChanger) distBackfill( targetSpans []roachpb.Span, ) error { inMemoryStatusEnabled := cluster.Version.IsActive( - ctx, sc.execCfg.Settings, cluster.VersionAtomicChangeReplicasTrigger) + ctx, sc.execCfg.Settings, clusterversion.VersionAtomicChangeReplicasTrigger) duration := checkpointInterval if sc.testingKnobs.WriteCheckpointInterval > 0 { duration = sc.testingKnobs.WriteCheckpointInterval diff --git a/pkg/sql/create_index.go b/pkg/sql/create_index.go index 4e2079b49802..2e98f0d7283e 100644 --- a/pkg/sql/create_index.go +++ b/pkg/sql/create_index.go @@ -13,6 +13,7 @@ package sql import ( "context" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" @@ -111,7 +112,7 @@ func (n *createIndexNode) startExec(params runParams) error { // If all nodes in the cluster know how to handle secondary indexes with column families, // write the new version into the index descriptor. encodingVersion := sqlbase.BaseIndexFormatVersion - if cluster.Version.IsActive(params.ctx, params.p.EvalContext().Settings, cluster.VersionSecondaryIndexColumnFamilies) { + if cluster.Version.IsActive(params.ctx, params.p.EvalContext().Settings, clusterversion.VersionSecondaryIndexColumnFamilies) { encodingVersion = sqlbase.SecondaryIndexFamilyFormatVersion } indexDesc.Version = encodingVersion diff --git a/pkg/sql/create_table.go b/pkg/sql/create_table.go index c9341acd04ac..4dfd75a0b641 100644 --- a/pkg/sql/create_table.go +++ b/pkg/sql/create_table.go @@ -18,6 +18,7 @@ import ( "sort" "strings" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/server/telemetry" @@ -1161,8 +1162,8 @@ func MakeTableDesc( // before the version has been initialized, leading to a panic. There are also // cases where this function is called in tests where st is nil. if st != nil { - if version := cluster.Version.ActiveVersionOrEmpty(ctx, st); version != (cluster.ClusterVersion{}) && - version.IsActive(cluster.VersionSecondaryIndexColumnFamilies) { + if version := cluster.Version.ActiveVersionOrEmpty(ctx, st); version != (clusterversion.ClusterVersion{}) && + version.IsActive(clusterversion.VersionSecondaryIndexColumnFamilies) { indexEncodingVersion = sqlbase.SecondaryIndexFamilyFormatVersion } } @@ -1331,8 +1332,8 @@ func MakeTableDesc( // If any nodes are not at version VersionPrimaryKeyColumnsOutOfFamilyZero, then return an error // if a primary key column is not in column family 0. if st != nil { - if version := cluster.Version.ActiveVersionOrEmpty(ctx, st); version != (cluster.ClusterVersion{}) && - !version.IsActive(cluster.VersionPrimaryKeyColumnsOutOfFamilyZero) { + if version := cluster.Version.ActiveVersionOrEmpty(ctx, st); version != (clusterversion.ClusterVersion{}) && + !version.IsActive(clusterversion.VersionPrimaryKeyColumnsOutOfFamilyZero) { var colsInFamZero util.FastIntSet for _, colID := range desc.Families[0].ColumnIDs { colsInFamZero.Add(int(colID)) diff --git a/pkg/sql/descriptor.go b/pkg/sql/descriptor.go index c73ebf191a32..eb71092b963d 100644 --- a/pkg/sql/descriptor.go +++ b/pkg/sql/descriptor.go @@ -13,6 +13,7 @@ package sql import ( "context" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" @@ -66,7 +67,7 @@ func (p *planner) createDatabase( // TODO(solon): This conditional can be removed in 20.2. Every database // is created with a public schema for cluster version >= 20.1, so we can remove // the `shouldCreatePublicSchema` logic as well. - if !cluster.Version.IsActive(ctx, p.ExecCfg().Settings, cluster.VersionNamespaceTableWithSchemas) { + if !cluster.Version.IsActive(ctx, p.ExecCfg().Settings, clusterversion.VersionNamespaceTableWithSchemas) { shouldCreatePublicSchema = false } diff --git a/pkg/sql/drop_index.go b/pkg/sql/drop_index.go index 044018e911e4..9bf6fd81b0eb 100644 --- a/pkg/sql/drop_index.go +++ b/pkg/sql/drop_index.go @@ -15,6 +15,7 @@ import ( "fmt" "strings" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" @@ -199,7 +200,7 @@ func (p *planner) dropIndexByName( // If we aren't at the cluster version where we have removed explicit foreign key IDs // from the foreign key descriptors, fall back to the existing drop index logic. // That means we pretend that we can never find replacements for any indexes. - if !cluster.Version.IsActive(ctx, p.ExecCfg().Settings, cluster.VersionNoExplicitForeignKeyIndexIDs) { + if !cluster.Version.IsActive(ctx, p.ExecCfg().Settings, clusterversion.VersionNoExplicitForeignKeyIndexIDs) { indexHasReplacementCandidate = func(func(*sqlbase.IndexDescriptor) bool) bool { return false } diff --git a/pkg/sql/plan.go b/pkg/sql/plan.go index 09e241df09be..e06fdf24db3b 100644 --- a/pkg/sql/plan.go +++ b/pkg/sql/plan.go @@ -13,6 +13,7 @@ package sql import ( "context" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" @@ -70,7 +71,7 @@ func (r *runParams) creationTimeForNewTableDescriptor() hlc.Timestamp { // upon reading use the MVCC timestamp to populate the values. var ts hlc.Timestamp if !cluster.Version.IsActive( - r.ctx, r.ExecCfg().Settings, cluster.VersionTableDescModificationTimeFromMVCC, + r.ctx, r.ExecCfg().Settings, clusterversion.VersionTableDescModificationTimeFromMVCC, ) { ts = r.p.txn.CommitTimestamp() } diff --git a/pkg/sql/rowexec/backfiller.go b/pkg/sql/rowexec/backfiller.go index 4aff7b8254d5..20e82fab199c 100644 --- a/pkg/sql/rowexec/backfiller.go +++ b/pkg/sql/rowexec/backfiller.go @@ -15,6 +15,7 @@ import ( "fmt" "github.com/cockroachdb/cockroach/pkg/internal/client" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/jobs" "github.com/cockroachdb/cockroach/pkg/jobs/jobspb" "github.com/cockroachdb/cockroach/pkg/roachpb" @@ -137,7 +138,7 @@ func (b *backfiller) doRun(ctx context.Context) *execinfrapb.ProducerMetadata { return &execinfrapb.ProducerMetadata{Err: err} } st := b.flowCtx.Cfg.Settings - if !cluster.Version.IsActive(ctx, st, cluster.VersionAtomicChangeReplicasTrigger) { + if !cluster.Version.IsActive(ctx, st, clusterversion.VersionAtomicChangeReplicasTrigger) { // There is a node of older version which could be the coordinator. // So we communicate the finished work by writing to the jobs row. err = WriteResumeSpan(ctx, diff --git a/pkg/sql/split.go b/pkg/sql/split.go index 61f1e6e5b4e9..65412d3ef2c1 100644 --- a/pkg/sql/split.go +++ b/pkg/sql/split.go @@ -13,6 +13,7 @@ package sql import ( "context" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" @@ -41,7 +42,7 @@ type splitRun struct { func (n *splitNode) startExec(params runParams) error { st := params.EvalContext().Settings - stickyBitEnabled := cluster.Version.IsActive(params.ctx, st, cluster.VersionStickyBit) + stickyBitEnabled := cluster.Version.IsActive(params.ctx, st, clusterversion.VersionStickyBit) // TODO(jeffreyxiao): Remove this error, splitNode.force, and // experimental_force_split_at in v20.1. // This check is not intended to be foolproof. The setting could be outdated @@ -72,7 +73,7 @@ func (n *splitNode) Next(params runParams) (bool, error) { // TODO(jeffreyxiao): Remove this check in v20.1. // Don't set the manual flag if the cluster is not up-to-date. st := params.EvalContext().Settings - stickyBitEnabled := cluster.Version.IsActive(params.ctx, st, cluster.VersionStickyBit) + stickyBitEnabled := cluster.Version.IsActive(params.ctx, st, clusterversion.VersionStickyBit) expirationTime := hlc.Timestamp{} if stickyBitEnabled { expirationTime = n.expirationTime diff --git a/pkg/sql/sqlbase/metadata.go b/pkg/sql/sqlbase/metadata.go index ceb39113a391..0165ea277d62 100644 --- a/pkg/sql/sqlbase/metadata.go +++ b/pkg/sql/sqlbase/metadata.go @@ -21,6 +21,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/protoutil" + "github.com/cockroachdb/cockroach/pkg/clusterversion" ) var _ DescriptorProto = &DatabaseDescriptor{} @@ -123,7 +124,7 @@ func (ms MetadataSchema) SystemDescriptorCount() int { // in the schema. Also returns a list of split points (a split for each SQL // table descriptor part of the initial values). Both returned sets are sorted. func (ms MetadataSchema) GetInitialValues( - bootstrapVersion cluster.ClusterVersion, + bootstrapVersion clusterversion.ClusterVersion, ) ([]roachpb.KeyValue, []roachpb.RKey) { var ret []roachpb.KeyValue var splits []roachpb.RKey @@ -146,7 +147,7 @@ func (ms MetadataSchema) GetInitialValues( // TODO(solon): This if/else can be removed in 20.2, as there will be no // need to support the deprecated namespace table. - if bootstrapVersion.IsActive(cluster.VersionNamespaceTableWithSchemas) { + if bootstrapVersion.IsActive(clusterversion.VersionNamespaceTableWithSchemas) { if parentID != keys.RootNamespaceID { ret = append(ret, roachpb.KeyValue{ Key: NewPublicTableKey(parentID, desc.GetName()).Key(), @@ -255,7 +256,7 @@ func LookupSystemTableDescriptorID( } if settings != nil && - !cluster.Version.IsActive(ctx, settings, cluster.VersionNamespaceTableWithSchemas) && + !cluster.Version.IsActive(ctx, settings, clusterversion.VersionNamespaceTableWithSchemas) && tableName == NamespaceTable.Name { return DeprecatedNamespaceTable.ID } diff --git a/pkg/sql/sqlbase/namespace.go b/pkg/sql/sqlbase/namespace.go index 19767504ac05..d7959cb01c46 100644 --- a/pkg/sql/sqlbase/namespace.go +++ b/pkg/sql/sqlbase/namespace.go @@ -13,6 +13,7 @@ package sqlbase import ( "context" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/settings/cluster" @@ -118,7 +119,7 @@ func MakeObjectNameKey( ctx context.Context, settings *cluster.Settings, parentID ID, parentSchemaID ID, name string, ) DescriptorKey { // TODO(solon): This if condition can be removed in 20.2 - if !cluster.Version.IsActive(ctx, settings, cluster.VersionNamespaceTableWithSchemas) { + if !cluster.Version.IsActive(ctx, settings, clusterversion.VersionNamespaceTableWithSchemas) { return NewDeprecatedTableKey(parentID, name) } var key DescriptorKey diff --git a/pkg/sql/sqlbase/structured.go b/pkg/sql/sqlbase/structured.go index 0bd5563f3aa9..6554b0dec83b 100644 --- a/pkg/sql/sqlbase/structured.go +++ b/pkg/sql/sqlbase/structured.go @@ -34,6 +34,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/cockroachdb/errors" + "github.com/cockroachdb/cockroach/pkg/clusterversion" ) // ID, ColumnID, FamilyID, and IndexID are all uint32, but are each given a @@ -1490,7 +1491,7 @@ func (desc *MutableTableDescriptor) MaybeIncrementVersion( // // TODO(ajwerner): remove this check in 20.1. var modTime hlc.Timestamp - if !cluster.Version.IsActive(ctx, settings, cluster.VersionTableDescModificationTimeFromMVCC) { + if !cluster.Version.IsActive(ctx, settings, clusterversion.VersionTableDescModificationTimeFromMVCC) { modTime = txn.CommitTimestamp() } desc.ModificationTime = modTime diff --git a/pkg/sql/unsplit.go b/pkg/sql/unsplit.go index d8ffe8c0eb04..b659ef94f1e6 100644 --- a/pkg/sql/unsplit.go +++ b/pkg/sql/unsplit.go @@ -13,6 +13,7 @@ package sql import ( "context" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" @@ -38,12 +39,12 @@ type unsplitRun struct { func (n *unsplitNode) startExec(params runParams) error { st := params.EvalContext().Settings - stickyBitEnabled := cluster.Version.IsActive(params.ctx, st, cluster.VersionStickyBit) + stickyBitEnabled := cluster.Version.IsActive(params.ctx, st, clusterversion.VersionStickyBit) // TODO(jeffreyxiao): Remove this error in v20.1. if !stickyBitEnabled { return pgerror.Newf(pgcode.ObjectNotInPrerequisiteState, `UNSPLIT AT requires all nodes to be upgraded to %s`, - cluster.VersionByKey(cluster.VersionStickyBit), + clusterversion.VersionByKey(clusterversion.VersionStickyBit), ) } return nil @@ -98,12 +99,12 @@ type unsplitAllRun struct { func (n *unsplitAllNode) startExec(params runParams) error { st := params.EvalContext().Settings - stickyBitEnabled := cluster.Version.IsActive(params.ctx, st, cluster.VersionStickyBit) + stickyBitEnabled := cluster.Version.IsActive(params.ctx, st, clusterversion.VersionStickyBit) // TODO(jeffreyxiao): Remove this error in v20.1. if !stickyBitEnabled { return pgerror.Newf(pgcode.ObjectNotInPrerequisiteState, `UNSPLIT AT requires all nodes to be upgraded to %s`, - cluster.VersionByKey(cluster.VersionStickyBit), + clusterversion.VersionByKey(clusterversion.VersionStickyBit), ) } // Use the internal executor to retrieve the split keys. diff --git a/pkg/storage/engine/mvcc.go b/pkg/storage/engine/mvcc.go index f419cd7cda3c..24fdbd8cf0c3 100644 --- a/pkg/storage/engine/mvcc.go +++ b/pkg/storage/engine/mvcc.go @@ -22,7 +22,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" - "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/storage/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/util/encoding" "github.com/cockroachdb/cockroach/pkg/util/hlc" @@ -219,7 +219,7 @@ func updateStatsOnMerge(key roachpb.Key, valSize, nowNanos int64) enginepb.MVCCS sys := isSysLocal(key) ms.AgeTo(nowNanos) - _ = cluster.VersionContainsEstimatesCounter // see for info on ContainsEstimates migration + _ = clusterversion.VersionContainsEstimatesCounter // see for info on ContainsEstimates migration ms.ContainsEstimates = 1 if sys { From b2abf1c981288546b16fc5004d974b719de6d11a Mon Sep 17 00:00:00 2001 From: Daniel Harrison Date: Thu, 6 Feb 2020 11:52:50 -0800 Subject: [PATCH 2/2] WIP clusterversion,migration: begin long-running migration prototype Release note (): --- docs/generated/settings/settings.html | 2 +- pkg/ccl/backupccl/backup.go | 3 +- pkg/ccl/importccl/import_stmt.go | 3 +- pkg/cli/context.go | 3 +- pkg/clusterversion/cluster_version.go | 43 +- pkg/clusterversion/cockroach_versions.go | 63 +- pkg/clusterversion/everynode/everynode.go | 221 +++++++ pkg/clusterversion/everynode/everynode.pb.go | 579 ++++++++++++++++++ pkg/clusterversion/everynode/everynode.proto | 34 + pkg/clusterversion/keyed_versions.go | 12 +- pkg/clusterversion/migration/migration.go | 218 +++++++ pkg/clusterversion/migration/migration.pb.go | 566 +++++++++++++++++ pkg/clusterversion/migration/migration.proto | 29 + pkg/keys/constants.go | 10 + pkg/kv/txn_interceptor_committer.go | 2 +- pkg/rpc/context.go | 2 +- pkg/rpc/heartbeat.go | 4 +- pkg/server/node.go | 24 +- pkg/server/server.go | 33 +- pkg/server/testserver.go | 3 +- pkg/settings/cluster/settings.go | 55 +- pkg/sql/alter_table.go | 3 +- pkg/sql/alter_user.go | 3 +- pkg/sql/backfill.go | 5 +- pkg/sql/create_index.go | 3 +- pkg/sql/create_table.go | 4 +- pkg/sql/descriptor.go | 2 +- pkg/sql/drop_index.go | 3 +- pkg/sql/pgwire/auth_methods.go | 12 +- pkg/sql/pgwire/hba_conf.go | 13 +- pkg/sql/plan.go | 5 +- pkg/sql/rowexec/backfiller.go | 5 +- pkg/sql/sem/builtins/builtins.go | 2 +- pkg/sql/split.go | 5 +- pkg/sql/sqlbase/metadata.go | 4 +- pkg/sql/sqlbase/namespace.go | 2 +- pkg/sql/sqlbase/structured.go | 4 +- pkg/sql/unsplit.go | 5 +- pkg/sqlmigrations/migrations.go | 25 +- pkg/storage/batcheval/cmd_add_sstable.go | 4 +- pkg/storage/batcheval/cmd_end_transaction.go | 4 +- pkg/storage/batcheval/cmd_query_txn.go | 6 +- pkg/storage/batcheval/cmd_recompute_stats.go | 4 +- pkg/storage/bulk/sst_batcher.go | 7 +- .../replica_application_state_machine.go | 4 +- pkg/storage/replica_command.go | 25 +- pkg/storage/replica_consistency.go | 3 +- pkg/storage/replica_proposal.go | 5 +- pkg/storage/replicate_queue.go | 4 +- pkg/storage/store.go | 9 +- pkg/storage/store_bootstrap.go | 6 +- pkg/storage/stores.go | 30 +- 52 files changed, 1965 insertions(+), 160 deletions(-) create mode 100644 pkg/clusterversion/everynode/everynode.go create mode 100644 pkg/clusterversion/everynode/everynode.pb.go create mode 100644 pkg/clusterversion/everynode/everynode.proto create mode 100644 pkg/clusterversion/migration/migration.go create mode 100644 pkg/clusterversion/migration/migration.pb.go create mode 100644 pkg/clusterversion/migration/migration.proto diff --git a/docs/generated/settings/settings.html b/docs/generated/settings/settings.html index 8c9baea5daf8..384df9c71dbd 100644 --- a/docs/generated/settings/settings.html +++ b/docs/generated/settings/settings.html @@ -61,6 +61,6 @@ trace.debug.enablebooleanfalseif set, traces for recent requests can be seen in the /debug page trace.lightstep.tokenstringif set, traces go to Lightstep using this token trace.zipkin.collectorstringif set, traces go to the given Zipkin instance (example: '127.0.0.1:9411'); ignored if trace.lightstep.token is set -versioncustom validation19.2-11set the active cluster version in the format '.' +versioncustom validation19.2-12set the active cluster version in the format '.' diff --git a/pkg/ccl/backupccl/backup.go b/pkg/ccl/backupccl/backup.go index 6dfa351f2b83..1db84c223e9f 100644 --- a/pkg/ccl/backupccl/backup.go +++ b/pkg/ccl/backupccl/backup.go @@ -20,6 +20,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/build" "github.com/cockroachdb/cockroach/pkg/ccl/utilccl" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/jobs" @@ -1098,7 +1099,7 @@ func backupPlanHook( return err } if len(to) > 1 && - !cluster.Version.IsActive(ctx, p.ExecCfg().Settings, cluster.VersionPartitionedBackup) { + !p.ExecCfg().Settings.Version.IsActive(ctx, clusterversion.VersionPartitionedBackup) { return errors.Errorf("partitioned backups can only be made on a cluster that has been fully upgraded to version 19.2") } diff --git a/pkg/ccl/importccl/import_stmt.go b/pkg/ccl/importccl/import_stmt.go index 64f617a7a266..fdbe5dd5c7ce 100644 --- a/pkg/ccl/importccl/import_stmt.go +++ b/pkg/ccl/importccl/import_stmt.go @@ -17,6 +17,7 @@ import ( "strings" "github.com/cockroachdb/cockroach/pkg/ccl/backupccl" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/jobs" "github.com/cockroachdb/cockroach/pkg/jobs/jobspb" @@ -151,7 +152,7 @@ func importPlanHook( return nil, nil, nil, false, nil } - if !cluster.Version.IsActive(ctx, p.ExecCfg().Settings, cluster.VersionPartitionedBackup) { + if !p.ExecCfg().Settings.Version.IsActive(ctx, clusterversion.VersionPartitionedBackup) { return nil, nil, nil, false, errors.Errorf("IMPORT requires a cluster fully upgraded to version >= 19.2") } diff --git a/pkg/cli/context.go b/pkg/cli/context.go index 1b80ba0bbb2f..aa42c23d8510 100644 --- a/pkg/cli/context.go +++ b/pkg/cli/context.go @@ -24,6 +24,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/storage/engine" "github.com/cockroachdb/cockroach/pkg/util/log" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/mattn/go-isatty" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -33,7 +34,7 @@ import ( // parameters for CLI utilities (other than `cockroach start`, which // constructs a proper server.Config for the newly created server). var serverCfg = func() server.Config { - st := cluster.MakeClusterSettings(cluster.BinaryMinimumSupportedVersion, cluster.BinaryServerVersion) + st := cluster.MakeClusterSettings(clusterversion.BinaryMinimumSupportedVersion, clusterversion.BinaryServerVersion) settings.SetCanonicalValuesContainer(&st.SV) s := server.MakeConfig(context.Background(), st) diff --git a/pkg/clusterversion/cluster_version.go b/pkg/clusterversion/cluster_version.go index 31836a0c39ca..ee9bfe62a365 100644 --- a/pkg/clusterversion/cluster_version.go +++ b/pkg/clusterversion/cluster_version.go @@ -8,9 +8,41 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. +// Package clusterversion is a logical version that allows for backward +// incompatible behaviors. +// +// Ideally, every code change in a database would be backward compatible, but +// this is not always possible. Some features, fixes, or cleanups need to +// introduce a backward incompatiblity and others are dramatically simplified by +// it. This package provides a way to do this safely with (hopefully) minimal +// disruption. It works as follows: +// +// - Each node in the cluster is running a binary that was released as some +// version. We allow for rolling upgrades, so two nodes in the cluster may be +// running different binary versions. All nodes in a given cluster must be +// within 1 major release of each other (i.e. to upgrade two major releases, +// the cluster must first be rolled onto X+1 and then onto X+2). +// - Separate from the build versions of the binaries, the cluster itself has a +// logical "cluster version". This is used for two related things: first as a +// promise from the user that they'll never downgrade any nodes in the cluster +// to a binary below some version and second to unlock features that are +// backward compatible (which is now safe given that the old binary will never +// be used). +// - Each binary can operation within a range of some cluster versions. When a +// cluster is initializes, the binary doing the initialization uses the upper +// end of its supported range as the initial cluster version. Each node that +// joins this cluster then must be compatible with this cluster version. +// +// This package handles the feature gates and so must maintain a fairly +// lightweight set of dependencies. The migration sub-package handles advancing +// a cluster from one version to a later one. package clusterversion -import "github.com/cockroachdb/cockroach/pkg/roachpb" +import ( + "context" + + "github.com/cockroachdb/cockroach/pkg/roachpb" +) // IsActiveVersion returns true if the features of the supplied version are active at the running // version. @@ -28,3 +60,12 @@ func (cv ClusterVersion) IsActive(versionKey VersionKey) bool { func (cv ClusterVersion) String() string { return cv.Version.String() } + +// Handle is a read-only view +type Handle interface { + ActiveVersion(context.Context) ClusterVersion + ActiveVersionOrEmpty(context.Context) ClusterVersion + IsActive(context.Context, VersionKey) bool + BinaryVersion() roachpb.Version + BinaryMinSupportedVersion() roachpb.Version +} diff --git a/pkg/clusterversion/cockroach_versions.go b/pkg/clusterversion/cockroach_versions.go index 7a327d8fb691..cd968bfbcbf8 100644 --- a/pkg/clusterversion/cockroach_versions.go +++ b/pkg/clusterversion/cockroach_versions.go @@ -10,7 +10,13 @@ package clusterversion -import "github.com/cockroachdb/cockroach/pkg/roachpb" +import ( + "context" + + "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/util/syncutil" + "github.com/cockroachdb/errors" +) // VersionKey is a unique identifier for a version of CockroachDB. type VersionKey int @@ -56,6 +62,7 @@ const ( VersionPrimaryKeyColumnsOutOfFamilyZero VersionRootPassword VersionNoExplicitForeignKeyIndexIDs + VersionFoo // Add new versions here (step one of two). ) @@ -399,6 +406,12 @@ var versionsSingleton = keyedVersions([]keyedVersion{ Version: roachpb.Version{Major: 19, Minor: 2, Unstable: 11}, }, + { + Key: VersionFoo, + Version: roachpb.Version{Major: 19, Minor: 2, Unstable: 12}, + Hook: preHook, + }, + // Add new versions here (step two of two). }) @@ -425,5 +438,51 @@ var ( // VersionByKey returns the roachpb.Version for a given key. // It is a fatal error to use an invalid key. func VersionByKey(key VersionKey) roachpb.Version { - return versionsSingleton.MustByKey(key) + v, _ := VersionAndHookByKey(key) + return v +} + +// VersionAndHookByKey returns the roachpb.Version and (optional) Hook for a +// given key. It is a fatal error to use an invalid key. +func VersionAndHookByKey(key VersionKey) (roachpb.Version, HookFn) { + v := versionsSingleton.MustByKey(key) + hookRegistry.mu.Lock() + defer hookRegistry.mu.Unlock() + hook := hookRegistry.mu.hooks[key] + switch v.Hook { + case noHook: + if hook != nil { + panic(errors.AssertionFailedf(`unexpected hook for key: %s`, key)) + } + case preHook: + if hook == nil { + panic(errors.AssertionFailedf(`missing hook or key: %s`, key)) + } + } + return v.Version, hook +} + +// HookArgs is a marker for WIP. This is a low level package that can't have the +// deps of WIP. +type HookArgs interface{} + +type HookFn func(context.Context, HookArgs) error + +var hookRegistry struct { + mu struct { + syncutil.Mutex + hooks map[VersionKey]HookFn + } +} + +func RegisterHook(key VersionKey, hook HookFn) { + hookRegistry.mu.Lock() + defer hookRegistry.mu.Unlock() + if hookRegistry.mu.hooks == nil { + hookRegistry.mu.hooks = make(map[VersionKey]HookFn) + } + if _, ok := hookRegistry.mu.hooks[key]; ok { + panic(errors.AssertionFailedf(`multiple hooks registered for key: %s`, key)) + } + hookRegistry.mu.hooks[key] = hook } diff --git a/pkg/clusterversion/everynode/everynode.go b/pkg/clusterversion/everynode/everynode.go new file mode 100644 index 000000000000..0234f05214f2 --- /dev/null +++ b/pkg/clusterversion/everynode/everynode.go @@ -0,0 +1,221 @@ +// Copyright 2020 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +// Package everynode provides a mechanism for running an idempotent closure on +// each engine in the cluster during upgrades. This is written as a general +// mechanism but primarily intended for use in pkg/clusterversion/migration +// hooks. +// +// The primary external interface to this package is RunHookOnEveryNode. This +// method ensures that the requested hook (and all previous hooks) are run on +// the engines of every node in the cluster. In this context, "every node" means +// all nodes that have ever joined the cluster and haven't finished being +// decommissioned. This means that RunHookOnEveryNode will block if there is any +// node unavailibility, it is the responsibility of the cluster operator to +// ensure that all nodes are available during an upgrade. +// +// Before running the hooks on each node, this new low-water mark for required +// hooks is written to replicated kv. Then a synchronous RPC (`HookService` +// implemented in this package by `*Server`) is used to distribute the work. +// After an individual node runs the hook but before the RPC returns, the new +// hook high-water is persisted on each engine. When a node starts (is added, +// restarts, or is recommissioned), it looks up the required low-water. Then for +// each engine, the high-water is read and any necessary hooks are run before +// the node is allowed to join the cluster. (The per-engine high-water is an +// optimization, the hooks must be idempotent, so it would be correct but +// unecessary to run them every time a node started.) +package everynode + +import ( + "context" + "fmt" + "sync/atomic" + + "github.com/cockroachdb/cockroach/pkg/gossip" + "github.com/cockroachdb/cockroach/pkg/internal/client" + "github.com/cockroachdb/cockroach/pkg/keys" + "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/rpc" + "github.com/cockroachdb/cockroach/pkg/storage/engine" + "github.com/cockroachdb/cockroach/pkg/util/ctxgroup" + "github.com/cockroachdb/cockroach/pkg/util/hlc" + "github.com/cockroachdb/cockroach/pkg/util/syncutil" + "github.com/cockroachdb/errors" +) + +// HookRunFn is an idempotent function. +type HookRunFn func(context.Context, engine.Engine) error + +// RunHookOnEveryNode executes the requested hook closure on each node in the +// cluster and each node added to the cluster in the future. If no error is +// returned, the caller is guaranteed that no node will ever again connect to +// the cluster without first having run this hook. +// +// This is used during cluster version upgrades and requires that each node in +// the cluster be available. It blocks until each node has been contacted via +// RPC and returned a successful response. +// +// WIP It's currently "legal" for a node to be decommissioned and later +// re-added. If we wanted to prevent this, we could add a NodeID graveyard +// against which incoming Connect calls are checked. Probably better to instead +// have these node do the right thing on startup (get the current list of +// required hooks from Connect and run anything that hasn't been run). +func RunHookOnEveryNode( + ctx context.Context, + db *client.DB, + gossip *gossip.Gossip, + rpcCtx *rpc.Context, + key HookKey, + progressFn func(string), +) error { + if err := updateClusterMinimum(ctx, db, key); err != nil { + return err + } + + // WIP: Get every non-decommissioned node. Can we do this transactionally with + // the updateClusterMinimum call above? + var nodeIDs []roachpb.NodeID + + var finishedAtomic int64 + g := ctxgroup.WithContext(ctx) + for i := range nodeIDs { + nodeID := nodeIDs[i] + g.GoCtx(func(ctx context.Context) error { + defer func() { + finished := atomic.AddInt64(&finishedAtomic, 1) + // WIP there should be some sort of rate limiting here + progressFn(fmt.Sprintf(`finished hook on %d of %d nodes`, finished, len(nodeIDs))) + }() + + // WIP: How should retries work? + // + // WIP: If something gets stuck, we need to be able to push the status + // back up to the job so the user can see what's blocking the upgrade. + address, err := gossip.GetNodeIDAddress(nodeID) + if err != nil { + return err + } + conn, err := rpcCtx.GRPCDialNode(address.String(), nodeID, rpc.SystemClass).Connect(ctx) + if err != nil { + return err + } + client := NewHookClient(conn) + // WIP: What sort of cleanup is needed for conn and client? + res, err := client.RunHook(ctx, &RunHookRequest{Key: key}) + if err != nil { + return err + } + if res.Err != nil { + return errors.DecodeError(ctx, *res.Err) + } + return nil + }) + } + return g.Wait() +} + +func updateClusterMinimum(ctx context.Context, db *client.DB, key HookKey) error { + err := db.Put(ctx, keys.EveryNodeRequiredMinimum, int(key)) + return errors.Wrapf(err, "persisting new cluster-level everynode required minimum: %s", key) +} + +// RunHooksOnThisNode executes every hook up to and including the requested one. +func RunHooksOnThisNode(ctx context.Context, engs []engine.Engine, key HookKey) error { + g := ctxgroup.WithContext(ctx) + for i := range engs { + eng := engs[i] + g.GoCtx(func(ctx context.Context) error { + return runHooksOnThisEngine(ctx, eng, key) + }) + } + return g.Wait() +} + +func runHooksOnThisEngine(ctx context.Context, eng engine.Engine, target HookKey) error { + finishedKey := keys.MakeStoreKey(keys.LocalStoreFinishedEveryNodeSuffix, nil) + finishedVal, _, err := engine.MVCCGet( + ctx, eng, finishedKey, hlc.Timestamp{}, engine.MVCCGetOptions{}) + if err != nil { + return err + } + finished, err := finishedVal.GetInt() + if err != nil { + return err + } + + current := HookKey(finished) + if current > target { + // Fast path, skip writing our progress. + return nil + } + for ; current < target; current++ { + registry.mu.Lock() + hookRunFn, ok := registry.mu.hooks[current] + registry.mu.Unlock() + if !ok { + return errors.Errorf(`could not find hook for key: %s`, current) + } + if err := hookRunFn(ctx, eng); err != nil { + return errors.Wrapf(err, `running hook: %s`, current) + } + } + + // Save the progress on this engine so we don't run the hooks needlessly. + finishedVal.ClearChecksum() + finishedVal.SetInt(int64(target)) + if err := engine.MVCCPut( + ctx, eng, nil /* ms */, finishedKey, hlc.Timestamp{}, *finishedVal, nil, /* txn */ + ); err != nil { + return err + } + return nil +} + +var registry struct { + mu struct { + syncutil.Mutex + hooks map[HookKey]HookRunFn + } +} + +// Register is called in an init func to add a hook implementation for use with +// RunHookOnEveryNode. +func Register(key HookKey, fn HookRunFn) { + registry.mu.Lock() + defer registry.mu.Unlock() + if registry.mu.hooks == nil { + registry.mu.hooks = make(map[HookKey]HookRunFn) + } + if _, ok := registry.mu.hooks[key]; ok { + panic(errors.AssertionFailedf(`multiple hooks registerd for key: %s`, key)) + } + registry.mu.hooks[key] = fn +} + +type Server struct { + engs []engine.Engine +} + +var _ HookServer = (*Server)(nil) + +func NewServer(engs []engine.Engine) *Server { + return &Server{engs: engs} +} + +func (s *Server) RunHook( + ctx context.Context, req *RunHookRequest, +) (*RunHookResponse, error) { + err := RunHooksOnThisNode(ctx, s.engs, req.Key) + if err != nil { + resErr := errors.EncodeError(ctx, err) + return &RunHookResponse{Err: &resErr}, nil + } + return &RunHookResponse{}, nil +} diff --git a/pkg/clusterversion/everynode/everynode.pb.go b/pkg/clusterversion/everynode/everynode.pb.go new file mode 100644 index 000000000000..74ab6c3b426e --- /dev/null +++ b/pkg/clusterversion/everynode/everynode.pb.go @@ -0,0 +1,579 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: migration/everynode/everynode.proto + +package everynode + +import proto "github.com/gogo/protobuf/proto" +import fmt "fmt" +import math "math" +import errorspb "github.com/cockroachdb/errors/errorspb" + +import ( + context "context" + grpc "google.golang.org/grpc" +) + +import io "io" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package + +type HookKey int32 + +const ( + HookKey_Version_20_1 HookKey = 0 + HookKey_MigrateRaftLog HookKey = 1 + HookKey_Version_20_2 HookKey = 2 +) + +var HookKey_name = map[int32]string{ + 0: "Version_20_1", + 1: "MigrateRaftLog", + 2: "Version_20_2", +} +var HookKey_value = map[string]int32{ + "Version_20_1": 0, + "MigrateRaftLog": 1, + "Version_20_2": 2, +} + +func (x HookKey) String() string { + return proto.EnumName(HookKey_name, int32(x)) +} +func (HookKey) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_everynode_1aa65deb936ac311, []int{0} +} + +type RunHookRequest struct { + Key HookKey `protobuf:"varint,1,opt,name=key,proto3,enum=cockroach.everynode.HookKey" json:"key,omitempty"` +} + +func (m *RunHookRequest) Reset() { *m = RunHookRequest{} } +func (m *RunHookRequest) String() string { return proto.CompactTextString(m) } +func (*RunHookRequest) ProtoMessage() {} +func (*RunHookRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_everynode_1aa65deb936ac311, []int{0} +} +func (m *RunHookRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *RunHookRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil +} +func (dst *RunHookRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_RunHookRequest.Merge(dst, src) +} +func (m *RunHookRequest) XXX_Size() int { + return m.Size() +} +func (m *RunHookRequest) XXX_DiscardUnknown() { + xxx_messageInfo_RunHookRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_RunHookRequest proto.InternalMessageInfo + +type RunHookResponse struct { + Err *errorspb.EncodedError `protobuf:"bytes,1,opt,name=err,proto3" json:"err,omitempty"` +} + +func (m *RunHookResponse) Reset() { *m = RunHookResponse{} } +func (m *RunHookResponse) String() string { return proto.CompactTextString(m) } +func (*RunHookResponse) ProtoMessage() {} +func (*RunHookResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_everynode_1aa65deb936ac311, []int{1} +} +func (m *RunHookResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *RunHookResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil +} +func (dst *RunHookResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_RunHookResponse.Merge(dst, src) +} +func (m *RunHookResponse) XXX_Size() int { + return m.Size() +} +func (m *RunHookResponse) XXX_DiscardUnknown() { + xxx_messageInfo_RunHookResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_RunHookResponse proto.InternalMessageInfo + +func init() { + proto.RegisterType((*RunHookRequest)(nil), "cockroach.everynode.RunHookRequest") + proto.RegisterType((*RunHookResponse)(nil), "cockroach.everynode.RunHookResponse") + proto.RegisterEnum("cockroach.everynode.HookKey", HookKey_name, HookKey_value) +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// HookClient is the client API for Hook service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type HookClient interface { + RunHook(ctx context.Context, in *RunHookRequest, opts ...grpc.CallOption) (*RunHookResponse, error) +} + +type hookClient struct { + cc *grpc.ClientConn +} + +func NewHookClient(cc *grpc.ClientConn) HookClient { + return &hookClient{cc} +} + +func (c *hookClient) RunHook(ctx context.Context, in *RunHookRequest, opts ...grpc.CallOption) (*RunHookResponse, error) { + out := new(RunHookResponse) + err := c.cc.Invoke(ctx, "/cockroach.everynode.Hook/RunHook", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// HookServer is the server API for Hook service. +type HookServer interface { + RunHook(context.Context, *RunHookRequest) (*RunHookResponse, error) +} + +func RegisterHookServer(s *grpc.Server, srv HookServer) { + s.RegisterService(&_Hook_serviceDesc, srv) +} + +func _Hook_RunHook_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RunHookRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(HookServer).RunHook(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/cockroach.everynode.Hook/RunHook", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(HookServer).RunHook(ctx, req.(*RunHookRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _Hook_serviceDesc = grpc.ServiceDesc{ + ServiceName: "cockroach.everynode.Hook", + HandlerType: (*HookServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "RunHook", + Handler: _Hook_RunHook_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "migration/everynode/everynode.proto", +} + +func (m *RunHookRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RunHookRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Key != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintEverynode(dAtA, i, uint64(m.Key)) + } + return i, nil +} + +func (m *RunHookResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RunHookResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Err != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintEverynode(dAtA, i, uint64(m.Err.Size())) + n1, err := m.Err.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n1 + } + return i, nil +} + +func encodeVarintEverynode(dAtA []byte, offset int, v uint64) int { + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return offset + 1 +} +func (m *RunHookRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Key != 0 { + n += 1 + sovEverynode(uint64(m.Key)) + } + return n +} + +func (m *RunHookResponse) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Err != nil { + l = m.Err.Size() + n += 1 + l + sovEverynode(uint64(l)) + } + return n +} + +func sovEverynode(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozEverynode(x uint64) (n int) { + return sovEverynode(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *RunHookRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEverynode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RunHookRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RunHookRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) + } + m.Key = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEverynode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Key |= (HookKey(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipEverynode(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthEverynode + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RunHookResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEverynode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RunHookResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RunHookResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Err", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEverynode + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthEverynode + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Err == nil { + m.Err = &errorspb.EncodedError{} + } + if err := m.Err.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipEverynode(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthEverynode + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipEverynode(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowEverynode + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowEverynode + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowEverynode + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + iNdEx += length + if length < 0 { + return 0, ErrInvalidLengthEverynode + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowEverynode + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipEverynode(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthEverynode = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowEverynode = fmt.Errorf("proto: integer overflow") +) + +func init() { + proto.RegisterFile("migration/everynode/everynode.proto", fileDescriptor_everynode_1aa65deb936ac311) +} + +var fileDescriptor_everynode_1aa65deb936ac311 = []byte{ + // 300 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x90, 0x4f, 0x4b, 0xc3, 0x30, + 0x18, 0xc6, 0x13, 0x27, 0x0e, 0xa3, 0xcc, 0x12, 0x15, 0x64, 0x48, 0x18, 0x9b, 0x07, 0x51, 0xc8, + 0x34, 0x7e, 0x01, 0x15, 0x06, 0x82, 0x7a, 0xe9, 0x61, 0x07, 0x0f, 0x8e, 0xad, 0x7b, 0xad, 0xa5, + 0x98, 0xb7, 0xa6, 0x9d, 0xd0, 0x6f, 0xe1, 0xc7, 0xda, 0x71, 0xc7, 0x1d, 0xb5, 0xfd, 0x22, 0x92, + 0xb6, 0xd4, 0x3f, 0x0c, 0x6f, 0x4f, 0x9b, 0x5f, 0x7e, 0x79, 0xde, 0x97, 0xf5, 0x5e, 0x02, 0xdf, + 0x8c, 0x93, 0x00, 0x75, 0x1f, 0xde, 0xc0, 0xa4, 0x1a, 0xa7, 0xf0, 0x9d, 0x64, 0x64, 0x30, 0x41, + 0xbe, 0xeb, 0xa1, 0x17, 0x1a, 0x1c, 0x7b, 0xcf, 0xb2, 0x3e, 0x6a, 0xef, 0x83, 0x31, 0x68, 0xe2, + 0x68, 0xd2, 0x2f, 0x43, 0xc9, 0xb6, 0xf7, 0x7c, 0xf4, 0xb1, 0x88, 0x7d, 0x9b, 0xca, 0xbf, 0xdd, + 0x4b, 0xd6, 0x72, 0x67, 0xfa, 0x06, 0x31, 0x74, 0xe1, 0x75, 0x06, 0x71, 0xc2, 0x25, 0x6b, 0x84, + 0x90, 0x1e, 0xd0, 0x0e, 0x3d, 0x6e, 0xa9, 0x43, 0xb9, 0xe2, 0x05, 0x69, 0xf1, 0x5b, 0x48, 0x5d, + 0x0b, 0x76, 0x07, 0x6c, 0xa7, 0x36, 0xc4, 0x11, 0xea, 0x18, 0xb8, 0x62, 0x0d, 0x30, 0xa6, 0x50, + 0x6c, 0xa9, 0xce, 0x4f, 0x45, 0xd5, 0x4c, 0x0e, 0xb4, 0x87, 0x53, 0x98, 0x0e, 0xec, 0xb7, 0x6b, + 0xe1, 0x93, 0x2b, 0xd6, 0xac, 0xb4, 0xdc, 0x61, 0xdb, 0x43, 0x30, 0x71, 0x80, 0x7a, 0xa4, 0xce, + 0x46, 0xe7, 0x0e, 0xe1, 0x9c, 0xb5, 0xee, 0x8b, 0x75, 0x80, 0x3b, 0x7e, 0x4a, 0xee, 0xd0, 0x77, + 0xe8, 0x1f, 0x4a, 0x39, 0x6b, 0xea, 0x91, 0xad, 0x5b, 0x05, 0x1f, 0xb2, 0x66, 0xd5, 0x88, 0xf7, + 0x56, 0xf6, 0xff, 0x3d, 0x71, 0xfb, 0xe8, 0x7f, 0xa8, 0x1c, 0xaa, 0x4b, 0xae, 0x4f, 0xe7, 0x9f, + 0x82, 0xcc, 0x33, 0x41, 0x17, 0x99, 0xa0, 0xcb, 0x4c, 0xd0, 0x8f, 0x4c, 0xd0, 0xf7, 0x5c, 0x90, + 0x45, 0x2e, 0xc8, 0x32, 0x17, 0xe4, 0x61, 0xb3, 0xbe, 0x3e, 0xd9, 0x28, 0xf6, 0x7b, 0xf1, 0x15, + 0x00, 0x00, 0xff, 0xff, 0x06, 0x84, 0xce, 0xa6, 0xc8, 0x01, 0x00, 0x00, +} diff --git a/pkg/clusterversion/everynode/everynode.proto b/pkg/clusterversion/everynode/everynode.proto new file mode 100644 index 000000000000..f0a9e2b60bb5 --- /dev/null +++ b/pkg/clusterversion/everynode/everynode.proto @@ -0,0 +1,34 @@ +// Copyright 2020 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +syntax = "proto3"; +package cockroach.everynode; +option go_package = "everynode"; + +import "errorspb/errors.proto"; +import "gogoproto/gogo.proto"; + +enum HookKey { + Version_20_1 = 0; + MigrateRaftLog = 1; + Version_20_2 = 2; +} + +message RunHookRequest { + HookKey key = 1; +} + +message RunHookResponse { + errorspb.EncodedError err = 1; +} + +service Hook { + rpc RunHook (RunHookRequest) returns (RunHookResponse) {} +} diff --git a/pkg/clusterversion/keyed_versions.go b/pkg/clusterversion/keyed_versions.go index c037db9dfeae..d5ce03c27072 100644 --- a/pkg/clusterversion/keyed_versions.go +++ b/pkg/clusterversion/keyed_versions.go @@ -21,23 +21,31 @@ import ( "github.com/pkg/errors" ) +type hookType string + +const ( + noHook hookType = "" + preHook hookType = "preHook" +) + // keyedVersion associates a key to a version. type keyedVersion struct { Key VersionKey roachpb.Version + Hook hookType } // keyedVersions is a container for managing the versions of CockroachDB. type keyedVersions []keyedVersion // MustByKey asserts that the version specified by this key exists, and returns it. -func (kv keyedVersions) MustByKey(k VersionKey) roachpb.Version { +func (kv keyedVersions) MustByKey(k VersionKey) keyedVersion { key := int(k) if key >= len(kv) || key < 0 { log.Fatalf(context.Background(), "version with key %d does not exist, have:\n%s", key, pretty.Sprint(kv)) } - return kv[key].Version + return kv[key] } // Validate makes sure that the keyedVersions are sorted chronologically, that diff --git a/pkg/clusterversion/migration/migration.go b/pkg/clusterversion/migration/migration.go new file mode 100644 index 000000000000..050d4c7f0d34 --- /dev/null +++ b/pkg/clusterversion/migration/migration.go @@ -0,0 +1,218 @@ +// Copyright 2020 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +// Package migration contains mechanisms for advancing the pkg/clusterversion of +// a cluster. These clusterversions serve a dual purpose: first to act as a gate +// for new features and second to provide a low-water mark for backward +// compatiblity between the binaries of nodes in a cluster. +// +// To upgrade from X to X+1 call UpgradeCluster on a *Coordinator (X here is the +// versions as ordered by the VersionKey, so in practice this looks like an +// upgrade from vA.B.C to vA.B.C+1 or from vA.B.C to vA.B+1.0). This first +// updates the version on each node in the cluster such taht no `IsActive(X+1)` +// check will ever return false again on any node in this cluster. After this +// finishes, the version's idempotent migration hook is run (if there is one). +// This hook can and often will depend on the preceding `IsActive(X+1)` +// guarantee. +// +// The `IsActive(X+1)` guarantee is provided as follows. First the new version +// is written to replicated kv storage, so that all new nodes start with at +// least this version. Then it uses a synchronous rpc to each existing node to +// update the version on that node, which is persisted to all stores before +// returning. If any nodes are unavailable, this blocks until they're back or +// completely decomissioned. WIP what happens if a node starts with stores that +// don't have the same version? +package migration + +import ( + "context" + fmt "fmt" + + "github.com/cockroachdb/cockroach/pkg/clusterversion" + "github.com/cockroachdb/cockroach/pkg/gossip" + "github.com/cockroachdb/cockroach/pkg/internal/client" + "github.com/cockroachdb/cockroach/pkg/rpc" + "github.com/cockroachdb/cockroach/pkg/storage/engine" + "github.com/cockroachdb/errors" + + "github.com/cockroachdb/cockroach/pkg/roachpb" + + "github.com/cockroachdb/cockroach/pkg/keys" + "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/util/ctxgroup" + "github.com/cockroachdb/cockroach/pkg/util/log" + "github.com/cockroachdb/cockroach/pkg/util/syncutil" +) + +type Server struct { + engs []engine.Engine + mu struct { + syncutil.Mutex + current clusterversion.ClusterVersion + } +} + +func NewServer( + bootVersion clusterversion.ClusterVersion, engs []engine.Engine, +) (*Server, clusterversion.Handle) { + s := &Server{ + engs: engs, + } + s.mu.current = bootVersion + return s, s +} + +var _ ClusterVersionServer = (*Server)(nil) +var _ clusterversion.Handle = (*Server)(nil) + +// IsActive implements the Handle interface. +func (s *Server) IsActive(ctx context.Context, feature clusterversion.VersionKey) bool { + return s.ActiveVersion(ctx).IsActive(feature) +} + +// ActiveVersion implements the Handle interface. +func (s *Server) ActiveVersion(ctx context.Context) clusterversion.ClusterVersion { + ver := s.ActiveVersionOrEmpty(ctx) + if ver == (clusterversion.ClusterVersion{}) { + log.Fatalf(ctx, "version not initialized") + } + return ver +} + +// ActiveVersionOrEmpty implements the Handle interface. +func (s *Server) ActiveVersionOrEmpty(_ context.Context) clusterversion.ClusterVersion { + s.mu.Lock() + defer s.mu.Unlock() + return s.mu.current +} + +// BinaryVersion implements the Handle interface. +func (s *Server) BinaryVersion() roachpb.Version { + panic(`WIP refactor this out of cluster.Settings`) +} + +// BinaryMinSupportedVersion implements the Handle interface. +func (s *Server) BinaryMinSupportedVersion() roachpb.Version { + panic(`WIP refactor this out of cluster.Settings`) +} + +// ForwardNodeVersion implements the ClusterVersionServer interface. +func (s *Server) ForwardNodeVersion( + ctx context.Context, req *ForwardNodeVersionRequest, +) (*ForwardNodeVersionResponse, error) { + err := s.forwardVersion(ctx, req.Version) + if err != nil { + resErr := errors.EncodeError(ctx, err) + return &ForwardNodeVersionResponse{Err: &resErr}, nil + } + return &ForwardNodeVersionResponse{}, nil +} + +func (s *Server) forwardVersion(ctx context.Context, target clusterversion.ClusterVersion) error { + // WIP: Validation. + s.mu.Lock() + defer s.mu.Unlock() + if s.mu.current.Less(target.Version) { + s.mu.current = target + if err := storage.WriteClusterVersionToEngines(ctx, s.engs, target); err != nil { + return err + } + } + return nil +} + +func (s *Server) NewCoordinator( + db *client.DB, rpcCtx *rpc.Context, gossip *gossip.Gossip, +) *Coordinator { + return &Coordinator{s: s, db: db, rpcCtx: rpcCtx, gossip: gossip} +} + +type Coordinator struct { + s *Server + db *client.DB + rpcCtx *rpc.Context + gossip *gossip.Gossip +} + +func (c *Coordinator) UpgradeCluster( + ctx context.Context, target clusterversion.ClusterVersion, progressFn func(string), +) error { + // WIP this is an awkward way to step through versions, expose something in + // the clusterversion pkg to make this less brittle + for key := clusterversion.Version19_1; ; key++ { + v, hookFn := clusterversion.VersionAndHookByKey(key) + if v.Less(target.Version) { + continue + } + if hookFn != nil { + progressFn(fmt.Sprintf(`running migration hook for %s`, v)) + if err := hookFn(ctx, nil /* WIP */); err != nil { + return err + } + } + progressFn(fmt.Sprintf(`activating %s on all nodes`, v)) + if err := c.communicateNewVersion(ctx, clusterversion.ClusterVersion{Version: v}); err != nil { + return err + } + // WIP don't use == here, need to generate a proto one + if v == target.Version { + break + } + } + return nil +} + +// communicateNewVersion migrates this cluster to activate the target features. +func (c *Coordinator) communicateNewVersion( + ctx context.Context, target clusterversion.ClusterVersion, +) error { + if err := updateClusterMinimum(ctx, c.db, target); err != nil { + return err + } + + // WIP: Get every non-decommissioned node. Can we do this transactionally with + // the updateClusterMinimum call above? + var nodeIDs []roachpb.NodeID + + g := ctxgroup.WithContext(ctx) + for i := range nodeIDs { + nodeID := nodeIDs[i] + g.GoCtx(func(ctx context.Context) error { + // WIP: How should retries work? + // + // WIP: If something gets stuck, we need to be able to push the status + // back up to the job so the user can see what's blocking the upgrade. + address, err := c.gossip.GetNodeIDAddress(nodeID) + if err != nil { + return err + } + conn, err := c.rpcCtx.GRPCDialNode(address.String(), nodeID, rpc.SystemClass).Connect(ctx) + if err != nil { + return err + } + client := NewClusterVersionClient(conn) + // WIP: What sort of cleanup is needed for conn and client? + res, err := client.ForwardNodeVersion(ctx, &ForwardNodeVersionRequest{Version: target}) + if err != nil { + return err + } + if res.Err != nil { + return errors.DecodeError(ctx, *res.Err) + } + return nil + }) + } + return g.Wait() +} + +func updateClusterMinimum(ctx context.Context, db *client.DB, version clusterversion.ClusterVersion) error { + err := db.Put(ctx, keys.ClusterVersionCurrent, version.Version.String()) + return errors.Wrapf(err, "persisting new cluster version: %s", version.Version) +} diff --git a/pkg/clusterversion/migration/migration.pb.go b/pkg/clusterversion/migration/migration.pb.go new file mode 100644 index 000000000000..13dbf1e9758b --- /dev/null +++ b/pkg/clusterversion/migration/migration.pb.go @@ -0,0 +1,566 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: clusterversion/migration/migration.proto + +package migration + +import proto "github.com/gogo/protobuf/proto" +import fmt "fmt" +import math "math" +import clusterversion "github.com/cockroachdb/cockroach/pkg/clusterversion" +import errorspb "github.com/cockroachdb/errors/errorspb" + +import ( + context "context" + grpc "google.golang.org/grpc" +) + +import io "io" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package + +type ForwardNodeVersionRequest struct { + Version clusterversion.ClusterVersion `protobuf:"bytes,1,opt,name=version,proto3" json:"version"` +} + +func (m *ForwardNodeVersionRequest) Reset() { *m = ForwardNodeVersionRequest{} } +func (m *ForwardNodeVersionRequest) String() string { return proto.CompactTextString(m) } +func (*ForwardNodeVersionRequest) ProtoMessage() {} +func (*ForwardNodeVersionRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_migration_c37d1ead68dc86d4, []int{0} +} +func (m *ForwardNodeVersionRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ForwardNodeVersionRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil +} +func (dst *ForwardNodeVersionRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ForwardNodeVersionRequest.Merge(dst, src) +} +func (m *ForwardNodeVersionRequest) XXX_Size() int { + return m.Size() +} +func (m *ForwardNodeVersionRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ForwardNodeVersionRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ForwardNodeVersionRequest proto.InternalMessageInfo + +type ForwardNodeVersionResponse struct { + Err *errorspb.EncodedError `protobuf:"bytes,1,opt,name=err,proto3" json:"err,omitempty"` +} + +func (m *ForwardNodeVersionResponse) Reset() { *m = ForwardNodeVersionResponse{} } +func (m *ForwardNodeVersionResponse) String() string { return proto.CompactTextString(m) } +func (*ForwardNodeVersionResponse) ProtoMessage() {} +func (*ForwardNodeVersionResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_migration_c37d1ead68dc86d4, []int{1} +} +func (m *ForwardNodeVersionResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ForwardNodeVersionResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil +} +func (dst *ForwardNodeVersionResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ForwardNodeVersionResponse.Merge(dst, src) +} +func (m *ForwardNodeVersionResponse) XXX_Size() int { + return m.Size() +} +func (m *ForwardNodeVersionResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ForwardNodeVersionResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_ForwardNodeVersionResponse proto.InternalMessageInfo + +func init() { + proto.RegisterType((*ForwardNodeVersionRequest)(nil), "cockroach.clusterversion.migration.ForwardNodeVersionRequest") + proto.RegisterType((*ForwardNodeVersionResponse)(nil), "cockroach.clusterversion.migration.ForwardNodeVersionResponse") +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// ClusterVersionClient is the client API for ClusterVersion service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type ClusterVersionClient interface { + ForwardNodeVersion(ctx context.Context, in *ForwardNodeVersionRequest, opts ...grpc.CallOption) (*ForwardNodeVersionResponse, error) +} + +type clusterVersionClient struct { + cc *grpc.ClientConn +} + +func NewClusterVersionClient(cc *grpc.ClientConn) ClusterVersionClient { + return &clusterVersionClient{cc} +} + +func (c *clusterVersionClient) ForwardNodeVersion(ctx context.Context, in *ForwardNodeVersionRequest, opts ...grpc.CallOption) (*ForwardNodeVersionResponse, error) { + out := new(ForwardNodeVersionResponse) + err := c.cc.Invoke(ctx, "/cockroach.clusterversion.migration.ClusterVersion/ForwardNodeVersion", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ClusterVersionServer is the server API for ClusterVersion service. +type ClusterVersionServer interface { + ForwardNodeVersion(context.Context, *ForwardNodeVersionRequest) (*ForwardNodeVersionResponse, error) +} + +func RegisterClusterVersionServer(s *grpc.Server, srv ClusterVersionServer) { + s.RegisterService(&_ClusterVersion_serviceDesc, srv) +} + +func _ClusterVersion_ForwardNodeVersion_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ForwardNodeVersionRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ClusterVersionServer).ForwardNodeVersion(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/cockroach.clusterversion.migration.ClusterVersion/ForwardNodeVersion", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ClusterVersionServer).ForwardNodeVersion(ctx, req.(*ForwardNodeVersionRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _ClusterVersion_serviceDesc = grpc.ServiceDesc{ + ServiceName: "cockroach.clusterversion.migration.ClusterVersion", + HandlerType: (*ClusterVersionServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "ForwardNodeVersion", + Handler: _ClusterVersion_ForwardNodeVersion_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "clusterversion/migration/migration.proto", +} + +func (m *ForwardNodeVersionRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ForwardNodeVersionRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + dAtA[i] = 0xa + i++ + i = encodeVarintMigration(dAtA, i, uint64(m.Version.Size())) + n1, err := m.Version.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n1 + return i, nil +} + +func (m *ForwardNodeVersionResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ForwardNodeVersionResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Err != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintMigration(dAtA, i, uint64(m.Err.Size())) + n2, err := m.Err.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n2 + } + return i, nil +} + +func encodeVarintMigration(dAtA []byte, offset int, v uint64) int { + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return offset + 1 +} +func (m *ForwardNodeVersionRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = m.Version.Size() + n += 1 + l + sovMigration(uint64(l)) + return n +} + +func (m *ForwardNodeVersionResponse) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Err != nil { + l = m.Err.Size() + n += 1 + l + sovMigration(uint64(l)) + } + return n +} + +func sovMigration(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozMigration(x uint64) (n int) { + return sovMigration(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *ForwardNodeVersionRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMigration + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ForwardNodeVersionRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ForwardNodeVersionRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMigration + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthMigration + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Version.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipMigration(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthMigration + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ForwardNodeVersionResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMigration + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ForwardNodeVersionResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ForwardNodeVersionResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Err", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMigration + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthMigration + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Err == nil { + m.Err = &errorspb.EncodedError{} + } + if err := m.Err.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipMigration(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthMigration + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipMigration(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowMigration + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowMigration + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowMigration + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + iNdEx += length + if length < 0 { + return 0, ErrInvalidLengthMigration + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowMigration + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipMigration(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthMigration = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowMigration = fmt.Errorf("proto: integer overflow") +) + +func init() { + proto.RegisterFile("clusterversion/migration/migration.proto", fileDescriptor_migration_c37d1ead68dc86d4) +} + +var fileDescriptor_migration_c37d1ead68dc86d4 = []byte{ + // 290 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xd2, 0x48, 0xce, 0x29, 0x2d, + 0x2e, 0x49, 0x2d, 0x2a, 0x4b, 0x2d, 0x2a, 0xce, 0xcc, 0xcf, 0xd3, 0xcf, 0xcd, 0x4c, 0x2f, 0x4a, + 0x2c, 0x41, 0x61, 0xe9, 0x15, 0x14, 0xe5, 0x97, 0xe4, 0x0b, 0x29, 0x25, 0xe7, 0x27, 0x67, 0x17, + 0xe5, 0x27, 0x26, 0x67, 0xe8, 0xa1, 0xea, 0xd1, 0x83, 0xab, 0x94, 0x12, 0x4d, 0x2d, 0x2a, 0xca, + 0x2f, 0x2a, 0x2e, 0x48, 0xd2, 0x87, 0x30, 0x20, 0x5a, 0xa5, 0x44, 0xd2, 0xf3, 0xd3, 0xf3, 0xc1, + 0x4c, 0x7d, 0x10, 0x0b, 0x2a, 0xaa, 0x82, 0x66, 0x35, 0x94, 0x1b, 0x0f, 0x33, 0x16, 0xac, 0x4a, + 0x29, 0x9a, 0x4b, 0xd2, 0x2d, 0xbf, 0xa8, 0x3c, 0xb1, 0x28, 0xc5, 0x2f, 0x3f, 0x25, 0x35, 0x0c, + 0x22, 0x17, 0x94, 0x5a, 0x58, 0x9a, 0x5a, 0x5c, 0x22, 0x64, 0xc7, 0xc5, 0x0e, 0x55, 0x2d, 0xc1, + 0xa8, 0xc0, 0xa8, 0xc1, 0x6d, 0x24, 0xa7, 0x87, 0x70, 0x65, 0x52, 0x62, 0x71, 0xaa, 0x9e, 0x33, + 0xc4, 0x50, 0xa8, 0x3e, 0x27, 0x96, 0x13, 0xf7, 0xe4, 0x19, 0x82, 0x60, 0x9a, 0x94, 0x02, 0xb8, + 0xa4, 0xb0, 0x19, 0x5e, 0x5c, 0x90, 0x9f, 0x57, 0x9c, 0x2a, 0x64, 0xc4, 0xc5, 0x9c, 0x5a, 0x54, + 0x04, 0x35, 0x59, 0x01, 0xc9, 0x64, 0x98, 0x2f, 0xf5, 0x5c, 0xf3, 0x92, 0xf3, 0x53, 0x52, 0x53, + 0x5c, 0x41, 0xfc, 0x20, 0x90, 0x62, 0xa3, 0x15, 0x8c, 0x5c, 0x7c, 0xa8, 0x76, 0x0a, 0x4d, 0x65, + 0xe4, 0x12, 0xc2, 0xb4, 0x45, 0xc8, 0x56, 0x8f, 0x70, 0x80, 0xea, 0xe1, 0xf4, 0xba, 0x94, 0x1d, + 0xb9, 0xda, 0x21, 0x9e, 0x53, 0x62, 0x70, 0xd2, 0x3e, 0xf1, 0x50, 0x8e, 0xe1, 0xc4, 0x23, 0x39, + 0xc6, 0x0b, 0x8f, 0xe4, 0x18, 0x6f, 0x3c, 0x92, 0x63, 0x7c, 0xf0, 0x48, 0x8e, 0x71, 0xc2, 0x63, + 0x39, 0x86, 0x0b, 0x8f, 0xe5, 0x18, 0x6e, 0x3c, 0x96, 0x63, 0x88, 0xe2, 0x84, 0x9b, 0x94, 0xc4, + 0x06, 0x8e, 0x0d, 0x63, 0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0x78, 0xf8, 0x1b, 0x1c, 0x30, 0x02, + 0x00, 0x00, +} diff --git a/pkg/clusterversion/migration/migration.proto b/pkg/clusterversion/migration/migration.proto new file mode 100644 index 000000000000..4175c771452b --- /dev/null +++ b/pkg/clusterversion/migration/migration.proto @@ -0,0 +1,29 @@ +// Copyright 2020 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +syntax = "proto3"; +package cockroach.clusterversion.migration; +option go_package = "migration"; + +import "errorspb/errors.proto"; +import "gogoproto/gogo.proto"; +import "clusterversion/cluster_version.proto"; + +message ForwardNodeVersionRequest { + base.ClusterVersion version = 1 [(gogoproto.nullable) = false]; +} + +message ForwardNodeVersionResponse { + errorspb.EncodedError err = 1; +} + +service ClusterVersion { + rpc ForwardNodeVersion (ForwardNodeVersionRequest) returns (ForwardNodeVersionResponse) {} +} diff --git a/pkg/keys/constants.go b/pkg/keys/constants.go index 29aaf3b033c7..1c9b8d77c756 100644 --- a/pkg/keys/constants.go +++ b/pkg/keys/constants.go @@ -262,6 +262,16 @@ var ( TimeseriesPrefix = roachpb.Key(makeKey(SystemPrefix, roachpb.RKey("tsd"))) // TimeseriesKeyMax is the maximum value for any timeseries data. TimeseriesKeyMax = TimeseriesPrefix.PrefixEnd() + // + // ClusterVersionPrefix specifies the key prefix to store all cluster version details. + ClusterVersionPrefix = roachpb.Key(makeKey(SystemPrefix, roachpb.RKey("cluster-version/"))) + // ClusterVersionCurrent WIP. It is not guaranteed that every node+store has + // this version saved locally. However, this is the verison handed to new + // nodes. To ensure that all nodes are at least a target version, use + // fflag.Server.ForwardCluster. + ClusterVersionCurrent = roachpb.Key(makeKey(ClusterVersionPrefix, roachpb.RKey("current"))) + // ClusterVersionKeyMax is the maximum value for any everynode key. + ClusterVersionKeyMax = ClusterVersionPrefix.PrefixEnd() // 3. SQL keys // diff --git a/pkg/kv/txn_interceptor_committer.go b/pkg/kv/txn_interceptor_committer.go index 122235828ffb..b0c69aaf2f14 100644 --- a/pkg/kv/txn_interceptor_committer.go +++ b/pkg/kv/txn_interceptor_committer.go @@ -279,7 +279,7 @@ func (tc *txnCommitter) sendLockedWithElidedEndTxn( func (tc *txnCommitter) canCommitInParallelWithWrites( ctx context.Context, ba roachpb.BatchRequest, et *roachpb.EndTxnRequest, ) bool { - if !cluster.Version.IsActive(ctx, tc.st, clusterversion.VersionParallelCommits) { + if !tc.st.Version.IsActive(ctx, clusterversion.VersionParallelCommits) { return false } if !parallelCommitsEnabled.Get(&tc.st.SV) { diff --git a/pkg/rpc/context.go b/pkg/rpc/context.go index d0f25e37a76c..725a452fb0fd 100644 --- a/pkg/rpc/context.go +++ b/pkg/rpc/context.go @@ -1067,7 +1067,7 @@ func (ctx *Context) runHeartbeat( MaxOffsetNanos: maxOffsetNanos, ClusterID: &clusterID, NodeID: conn.remoteNodeID, - ServerVersion: cluster.Version.BinaryVersion(ctx.settings), + ServerVersion: ctx.settings.Version.BinaryVersion(), } var response *PingResponse diff --git a/pkg/rpc/heartbeat.go b/pkg/rpc/heartbeat.go index 772bc70e215c..e2da72113bd8 100644 --- a/pkg/rpc/heartbeat.go +++ b/pkg/rpc/heartbeat.go @@ -79,7 +79,7 @@ func checkClusterName(clusterName string, peerName string) error { } func checkVersion(ctx context.Context, st *cluster.Settings, peerVersion roachpb.Version) error { - activeVersion := cluster.Version.ActiveVersionOrEmpty(ctx, st) + activeVersion := st.Version.ActiveVersionOrEmpty(ctx) if activeVersion == (clusterversion.ClusterVersion{}) { // Cluster version has not yet been determined. return nil @@ -159,7 +159,7 @@ func (hs *HeartbeatService) Ping(ctx context.Context, args *PingRequest) (*PingR return &PingResponse{ Pong: args.Ping, ServerTime: hs.clock.PhysicalNow(), - ServerVersion: cluster.Version.BinaryVersion(hs.settings), + ServerVersion: hs.settings.Version.BinaryVersion(), ClusterName: hs.clusterName, DisableClusterNameVerification: hs.disableClusterNameVerification, }, nil diff --git a/pkg/server/node.go b/pkg/server/node.go index 1730fd8b2b59..27ca3a6cfa4d 100644 --- a/pkg/server/node.go +++ b/pkg/server/node.go @@ -19,6 +19,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/build" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/config/zonepb" "github.com/cockroachdb/cockroach/pkg/gossip" @@ -214,7 +215,7 @@ func GetBootstrapSchema( func bootstrapCluster( ctx context.Context, engines []engine.Engine, - bootstrapVersion cluster.ClusterVersion, + bootstrapVersion clusterversion.ClusterVersion, defaultZoneConfig *zonepb.ZoneConfig, defaultSystemZoneConfig *zonepb.ZoneConfig, ) (uuid.UUID, error) { @@ -282,8 +283,8 @@ func NewNode( metrics: makeNodeMetrics(reg, cfg.HistogramWindowInterval), stores: storage.NewStores( cfg.AmbientCtx, cfg.Clock, - cluster.Version.BinaryMinSupportedVersion(cfg.Settings), - cluster.Version.BinaryVersion(cfg.Settings)), + cfg.Settings.Version.BinaryMinSupportedVersion(), + cfg.Settings.Version.BinaryVersion()), txnMetrics: txnMetrics, eventLogger: eventLogger, clusterID: clusterID, @@ -317,7 +318,7 @@ func (n *Node) AnnotateCtxWithSpan( func (n *Node) bootstrapCluster( ctx context.Context, engines []engine.Engine, - bootstrapVersion cluster.ClusterVersion, + bootstrapVersion clusterversion.ClusterVersion, defaultZoneConfig *zonepb.ZoneConfig, defaultSystemZoneConfig *zonepb.ZoneConfig, ) error { @@ -335,7 +336,7 @@ func (n *Node) bootstrapCluster( return nil } -func (n *Node) onClusterVersionChange(ctx context.Context, cv cluster.ClusterVersion) { +func (n *Node) onClusterVersionChange(ctx context.Context, cv clusterversion.ClusterVersion) { if err := n.stores.OnClusterVersionChange(ctx, cv); err != nil { log.Fatal(ctx, errors.Wrapf(err, "updating cluster version to %v", cv)) } @@ -353,14 +354,9 @@ func (n *Node) start( clusterName string, attrs roachpb.Attributes, locality roachpb.Locality, - cv cluster.ClusterVersion, localityAddress []roachpb.LocalityAddress, nodeDescriptorCallback func(descriptor roachpb.NodeDescriptor), ) error { - if err := cluster.Version.Initialize(ctx, cv.Version, n.storeCfg.Settings); err != nil { - return err - } - // Obtaining the NodeID requires a dance of sorts. If the node has initialized // stores, the NodeID is persisted in each of them. If not, then we'll need to // use the KV store to get a NodeID assigned. @@ -402,7 +398,7 @@ func (n *Node) start( Locality: locality, LocalityAddress: localityAddress, ClusterName: clusterName, - ServerVersion: cluster.Version.BinaryVersion(n.storeCfg.Settings), + ServerVersion: n.storeCfg.Settings.Version.BinaryVersion(), BuildTag: build.GetInfo().Tag, StartedAt: n.startedAt, } @@ -504,10 +500,10 @@ func (n *Node) start( // It's important that we persist new versions to the engines before the node // starts using it, otherwise the node might regress the version after a // crash. - cluster.Version.SetBeforeChange(ctx, n.storeCfg.Settings, n.onClusterVersionChange) + cluster.SetBeforeChangeVersion(ctx, n.storeCfg.Settings, n.onClusterVersionChange) // Invoke the callback manually once so that we persist the updated value that // gossip might have already received. - clusterVersion := cluster.Version.ActiveVersion(ctx, n.storeCfg.Settings) + clusterVersion := n.storeCfg.Settings.Version.ActiveVersion(ctx) n.onClusterVersionChange(ctx, clusterVersion) // Be careful about moving this line above `startStores`; store migrations rely @@ -557,7 +553,7 @@ func (n *Node) addStore(store *storage.Store) { if err != nil { log.Fatal(context.TODO(), err) } - if cv == (cluster.ClusterVersion{}) { + if cv == (clusterversion.ClusterVersion{}) { // The store should have had a version written to it during the store // bootstrap process. log.Fatal(context.TODO(), "attempting to add a store without a version") diff --git a/pkg/server/server.go b/pkg/server/server.go index a651df723c0f..6f8309d663ec 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -32,6 +32,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/blobs" "github.com/cockroachdb/cockroach/pkg/blobs/blobspb" + "github.com/cockroachdb/cockroach/pkg/clusterversion" + "github.com/cockroachdb/cockroach/pkg/clusterversion/migration" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/jobs" @@ -163,6 +165,7 @@ type Server struct { clock *hlc.Clock startTime time.Time rpcContext *rpc.Context + migration *migration.Server // The gRPC server on which the different RPC handlers will be registered. grpc *grpcServer gossip *gossip.Gossip @@ -211,6 +214,10 @@ func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) { } st := cfg.Settings + if st.Version != nil { + return nil, errors.New(`settings.Version was already set`) + } + st.Version = cluster.MakeLegacyVersionHandle(st) if cfg.AmbientCtx.Tracer == nil { panic(errors.New("no tracer set in AmbientCtx")) @@ -883,7 +890,7 @@ func inspectEngines( ) ( bootstrappedEngines []engine.Engine, emptyEngines []engine.Engine, - _ cluster.ClusterVersion, + _ clusterversion.ClusterVersion, _ error, ) { for _, engine := range engines { @@ -892,14 +899,14 @@ func inspectEngines( emptyEngines = append(emptyEngines, engine) continue } else if err != nil { - return nil, nil, cluster.ClusterVersion{}, err + return nil, nil, clusterversion.ClusterVersion{}, err } clusterID := clusterIDContainer.Get() if storeIdent.ClusterID != uuid.Nil { if clusterID == uuid.Nil { clusterIDContainer.Set(ctx, storeIdent.ClusterID) } else if storeIdent.ClusterID != clusterID { - return nil, nil, cluster.ClusterVersion{}, + return nil, nil, clusterversion.ClusterVersion{}, errors.Errorf("conflicting store cluster IDs: %s, %s", storeIdent.ClusterID, clusterID) } } @@ -908,7 +915,7 @@ func inspectEngines( cv, err := storage.SynthesizeClusterVersionFromEngines(ctx, bootstrappedEngines, minVersion, serverVersion) if err != nil { - return nil, nil, cluster.ClusterVersion{}, err + return nil, nil, clusterversion.ClusterVersion{}, err } return bootstrappedEngines, emptyEngines, cv, nil } @@ -1340,8 +1347,8 @@ func (s *Server) Start(ctx context.Context) error { bootstrappedEngines, _, _, err := inspectEngines( ctx, s.engines, - cluster.Version.BinaryMinSupportedVersion(s.cfg.Settings), - cluster.Version.BinaryVersion(s.cfg.Settings), + s.cfg.Settings.Version.BinaryMinSupportedVersion(), + s.cfg.Settings.Version.BinaryVersion(), &s.rpcContext.ClusterID) if err != nil { return errors.Wrap(err, "inspecting engines") @@ -1461,15 +1468,18 @@ func (s *Server) Start(ctx context.Context) error { // We ran this before, but might've bootstrapped in the meantime. This time // we'll get the actual list of bootstrapped and empty engines. - bootstrappedEngines, emptyEngines, cv, err := inspectEngines( + bootstrappedEngines, emptyEngines, bootVersion, err := inspectEngines( ctx, s.engines, - cluster.Version.BinaryMinSupportedVersion(s.cfg.Settings), - cluster.Version.BinaryVersion(s.cfg.Settings), + s.cfg.Settings.Version.BinaryMinSupportedVersion(), + s.cfg.Settings.Version.BinaryVersion(), &s.rpcContext.ClusterID) if err != nil { return errors.Wrap(err, "inspecting engines") } + // WIP move this way earlier in the boot process + s.migration, s.cfg.Settings.Version = migration.NewServer(bootVersion, s.engines) + // Record a walltime that is lower than the lowest hlc timestamp this current // instance of the node can use. We do not use startTime because it is lower // than the timestamp used to create the bootstrap schema. @@ -1485,7 +1495,6 @@ func (s *Server) Start(ctx context.Context) error { s.cfg.ClusterName, s.cfg.NodeAttributes, s.cfg.Locality, - cv, s.cfg.LocalityAddresses, s.execCfg.DistSQLPlanner.SetNodeDesc, ); err != nil { @@ -1967,7 +1976,7 @@ func (s *Server) startServeSQL( } func (s *Server) bootstrapVersion() roachpb.Version { - v := cluster.BinaryServerVersion + v := clusterversion.BinaryServerVersion if knobs := s.cfg.TestingKnobs.Server; knobs != nil { if ov := knobs.(*TestingKnobs).BootstrapVersionOverride; ov != (roachpb.Version{}) { v = ov @@ -1978,7 +1987,7 @@ func (s *Server) bootstrapVersion() roachpb.Version { func (s *Server) bootstrapCluster(ctx context.Context, bootstrapVersion roachpb.Version) error { if err := s.node.bootstrapCluster( - ctx, s.engines, cluster.ClusterVersion{Version: bootstrapVersion}, + ctx, s.engines, clusterversion.ClusterVersion{Version: bootstrapVersion}, &s.cfg.DefaultZoneConfig, &s.cfg.DefaultSystemZoneConfig, ); err != nil { return err diff --git a/pkg/server/testserver.go b/pkg/server/testserver.go index be11aacfcf7a..094606d9916a 100644 --- a/pkg/server/testserver.go +++ b/pkg/server/testserver.go @@ -23,6 +23,7 @@ import ( "github.com/cenkalti/backoff" circuit "github.com/cockroachdb/circuitbreaker" "github.com/cockroachdb/cockroach/pkg/base" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/config/zonepb" "github.com/cockroachdb/cockroach/pkg/gossip" @@ -104,7 +105,7 @@ func makeTestConfig(st *cluster.Settings) Config { func makeTestConfigFromParams(params base.TestServerArgs) Config { st := params.Settings if params.Settings == nil { - st = cluster.MakeClusterSettings(cluster.BinaryMinimumSupportedVersion, cluster.BinaryServerVersion) + st = cluster.MakeClusterSettings(clusterversion.BinaryMinimumSupportedVersion, clusterversion.BinaryServerVersion) } st.ExternalIODir = params.ExternalIODir cfg := makeTestConfig(st) diff --git a/pkg/settings/cluster/settings.go b/pkg/settings/cluster/settings.go index 793f4bb428bd..c2a1cfa0603d 100644 --- a/pkg/settings/cluster/settings.go +++ b/pkg/settings/cluster/settings.go @@ -53,6 +53,7 @@ type Settings struct { cpuProfiling int32 // atomic // Versions describing the range supported by this binary. + Version clusterversion.Handle binaryMinSupportedVersion roachpb.Version binaryServerVersion roachpb.Version beforeClusterVersionChangeMu struct { @@ -91,7 +92,7 @@ var NoSettings *Settings // = nil // KeyVersionSetting is the "version" settings key. const KeyVersionSetting = "version" -// Version represents the cluster's "active version". The active version is a +// version represents the cluster's "active version". The active version is a // cluster setting, but a special one. It can only advance to higher and higher // versions. The setting can be used to see if migrations are to be considered // enabled or disabled through the IsActive() method. @@ -118,7 +119,35 @@ const KeyVersionSetting = "version" // (typically a collection of `engine.Engine`s), which the caller will do by // registering itself via SetBeforeChange()`, which is invoked *before* exposing // the new version to callers of `IsActive()` and `Version()`. -var Version = registerClusterVersionSetting() +var version = registerClusterVersionSetting() + +type legacyVersionHandle struct { + settings *Settings +} + +var _ clusterversion.Handle = (*legacyVersionHandle)(nil) + +// MakeLegacyVersionHandle adapts the legacy settings-based ClusterVersion +// implementation into the new interface. +func MakeLegacyVersionHandle(settings *Settings) clusterversion.Handle { + return &legacyVersionHandle{settings: settings} +} + +func (h *legacyVersionHandle) ActiveVersion(ctx context.Context) clusterversion.ClusterVersion { + return version.ActiveVersion(ctx, h.settings) +} +func (h *legacyVersionHandle) ActiveVersionOrEmpty(ctx context.Context) clusterversion.ClusterVersion { + return version.ActiveVersionOrEmpty(ctx, h.settings) +} +func (h *legacyVersionHandle) IsActive(ctx context.Context, key clusterversion.VersionKey) bool { + return version.IsActive(ctx, h.settings, key) +} +func (h *legacyVersionHandle) BinaryVersion() roachpb.Version { + return h.settings.binaryServerVersion +} +func (h *legacyVersionHandle) BinaryMinSupportedVersion() roachpb.Version { + return h.settings.binaryMinSupportedVersion +} // registerClusterVersionSetting creates a clusterVersionSetting and registers // it with the cluster settings registry. @@ -144,7 +173,7 @@ var preserveDowngradeVersion = func() *settings.StringSetting { } opaque := sv.Opaque() st := opaque.(*Settings) - clusterVersion := Version.ActiveVersion(context.TODO(), st).Version + clusterVersion := version.ActiveVersion(context.TODO(), st).Version downgradeVersion, err := roachpb.ParseVersion(s) if err != nil { return err @@ -175,7 +204,7 @@ func MakeTestingClusterSettings() *Settings { func MakeTestingClusterSettingsWithVersion(minVersion, serverVersion roachpb.Version) *Settings { st := MakeClusterSettings(minVersion, serverVersion) // Initialize with all features enabled. - if err := Version.Initialize(context.TODO(), serverVersion, st); err != nil { + if err := version.Initialize(context.TODO(), serverVersion, st); err != nil { log.Fatalf(context.TODO(), "unable to initialize version: %s", err) } return st @@ -227,12 +256,11 @@ func makeClusterVersionSetting() clusterVersionSetting { return s } -func (cv clusterVersionSetting) BinaryVersion(st *Settings) roachpb.Version { - return st.binaryServerVersion -} - -func (cv clusterVersionSetting) BinaryMinSupportedVersion(st *Settings) roachpb.Version { - return st.binaryMinSupportedVersion +// InitializeVersion WIP +func InitializeVersion( + ctx context.Context, v roachpb.Version, st *Settings, +) error { + return version.Initialize(ctx, v, st) } // Initialize initializes cluster version. Before this method has been @@ -342,6 +370,13 @@ func (cv clusterVersionSetting) BeforeChange( st.beforeClusterVersionChangeMu.Unlock() } +// SetBeforeChangeVersion WIP +func SetBeforeChangeVersion( + ctx context.Context, st *Settings, cb func(ctx context.Context, newVersion clusterversion.ClusterVersion), +) { + version.SetBeforeChange(ctx, st, cb) +} + // SetBeforeChange registers a callback to be called before the cluster version // is updated. The new cluster version will only become "visible" after the // callback has returned. diff --git a/pkg/sql/alter_table.go b/pkg/sql/alter_table.go index caef4e9660a8..2b51866cfb01 100644 --- a/pkg/sql/alter_table.go +++ b/pkg/sql/alter_table.go @@ -19,7 +19,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/server/telemetry" - "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" "github.com/cockroachdb/cockroach/pkg/sql/privilege" @@ -314,7 +313,7 @@ func (n *alterTableNode) startExec(params runParams) error { case *tree.AlterTableAlterPrimaryKey: // Make sure that all nodes in the cluster are able to perform primary key changes before proceeding. - version := cluster.Version.ActiveVersionOrEmpty(params.ctx, params.p.ExecCfg().Settings) + version := params.p.ExecCfg().Settings.Version.ActiveVersionOrEmpty(params.ctx) if !version.IsActive(clusterversion.VersionPrimaryKeyChanges) { return pgerror.Newf(pgcode.FeatureNotSupported, "all nodes are not the correct version for primary key changes") diff --git a/pkg/sql/alter_user.go b/pkg/sql/alter_user.go index 9055a420780f..c495e1536b93 100644 --- a/pkg/sql/alter_user.go +++ b/pkg/sql/alter_user.go @@ -15,7 +15,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/security" - "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" "github.com/cockroachdb/cockroach/pkg/sql/privilege" @@ -69,7 +68,7 @@ func (n *alterUserSetPasswordNode) startExec(params runParams) error { // TODO(knz): Remove in 20.2. if normalizedUsername == security.RootUser && len(hashedPassword) > 0 && - !cluster.Version.IsActive(params.ctx, params.EvalContext().Settings, clusterversion.VersionRootPassword) { + !params.EvalContext().Settings.Version.IsActive(params.ctx, clusterversion.VersionRootPassword) { return pgerror.Newf(pgcode.ObjectNotInPrerequisiteState, `setting a root password requires all nodes to be upgraded to %s`, clusterversion.VersionByKey(clusterversion.VersionRootPassword), diff --git a/pkg/sql/backfill.go b/pkg/sql/backfill.go index f0832c68b85c..0a64fa5c0d83 100644 --- a/pkg/sql/backfill.go +++ b/pkg/sql/backfill.go @@ -23,7 +23,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings" - "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/backfill" "github.com/cockroachdb/cockroach/pkg/sql/execinfrapb" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" @@ -712,8 +711,8 @@ func (sc *SchemaChanger) distBackfill( filter backfill.MutationFilter, targetSpans []roachpb.Span, ) error { - inMemoryStatusEnabled := cluster.Version.IsActive( - ctx, sc.execCfg.Settings, clusterversion.VersionAtomicChangeReplicasTrigger) + inMemoryStatusEnabled := sc.execCfg.Settings.Version.IsActive( + ctx, clusterversion.VersionAtomicChangeReplicasTrigger) duration := checkpointInterval if sc.testingKnobs.WriteCheckpointInterval > 0 { duration = sc.testingKnobs.WriteCheckpointInterval diff --git a/pkg/sql/create_index.go b/pkg/sql/create_index.go index 2e98f0d7283e..33700aadd898 100644 --- a/pkg/sql/create_index.go +++ b/pkg/sql/create_index.go @@ -14,7 +14,6 @@ import ( "context" "github.com/cockroachdb/cockroach/pkg/clusterversion" - "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" "github.com/cockroachdb/cockroach/pkg/sql/privilege" @@ -112,7 +111,7 @@ func (n *createIndexNode) startExec(params runParams) error { // If all nodes in the cluster know how to handle secondary indexes with column families, // write the new version into the index descriptor. encodingVersion := sqlbase.BaseIndexFormatVersion - if cluster.Version.IsActive(params.ctx, params.p.EvalContext().Settings, clusterversion.VersionSecondaryIndexColumnFamilies) { + if params.p.EvalContext().Settings.Version.IsActive(params.ctx, clusterversion.VersionSecondaryIndexColumnFamilies) { encodingVersion = sqlbase.SecondaryIndexFamilyFormatVersion } indexDesc.Version = encodingVersion diff --git a/pkg/sql/create_table.go b/pkg/sql/create_table.go index 4dfd75a0b641..c91ea0040719 100644 --- a/pkg/sql/create_table.go +++ b/pkg/sql/create_table.go @@ -1162,7 +1162,7 @@ func MakeTableDesc( // before the version has been initialized, leading to a panic. There are also // cases where this function is called in tests where st is nil. if st != nil { - if version := cluster.Version.ActiveVersionOrEmpty(ctx, st); version != (clusterversion.ClusterVersion{}) && + if version := st.Version.ActiveVersionOrEmpty(ctx); version != (clusterversion.ClusterVersion{}) && version.IsActive(clusterversion.VersionSecondaryIndexColumnFamilies) { indexEncodingVersion = sqlbase.SecondaryIndexFamilyFormatVersion } @@ -1332,7 +1332,7 @@ func MakeTableDesc( // If any nodes are not at version VersionPrimaryKeyColumnsOutOfFamilyZero, then return an error // if a primary key column is not in column family 0. if st != nil { - if version := cluster.Version.ActiveVersionOrEmpty(ctx, st); version != (clusterversion.ClusterVersion{}) && + if version := st.Version.ActiveVersionOrEmpty(ctx); version != (clusterversion.ClusterVersion{}) && !version.IsActive(clusterversion.VersionPrimaryKeyColumnsOutOfFamilyZero) { var colsInFamZero util.FastIntSet for _, colID := range desc.Families[0].ColumnIDs { diff --git a/pkg/sql/descriptor.go b/pkg/sql/descriptor.go index eb71092b963d..5744b3ebf033 100644 --- a/pkg/sql/descriptor.go +++ b/pkg/sql/descriptor.go @@ -67,7 +67,7 @@ func (p *planner) createDatabase( // TODO(solon): This conditional can be removed in 20.2. Every database // is created with a public schema for cluster version >= 20.1, so we can remove // the `shouldCreatePublicSchema` logic as well. - if !cluster.Version.IsActive(ctx, p.ExecCfg().Settings, clusterversion.VersionNamespaceTableWithSchemas) { + if !p.ExecCfg().Settings.Version.IsActive(ctx, clusterversion.VersionNamespaceTableWithSchemas) { shouldCreatePublicSchema = false } diff --git a/pkg/sql/drop_index.go b/pkg/sql/drop_index.go index 9bf6fd81b0eb..d3c6c5f7a3c4 100644 --- a/pkg/sql/drop_index.go +++ b/pkg/sql/drop_index.go @@ -17,7 +17,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/roachpb" - "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" "github.com/cockroachdb/cockroach/pkg/sql/privilege" @@ -200,7 +199,7 @@ func (p *planner) dropIndexByName( // If we aren't at the cluster version where we have removed explicit foreign key IDs // from the foreign key descriptors, fall back to the existing drop index logic. // That means we pretend that we can never find replacements for any indexes. - if !cluster.Version.IsActive(ctx, p.ExecCfg().Settings, clusterversion.VersionNoExplicitForeignKeyIndexIDs) { + if !p.ExecCfg().Settings.Version.IsActive(ctx, clusterversion.VersionNoExplicitForeignKeyIndexIDs) { indexHasReplacementCandidate = func(func(*sqlbase.IndexDescriptor) bool) bool { return false } diff --git a/pkg/sql/pgwire/auth_methods.go b/pkg/sql/pgwire/auth_methods.go index 773faf179c6d..70f9c92cb0d7 100644 --- a/pkg/sql/pgwire/auth_methods.go +++ b/pkg/sql/pgwire/auth_methods.go @@ -16,8 +16,8 @@ import ( "crypto/tls" "fmt" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/security" - "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/hba" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" @@ -40,26 +40,26 @@ func loadDefaultMethods() { // // Care should be taken by administrators to only accept this auth // method over secure connections, e.g. those encrypted using SSL. - RegisterAuthMethod("password", authPassword, cluster.Version19_1, hba.ConnAny, nil) + RegisterAuthMethod("password", authPassword, clusterversion.Version19_1, hba.ConnAny, nil) // The "cert" method requires a valid client certificate for the // user attempting to connect. // // This method is only usable over SSL connections. - RegisterAuthMethod("cert", authCert, cluster.Version19_1, hba.ConnHostSSL, nil) + RegisterAuthMethod("cert", authCert, clusterversion.Version19_1, hba.ConnHostSSL, nil) // The "cert-password" method requires either a valid client // certificate for the connecting user, or, if no cert is provided, // a cleartext password. - RegisterAuthMethod("cert-password", authCertPassword, cluster.Version19_1, hba.ConnAny, nil) + RegisterAuthMethod("cert-password", authCertPassword, clusterversion.Version19_1, hba.ConnAny, nil) // The "reject" method rejects any connection attempt that matches // the current rule. - RegisterAuthMethod("reject", authReject, cluster.VersionAuthLocalAndTrustRejectMethods, hba.ConnAny, nil) + RegisterAuthMethod("reject", authReject, clusterversion.VersionAuthLocalAndTrustRejectMethods, hba.ConnAny, nil) // The "trust" method accepts any connection attempt that matches // the current rule. - RegisterAuthMethod("trust", authTrust, cluster.VersionAuthLocalAndTrustRejectMethods, hba.ConnAny, nil) + RegisterAuthMethod("trust", authTrust, clusterversion.VersionAuthLocalAndTrustRejectMethods, hba.ConnAny, nil) } diff --git a/pkg/sql/pgwire/hba_conf.go b/pkg/sql/pgwire/hba_conf.go index 1a4891e092a1..05bc93a2db8f 100644 --- a/pkg/sql/pgwire/hba_conf.go +++ b/pkg/sql/pgwire/hba_conf.go @@ -18,6 +18,7 @@ import ( "sort" "strings" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/security" "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/settings/cluster" @@ -144,10 +145,10 @@ func checkHBASyntaxBeforeUpdatingSetting(values *settings.Values, s string) erro case hba.ConnHostAny: case hba.ConnLocal: if st != nil && - !cluster.Version.IsActive(context.TODO(), st, cluster.VersionAuthLocalAndTrustRejectMethods) { + !st.Version.IsActive(context.TODO(), clusterversion.VersionAuthLocalAndTrustRejectMethods) { return pgerror.Newf(pgcode.ObjectNotInPrerequisiteState, `authentication rule type 'local' requires all nodes to be upgraded to %s`, - cluster.VersionByKey(cluster.VersionAuthLocalAndTrustRejectMethods), + clusterversion.VersionByKey(clusterversion.VersionAuthLocalAndTrustRejectMethods), ) } default: @@ -190,11 +191,11 @@ func checkHBASyntaxBeforeUpdatingSetting(values *settings.Values, s string) erro "Supported methods: %s", listRegisteredMethods()) } // Verify that the cluster setting is at least the required version. - if st != nil && !cluster.Version.IsActive(context.TODO(), st, method.minReqVersion) { + if st != nil && !st.Version.IsActive(context.TODO(), method.minReqVersion) { return pgerror.Newf(pgcode.ObjectNotInPrerequisiteState, `authentication method '%s' requires all nodes to be upgraded to %s`, entry.Method.Value, - cluster.VersionByKey(method.minReqVersion)) + clusterversion.VersionByKey(method.minReqVersion)) } // Run the per-method validation. if check := hbaCheckHBAEntries[entry.Method.Value]; check != nil { @@ -308,7 +309,7 @@ func (s *Server) GetAuthenticationConfiguration() *hba.Conf { func RegisterAuthMethod( method string, fn AuthMethod, - minReqVersion cluster.VersionKey, + minReqVersion clusterversion.VersionKey, validConnTypes hba.ConnType, checkEntry CheckHBAEntry, ) { @@ -336,7 +337,7 @@ var ( type authMethodEntry struct { methodInfo - minReqVersion cluster.VersionKey + minReqVersion clusterversion.VersionKey } type methodInfo struct { diff --git a/pkg/sql/plan.go b/pkg/sql/plan.go index e06fdf24db3b..9b42a56d6584 100644 --- a/pkg/sql/plan.go +++ b/pkg/sql/plan.go @@ -16,7 +16,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/roachpb" - "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sessiondata" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" @@ -70,8 +69,8 @@ func (r *runParams) creationTimeForNewTableDescriptor() hlc.Timestamp { // CreateAsOfTime and ModificationTime when creating a table descriptor and then // upon reading use the MVCC timestamp to populate the values. var ts hlc.Timestamp - if !cluster.Version.IsActive( - r.ctx, r.ExecCfg().Settings, clusterversion.VersionTableDescModificationTimeFromMVCC, + if !r.ExecCfg().Settings.Version.IsActive( + r.ctx, clusterversion.VersionTableDescModificationTimeFromMVCC, ) { ts = r.p.txn.CommitTimestamp() } diff --git a/pkg/sql/rowexec/backfiller.go b/pkg/sql/rowexec/backfiller.go index 20e82fab199c..ea957179b576 100644 --- a/pkg/sql/rowexec/backfiller.go +++ b/pkg/sql/rowexec/backfiller.go @@ -14,12 +14,11 @@ import ( "context" "fmt" - "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/clusterversion" + "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/jobs" "github.com/cockroachdb/cockroach/pkg/jobs/jobspb" "github.com/cockroachdb/cockroach/pkg/roachpb" - "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/backfill" "github.com/cockroachdb/cockroach/pkg/sql/execinfra" "github.com/cockroachdb/cockroach/pkg/sql/execinfrapb" @@ -138,7 +137,7 @@ func (b *backfiller) doRun(ctx context.Context) *execinfrapb.ProducerMetadata { return &execinfrapb.ProducerMetadata{Err: err} } st := b.flowCtx.Cfg.Settings - if !cluster.Version.IsActive(ctx, st, clusterversion.VersionAtomicChangeReplicasTrigger) { + if !st.Version.IsActive(ctx, clusterversion.VersionAtomicChangeReplicasTrigger) { // There is a node of older version which could be the coordinator. // So we communicate the finished work by writing to the jobs row. err = WriteResumeSpan(ctx, diff --git a/pkg/sql/sem/builtins/builtins.go b/pkg/sql/sem/builtins/builtins.go index 42aa83e88359..d5ffc88c46e5 100644 --- a/pkg/sql/sem/builtins/builtins.go +++ b/pkg/sql/sem/builtins/builtins.go @@ -3022,7 +3022,7 @@ may increase either contention or retry errors, or both.`, Types: tree.ArgTypes{}, ReturnType: tree.FixedReturnType(types.String), Fn: func(ctx *tree.EvalContext, args tree.Datums) (tree.Datum, error) { - v := cluster.Version.BinaryVersion(ctx.Settings).String() + v := ctx.Settings.Version.BinaryVersion().String() return tree.NewDString(v), nil }, Info: "Returns the version of CockroachDB this node is running.", diff --git a/pkg/sql/split.go b/pkg/sql/split.go index 65412d3ef2c1..d2ac2629bd06 100644 --- a/pkg/sql/split.go +++ b/pkg/sql/split.go @@ -15,7 +15,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/keys" - "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sqlbase" "github.com/cockroachdb/cockroach/pkg/storage/storagebase" @@ -42,7 +41,7 @@ type splitRun struct { func (n *splitNode) startExec(params runParams) error { st := params.EvalContext().Settings - stickyBitEnabled := cluster.Version.IsActive(params.ctx, st, clusterversion.VersionStickyBit) + stickyBitEnabled := st.Version.IsActive(params.ctx, clusterversion.VersionStickyBit) // TODO(jeffreyxiao): Remove this error, splitNode.force, and // experimental_force_split_at in v20.1. // This check is not intended to be foolproof. The setting could be outdated @@ -73,7 +72,7 @@ func (n *splitNode) Next(params runParams) (bool, error) { // TODO(jeffreyxiao): Remove this check in v20.1. // Don't set the manual flag if the cluster is not up-to-date. st := params.EvalContext().Settings - stickyBitEnabled := cluster.Version.IsActive(params.ctx, st, clusterversion.VersionStickyBit) + stickyBitEnabled := st.Version.IsActive(params.ctx, clusterversion.VersionStickyBit) expirationTime := hlc.Timestamp{} if stickyBitEnabled { expirationTime = n.expirationTime diff --git a/pkg/sql/sqlbase/metadata.go b/pkg/sql/sqlbase/metadata.go index 0165ea277d62..b641587246b7 100644 --- a/pkg/sql/sqlbase/metadata.go +++ b/pkg/sql/sqlbase/metadata.go @@ -15,13 +15,13 @@ import ( "fmt" "sort" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/config/zonepb" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/protoutil" - "github.com/cockroachdb/cockroach/pkg/clusterversion" ) var _ DescriptorProto = &DatabaseDescriptor{} @@ -256,7 +256,7 @@ func LookupSystemTableDescriptorID( } if settings != nil && - !cluster.Version.IsActive(ctx, settings, clusterversion.VersionNamespaceTableWithSchemas) && + !settings.Version.IsActive(ctx, clusterversion.VersionNamespaceTableWithSchemas) && tableName == NamespaceTable.Name { return DeprecatedNamespaceTable.ID } diff --git a/pkg/sql/sqlbase/namespace.go b/pkg/sql/sqlbase/namespace.go index d7959cb01c46..10ee8038ebcb 100644 --- a/pkg/sql/sqlbase/namespace.go +++ b/pkg/sql/sqlbase/namespace.go @@ -119,7 +119,7 @@ func MakeObjectNameKey( ctx context.Context, settings *cluster.Settings, parentID ID, parentSchemaID ID, name string, ) DescriptorKey { // TODO(solon): This if condition can be removed in 20.2 - if !cluster.Version.IsActive(ctx, settings, clusterversion.VersionNamespaceTableWithSchemas) { + if !settings.Version.IsActive(ctx, clusterversion.VersionNamespaceTableWithSchemas) { return NewDeprecatedTableKey(parentID, name) } var key DescriptorKey diff --git a/pkg/sql/sqlbase/structured.go b/pkg/sql/sqlbase/structured.go index 6554b0dec83b..dbc01c515ffd 100644 --- a/pkg/sql/sqlbase/structured.go +++ b/pkg/sql/sqlbase/structured.go @@ -17,6 +17,7 @@ import ( "strconv" "strings" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" @@ -34,7 +35,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/cockroachdb/errors" - "github.com/cockroachdb/cockroach/pkg/clusterversion" ) // ID, ColumnID, FamilyID, and IndexID are all uint32, but are each given a @@ -1491,7 +1491,7 @@ func (desc *MutableTableDescriptor) MaybeIncrementVersion( // // TODO(ajwerner): remove this check in 20.1. var modTime hlc.Timestamp - if !cluster.Version.IsActive(ctx, settings, clusterversion.VersionTableDescModificationTimeFromMVCC) { + if !settings.Version.IsActive(ctx, clusterversion.VersionTableDescModificationTimeFromMVCC) { modTime = txn.CommitTimestamp() } desc.ModificationTime = modTime diff --git a/pkg/sql/unsplit.go b/pkg/sql/unsplit.go index b659ef94f1e6..15ac62540423 100644 --- a/pkg/sql/unsplit.go +++ b/pkg/sql/unsplit.go @@ -15,7 +15,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/keys" - "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" @@ -39,7 +38,7 @@ type unsplitRun struct { func (n *unsplitNode) startExec(params runParams) error { st := params.EvalContext().Settings - stickyBitEnabled := cluster.Version.IsActive(params.ctx, st, clusterversion.VersionStickyBit) + stickyBitEnabled := st.Version.IsActive(params.ctx, clusterversion.VersionStickyBit) // TODO(jeffreyxiao): Remove this error in v20.1. if !stickyBitEnabled { return pgerror.Newf(pgcode.ObjectNotInPrerequisiteState, @@ -99,7 +98,7 @@ type unsplitAllRun struct { func (n *unsplitAllNode) startExec(params runParams) error { st := params.EvalContext().Settings - stickyBitEnabled := cluster.Version.IsActive(params.ctx, st, clusterversion.VersionStickyBit) + stickyBitEnabled := st.Version.IsActive(params.ctx, clusterversion.VersionStickyBit) // TODO(jeffreyxiao): Remove this error in v20.1. if !stickyBitEnabled { return pgerror.Newf(pgcode.ObjectNotInPrerequisiteState, diff --git a/pkg/sqlmigrations/migrations.go b/pkg/sqlmigrations/migrations.go index 569d9b1fb9b3..4e3ba89d76ce 100644 --- a/pkg/sqlmigrations/migrations.go +++ b/pkg/sqlmigrations/migrations.go @@ -17,6 +17,7 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/config/zonepb" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" @@ -191,35 +192,35 @@ var backwardCompatibleMigrations = []migrationDescriptor{ name: "create system.comment table", workFn: createCommentTable, // This migration has been introduced some time before 19.2. - includedInBootstrap: cluster.VersionByKey(cluster.Version19_2), + includedInBootstrap: clusterversion.VersionByKey(clusterversion.Version19_2), newDescriptorIDs: staticIDs(keys.CommentsTableID), }, { name: "create system.replication_constraint_stats table", workFn: createReplicationConstraintStatsTable, // This migration has been introduced some time before 19.2. - includedInBootstrap: cluster.VersionByKey(cluster.Version19_2), + includedInBootstrap: clusterversion.VersionByKey(clusterversion.Version19_2), newDescriptorIDs: staticIDs(keys.ReplicationConstraintStatsTableID), }, { name: "create system.replication_critical_localities table", workFn: createReplicationCriticalLocalitiesTable, // This migration has been introduced some time before 19.2. - includedInBootstrap: cluster.VersionByKey(cluster.Version19_2), + includedInBootstrap: clusterversion.VersionByKey(clusterversion.Version19_2), newDescriptorIDs: staticIDs(keys.ReplicationCriticalLocalitiesTableID), }, { name: "create system.reports_meta table", workFn: createReportsMetaTable, // This migration has been introduced some time before 19.2. - includedInBootstrap: cluster.VersionByKey(cluster.Version19_2), + includedInBootstrap: clusterversion.VersionByKey(clusterversion.Version19_2), newDescriptorIDs: staticIDs(keys.ReportsMetaTableID), }, { name: "create system.replication_stats table", workFn: createReplicationStatsTable, // This migration has been introduced some time before 19.2. - includedInBootstrap: cluster.VersionByKey(cluster.Version19_2), + includedInBootstrap: clusterversion.VersionByKey(clusterversion.Version19_2), newDescriptorIDs: staticIDs(keys.ReplicationStatsTableID), }, { @@ -236,7 +237,7 @@ var backwardCompatibleMigrations = []migrationDescriptor{ { // Introduced in v19.2. name: "change reports fields from timestamp to timestamptz", - includedInBootstrap: cluster.VersionByKey(cluster.Version19_2), + includedInBootstrap: clusterversion.VersionByKey(clusterversion.Version19_2), workFn: func(ctx context.Context, r runner) error { // Note that these particular schema changes are idempotent. if _, err := r.sqlExecutor.ExecEx(ctx, "update-reports-meta-generated", nil, /* txn */ @@ -264,7 +265,7 @@ var backwardCompatibleMigrations = []migrationDescriptor{ // TODO(ajwerner): Bake this migration into v20.2. name: "create system.protected_ts_meta table", workFn: createProtectedTimestampsMetaTable, - includedInBootstrap: cluster.VersionByKey(cluster.VersionProtectedTimestamps), + includedInBootstrap: clusterversion.VersionByKey(clusterversion.VersionProtectedTimestamps), newDescriptorIDs: staticIDs(keys.ProtectedTimestampsMetaTableID), }, { @@ -272,21 +273,21 @@ var backwardCompatibleMigrations = []migrationDescriptor{ // TODO(ajwerner): Bake this migration into v20.2. name: "create system.protected_ts_records table", workFn: createProtectedTimestampsRecordsTable, - includedInBootstrap: cluster.VersionByKey(cluster.VersionProtectedTimestamps), + includedInBootstrap: clusterversion.VersionByKey(clusterversion.VersionProtectedTimestamps), newDescriptorIDs: staticIDs(keys.ProtectedTimestampsRecordsTableID), }, { // Introduced in v20.1 name: "create new system.namespace table", workFn: createNewSystemNamespaceDescriptor, - includedInBootstrap: cluster.VersionByKey(cluster.VersionNamespaceTableWithSchemas), + includedInBootstrap: clusterversion.VersionByKey(clusterversion.VersionNamespaceTableWithSchemas), newDescriptorIDs: staticIDs(keys.NamespaceTableID), }, { // Introduced in v20.1 name: "migrate system.namespace_deprecated entries into system.namespace", workFn: migrateSystemNamespace, - includedInBootstrap: cluster.VersionByKey(cluster.VersionNamespaceTableWithSchemas), + includedInBootstrap: clusterversion.VersionByKey(clusterversion.VersionNamespaceTableWithSchemas), }, } @@ -831,10 +832,10 @@ func populateVersionSetting(ctx context.Context, r runner) error { if v == (roachpb.Version{}) { // The cluster was bootstrapped at v1.0 (or even earlier), so just use // the MinSupportedVersion of the binary. - v = cluster.BinaryMinimumSupportedVersion + v = clusterversion.BinaryMinimumSupportedVersion } - b, err := protoutil.Marshal(&cluster.ClusterVersion{Version: v}) + b, err := protoutil.Marshal(&clusterversion.ClusterVersion{Version: v}) if err != nil { return errors.Wrap(err, "while marshaling version") } diff --git a/pkg/storage/batcheval/cmd_add_sstable.go b/pkg/storage/batcheval/cmd_add_sstable.go index 8a775045492d..51d81c92a746 100644 --- a/pkg/storage/batcheval/cmd_add_sstable.go +++ b/pkg/storage/batcheval/cmd_add_sstable.go @@ -13,9 +13,9 @@ package batcheval import ( "context" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" - "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/storage/batcheval/result" "github.com/cockroachdb/cockroach/pkg/storage/engine" "github.com/cockroachdb/cockroach/pkg/storage/engine/enginepb" @@ -173,7 +173,7 @@ func EvalAddSSTable( stats.Subtract(skippedKVStats) stats.ContainsEstimates = 0 } else { - _ = cluster.VersionContainsEstimatesCounter // see for info on ContainsEstimates migration + _ = clusterversion.VersionContainsEstimatesCounter // see for info on ContainsEstimates migration stats.ContainsEstimates++ } diff --git a/pkg/storage/batcheval/cmd_end_transaction.go b/pkg/storage/batcheval/cmd_end_transaction.go index dc8b54b9bdc6..aa46e8953a51 100644 --- a/pkg/storage/batcheval/cmd_end_transaction.go +++ b/pkg/storage/batcheval/cmd_end_transaction.go @@ -17,9 +17,9 @@ import ( "math" "sync/atomic" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" - "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/storage/abortspan" "github.com/cockroachdb/cockroach/pkg/storage/batcheval/result" "github.com/cockroachdb/cockroach/pkg/storage/engine" @@ -1030,7 +1030,7 @@ func splitTriggerHelper( } deltaPostSplitLeft := h.DeltaPostSplitLeft() - if !cluster.Version.IsActive(ctx, rec.ClusterSettings(), cluster.VersionContainsEstimatesCounter) { + if !rec.ClusterSettings().Version.IsActive(ctx, clusterversion.VersionContainsEstimatesCounter) { deltaPostSplitLeft.ContainsEstimates = 0 } return deltaPostSplitLeft, pd, nil diff --git a/pkg/storage/batcheval/cmd_query_txn.go b/pkg/storage/batcheval/cmd_query_txn.go index 5769d1f4386b..55648caf96e9 100644 --- a/pkg/storage/batcheval/cmd_query_txn.go +++ b/pkg/storage/batcheval/cmd_query_txn.go @@ -14,9 +14,9 @@ import ( "bytes" "context" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" - "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/storage/batcheval/result" "github.com/cockroachdb/cockroach/pkg/storage/engine" "github.com/cockroachdb/cockroach/pkg/storage/spanset" @@ -55,8 +55,8 @@ func QueryTxn( // TODO(nvanbenschoten): old clusters didn't attach header timestamps to // QueryTxn requests, so only perform this check for clusters that will // always attach a valid timestamps. - checkHeaderTS := cluster.Version.IsActive( - ctx, cArgs.EvalCtx.ClusterSettings(), cluster.VersionQueryTxnTimestamp) + checkHeaderTS := cArgs.EvalCtx.ClusterSettings().Version.IsActive( + ctx, clusterversion.VersionQueryTxnTimestamp) if h.Timestamp.Less(args.Txn.WriteTimestamp) && checkHeaderTS { // This condition must hold for the timestamp cache access to be safe. return result.Result{}, errors.Errorf("request timestamp %s less than txn timestamp %s", h.Timestamp, args.Txn.WriteTimestamp) diff --git a/pkg/storage/batcheval/cmd_recompute_stats.go b/pkg/storage/batcheval/cmd_recompute_stats.go index 2cd75069d26e..90aa6f36efe6 100644 --- a/pkg/storage/batcheval/cmd_recompute_stats.go +++ b/pkg/storage/batcheval/cmd_recompute_stats.go @@ -13,9 +13,9 @@ package batcheval import ( "context" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" - "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/storage/batcheval/result" "github.com/cockroachdb/cockroach/pkg/storage/engine" "github.com/cockroachdb/cockroach/pkg/storage/engine/enginepb" @@ -99,7 +99,7 @@ func RecomputeStats( // stats for timeseries ranges (which go cold and the approximate stats are // wildly overcounting) and this is paced by the consistency checker, but it // means some extra engine churn. - if !cluster.Version.IsActive(ctx, cArgs.EvalCtx.ClusterSettings(), cluster.VersionContainsEstimatesCounter) { + if !cArgs.EvalCtx.ClusterSettings().Version.IsActive(ctx, clusterversion.VersionContainsEstimatesCounter) { // We are running with the older version of MVCCStats.ContainsEstimates // which was a boolean, so we should keep it in {0,1} and not reset it // to avoid racing with another command that sets it to true. diff --git a/pkg/storage/bulk/sst_batcher.go b/pkg/storage/bulk/sst_batcher.go index aad5964b554a..7598a538142f 100644 --- a/pkg/storage/bulk/sst_batcher.go +++ b/pkg/storage/bulk/sst_batcher.go @@ -15,6 +15,7 @@ import ( "context" "time" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv" "github.com/cockroachdb/cockroach/pkg/roachpb" @@ -180,7 +181,7 @@ func (b *SSTBatcher) Reset(ctx context.Context) error { // Create "Ingestion" SSTs in the newer RocksDBv2 format only if all nodes // in the cluster can support it. Until then, for backward compatibility, // create SSTs in the leveldb format ("backup" ones). - if cluster.Version.IsActive(ctx, b.settings, cluster.VersionStart20_1) { + if b.settings.Version.IsActive(ctx, clusterversion.VersionStart20_1) { b.sstWriter = engine.MakeIngestionSSTWriter(b.sstFile) } else { b.sstWriter = engine.MakeBackupSSTWriter(b.sstFile) @@ -480,7 +481,7 @@ func createSplitSSTable( ) (*sstSpan, *sstSpan, error) { sstFile := &engine.MemFile{} var w engine.SSTWriter - if cluster.Version.IsActive(ctx, settings, cluster.VersionStart20_1) { + if settings.Version.IsActive(ctx, clusterversion.VersionStart20_1) { w = engine.MakeIngestionSSTWriter(sstFile) } else { w = engine.MakeBackupSSTWriter(sstFile) @@ -514,7 +515,7 @@ func createSplitSSTable( disallowShadowing: disallowShadowing, } *sstFile = engine.MemFile{} - if cluster.Version.IsActive(ctx, settings, cluster.VersionStart20_1) { + if settings.Version.IsActive(ctx, clusterversion.VersionStart20_1) { w = engine.MakeIngestionSSTWriter(sstFile) } else { w = engine.MakeBackupSSTWriter(sstFile) diff --git a/pkg/storage/replica_application_state_machine.go b/pkg/storage/replica_application_state_machine.go index b2f51965b1d3..f9b4dcc0900b 100644 --- a/pkg/storage/replica_application_state_machine.go +++ b/pkg/storage/replica_application_state_machine.go @@ -15,8 +15,8 @@ import ( "fmt" "time" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/roachpb" - "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/storage/apply" "github.com/cockroachdb/cockroach/pkg/storage/engine" "github.com/cockroachdb/cockroach/pkg/storage/engine/enginepb" @@ -757,7 +757,7 @@ func (b *replicaAppBatch) stageTrivialReplicatedEvalResult( // has used the old cluster version: it's when the incoming // ContainsEstimates is 1. If so, we need to assume that an old node is processing // the same commands (as `true + true = true`), so make sure that `1 + 1 = 1`. - _ = cluster.VersionContainsEstimatesCounter // see for info on ContainsEstimates migration + _ = clusterversion.VersionContainsEstimatesCounter // see for info on ContainsEstimates migration deltaStats := res.Delta.ToStats() if deltaStats.ContainsEstimates == 1 && b.state.Stats.ContainsEstimates == 1 { deltaStats.ContainsEstimates = 0 diff --git a/pkg/storage/replica_command.go b/pkg/storage/replica_command.go index 49e45e31bc7f..cfd570a12632 100644 --- a/pkg/storage/replica_command.go +++ b/pkg/storage/replica_command.go @@ -20,6 +20,7 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" @@ -295,7 +296,7 @@ func (r *Replica) adminSplitWithDescriptor( delayable bool, reason string, ) (roachpb.AdminSplitResponse, error) { - if !cluster.Version.IsActive(ctx, r.store.ClusterSettings(), cluster.VersionStickyBit) { + if !r.store.ClusterSettings().Version.IsActive(ctx, clusterversion.VersionStickyBit) { // If sticky bits aren't supported yet but we receive one anyway, ignore // it. The callers are supposed to only pass hlc.Timestamp{} in that // case, but this is violated in at least one case (and there are lots of @@ -965,7 +966,7 @@ func (r *Replica) ChangeReplicas( // We execute the change serially if we're not allowed to run atomic // replication changes or if that was explicitly disabled. st := r.ClusterSettings() - unroll := !cluster.Version.IsActive(ctx, st, cluster.VersionAtomicChangeReplicas) || + unroll := !st.Version.IsActive(ctx, clusterversion.VersionAtomicChangeReplicas) || !UseAtomicReplicationChanges.Get(&st.SV) if unroll { @@ -1005,8 +1006,8 @@ func (r *Replica) changeReplicasImpl( } settings := r.ClusterSettings() - if useLearners := cluster.Version.IsActive( - ctx, settings, cluster.VersionLearnerReplicas, + if useLearners := settings.Version.IsActive( + ctx, clusterversion.VersionLearnerReplicas, ); !useLearners { // NB: we will never use atomic replication changes while learners are not // also active. @@ -1330,7 +1331,7 @@ func (r *Replica) atomicReplicationChange( } } - canUseDemotion := cluster.Version.IsActive(ctx, r.store.ClusterSettings(), cluster.VersionChangeReplicasDemotion) + canUseDemotion := r.store.ClusterSettings().Version.IsActive(ctx, clusterversion.VersionChangeReplicasDemotion) for _, target := range chgs.Removals() { typ := internalChangeTypeRemove if rDesc, ok := desc.GetReplicaDescriptor(target.StoreID); ok && rDesc.GetType() == roachpb.VOTER_FULL && canUseDemotion { @@ -1513,8 +1514,8 @@ func prepareChangeReplicasTrigger( updatedDesc := *desc updatedDesc.SetReplicas(desc.Replicas().DeepCopy()) - generationComparableEnabled := cluster.Version.IsActive( - ctx, store.ClusterSettings(), cluster.VersionGenerationComparable) + generationComparableEnabled := store.ClusterSettings().Version.IsActive( + ctx, clusterversion.VersionGenerationComparable) if generationComparableEnabled { updatedDesc.IncrementGeneration() updatedDesc.GenerationComparable = proto.Bool(true) @@ -1618,8 +1619,8 @@ func prepareChangeReplicasTrigger( } var crt *roachpb.ChangeReplicasTrigger - if !cluster.Version.IsActive( - ctx, store.ClusterSettings(), cluster.VersionAtomicChangeReplicasTrigger, + if !store.ClusterSettings().Version.IsActive( + ctx, clusterversion.VersionAtomicChangeReplicasTrigger, ) { var deprecatedChangeType roachpb.ReplicaChangeType var deprecatedRepDesc roachpb.ReplicaDescriptor @@ -2130,8 +2131,8 @@ func updateRangeDescriptor( func (s *Store) AdminRelocateRange( ctx context.Context, rangeDesc roachpb.RangeDescriptor, targets []roachpb.ReplicationTarget, ) error { - useAtomic := cluster.Version.IsActive( - ctx, s.ClusterSettings(), cluster.VersionAtomicChangeReplicas) + useAtomic := s.ClusterSettings().Version.IsActive( + ctx, clusterversion.VersionAtomicChangeReplicas) if useAtomic { // AdminChangeReplicas will only allow atomic replication changes when // this magic flag is set because we changed the corresponding request @@ -2550,7 +2551,7 @@ func (r *Replica) adminVerifyProtectedTimestamp( func maybeMarkGenerationComparable( ctx context.Context, st *cluster.Settings, desc *roachpb.RangeDescriptor, ) { - if cluster.Version.IsActive(ctx, st, cluster.VersionGenerationComparable) { + if st.Version.IsActive(ctx, clusterversion.VersionGenerationComparable) { desc.GenerationComparable = proto.Bool(true) } } diff --git a/pkg/storage/replica_consistency.go b/pkg/storage/replica_consistency.go index 47a06e4ce3bf..dde4872b4d67 100644 --- a/pkg/storage/replica_consistency.go +++ b/pkg/storage/replica_consistency.go @@ -26,7 +26,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/rpc" - "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/storage/batcheval" "github.com/cockroachdb/cockroach/pkg/storage/engine" "github.com/cockroachdb/cockroach/pkg/storage/engine/enginepb" @@ -238,7 +237,7 @@ func (r *Replica) CheckConsistency( }); err != nil { log.Infof(ctx, "while retrieving cluster bootstrap version: %s", err) // Intentionally continue with the assumption that it's the current version. - v = cluster.Version.ActiveVersion(ctx, r.store.cfg.Settings).Version + v = r.store.cfg.Settings.Version.ActiveVersion(ctx).Version } // For clusters that ever ran <19.1, we're not so sure that the stats are // consistent. Verify this only for clusters that started out on 19.1 or diff --git a/pkg/storage/replica_proposal.go b/pkg/storage/replica_proposal.go index 2b5bdc16be67..3b9999306100 100644 --- a/pkg/storage/replica_proposal.go +++ b/pkg/storage/replica_proposal.go @@ -21,6 +21,7 @@ import ( "unsafe" "github.com/cockroachdb/cockroach/pkg/base" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" @@ -760,8 +761,8 @@ func (r *Replica) evaluateProposal( res.Replicated.Timestamp = ba.Timestamp res.Replicated.Delta = ms.ToStatsDelta() - _ = cluster.VersionContainsEstimatesCounter // see for info on ContainsEstimates migration - if cluster.Version.IsActive(ctx, r.ClusterSettings(), cluster.VersionContainsEstimatesCounter) { + _ = clusterversion.VersionContainsEstimatesCounter // see for info on ContainsEstimates migration + if r.ClusterSettings().Version.IsActive(ctx, clusterversion.VersionContainsEstimatesCounter) { // Encode that this command (and any that follow) uses regular arithmetic for ContainsEstimates // by making sure ContainsEstimates is > 1. // This will be interpreted during command application. diff --git a/pkg/storage/replicate_queue.go b/pkg/storage/replicate_queue.go index 976471935b8c..e32042ad4fac 100644 --- a/pkg/storage/replicate_queue.go +++ b/pkg/storage/replicate_queue.go @@ -18,12 +18,12 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/config/zonepb" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings" - "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/storage/storagepb" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" @@ -441,7 +441,7 @@ func (rq *replicateQueue) addOrReplace( removeIdx = -1 } st := rq.store.cfg.Settings - if !cluster.Version.IsActive(ctx, st, cluster.VersionAtomicChangeReplicas) { + if !st.Version.IsActive(ctx, clusterversion.VersionAtomicChangeReplicas) { // If we can't swap yet, don't. removeIdx = -1 } diff --git a/pkg/storage/store.go b/pkg/storage/store.go index d1c43c7eebbd..fe3f819a9b39 100644 --- a/pkg/storage/store.go +++ b/pkg/storage/store.go @@ -26,6 +26,7 @@ import ( "unsafe" "github.com/cockroachdb/cockroach/pkg/base" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/config" "github.com/cockroachdb/cockroach/pkg/config/zonepb" "github.com/cockroachdb/cockroach/pkg/gossip" @@ -2506,20 +2507,20 @@ func (s *Store) ManuallyEnqueue( // GetClusterVersion reads the the cluster version from the store-local version // key. Returns an empty version if the key is not found. -func (s *Store) GetClusterVersion(ctx context.Context) (cluster.ClusterVersion, error) { +func (s *Store) GetClusterVersion(ctx context.Context) (clusterversion.ClusterVersion, error) { return ReadClusterVersion(ctx, s.engine) } // WriteClusterVersion writes the given cluster version to the store-local cluster version key. func WriteClusterVersion( - ctx context.Context, writer engine.ReadWriter, cv cluster.ClusterVersion, + ctx context.Context, writer engine.ReadWriter, cv clusterversion.ClusterVersion, ) error { return engine.MVCCPutProto(ctx, writer, nil, keys.StoreClusterVersionKey(), hlc.Timestamp{}, nil, &cv) } // ReadClusterVersion reads the the cluster version from the store-local version key. -func ReadClusterVersion(ctx context.Context, reader engine.Reader) (cluster.ClusterVersion, error) { - var cv cluster.ClusterVersion +func ReadClusterVersion(ctx context.Context, reader engine.Reader) (clusterversion.ClusterVersion, error) { + var cv clusterversion.ClusterVersion _, err := engine.MVCCGetProto(ctx, reader, keys.StoreClusterVersionKey(), hlc.Timestamp{}, &cv, engine.MVCCGetOptions{}) return cv, err diff --git a/pkg/storage/store_bootstrap.go b/pkg/storage/store_bootstrap.go index bdcbf00919f2..2a0e45a9eb10 100644 --- a/pkg/storage/store_bootstrap.go +++ b/pkg/storage/store_bootstrap.go @@ -13,9 +13,9 @@ package storage import ( "context" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" - "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/storage/engine" "github.com/cockroachdb/cockroach/pkg/storage/engine/enginepb" "github.com/cockroachdb/cockroach/pkg/storage/rditer" @@ -32,7 +32,7 @@ import ( // should be completely empty. It returns an error if called on a // non-empty engine. func InitEngine( - ctx context.Context, eng engine.Engine, ident roachpb.StoreIdent, cv cluster.ClusterVersion, + ctx context.Context, eng engine.Engine, ident roachpb.StoreIdent, cv clusterversion.ClusterVersion, ) error { exIdent, err := ReadStoreIdent(ctx, eng) if err == nil { @@ -150,7 +150,7 @@ func WriteInitialClusterData( EndKey: endKey, NextReplicaID: 2, } - if !bootstrapVersion.Less(cluster.VersionByKey(cluster.VersionGenerationComparable)) { + if !bootstrapVersion.Less(clusterversion.VersionByKey(clusterversion.VersionGenerationComparable)) { desc.GenerationComparable = proto.Bool(true) } replicas := []roachpb.ReplicaDescriptor{ diff --git a/pkg/storage/stores.go b/pkg/storage/stores.go index b887b232d487..6192b6a35ebe 100644 --- a/pkg/storage/stores.go +++ b/pkg/storage/stores.go @@ -15,11 +15,11 @@ import ( "fmt" "unsafe" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/gossip" "github.com/cockroachdb/cockroach/pkg/internal/client" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" - "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/storage/engine" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/log" @@ -282,11 +282,11 @@ func (ls *Stores) updateBootstrapInfoLocked(bi *gossip.BootstrapInfo) error { // engine, falling back to the zero value. func ReadVersionFromEngineOrZero( ctx context.Context, e engine.Engine, -) (cluster.ClusterVersion, error) { - var cv cluster.ClusterVersion +) (clusterversion.ClusterVersion, error) { + var cv clusterversion.ClusterVersion cv, err := ReadClusterVersion(ctx, e) if err != nil { - return cluster.ClusterVersion{}, err + return clusterversion.ClusterVersion{}, err } return cv, nil } @@ -294,7 +294,7 @@ func ReadVersionFromEngineOrZero( // WriteClusterVersionToEngines writes the given version to the given engines, // without any sanity checks. func WriteClusterVersionToEngines( - ctx context.Context, engines []engine.Engine, cv cluster.ClusterVersion, + ctx context.Context, engines []engine.Engine, cv clusterversion.ClusterVersion, ) error { for _, eng := range engines { if err := WriteClusterVersion(ctx, eng, cv); err != nil { @@ -317,7 +317,7 @@ func WriteClusterVersionToEngines( // returned if any engine has a higher version. func SynthesizeClusterVersionFromEngines( ctx context.Context, engines []engine.Engine, minSupportedVersion, serverVersion roachpb.Version, -) (cluster.ClusterVersion, error) { +) (clusterversion.ClusterVersion, error) { // Find the most recent bootstrap info. type originVersion struct { roachpb.Version @@ -336,10 +336,10 @@ func SynthesizeClusterVersionFromEngines( // constraints, which at the latest the second loop will achieve // (because then minStoreVersion don't change any more). for _, eng := range engines { - var cv cluster.ClusterVersion + var cv clusterversion.ClusterVersion cv, err := ReadVersionFromEngineOrZero(ctx, eng) if err != nil { - return cluster.ClusterVersion{}, err + return clusterversion.ClusterVersion{}, err } if cv.Version == (roachpb.Version{}) { // This is needed when a node first joins an existing cluster, in @@ -351,7 +351,7 @@ func SynthesizeClusterVersionFromEngines( // Avoid running a binary with a store that is too new. For example, // restarting into 1.1 after having upgraded to 1.2 doesn't work. if serverVersion.Less(cv.Version) { - return cluster.ClusterVersion{}, errors.Errorf( + return clusterversion.ClusterVersion{}, errors.Errorf( "cockroach version v%s is incompatible with data in store %s; use version v%s or later", serverVersion, eng, cv.Version) } @@ -371,7 +371,7 @@ func SynthesizeClusterVersionFromEngines( minStoreVersion.Version = minSupportedVersion } - cv := cluster.ClusterVersion{ + cv := clusterversion.ClusterVersion{ Version: minStoreVersion.Version, } log.Eventf(ctx, "read ClusterVersion %+v", cv) @@ -388,7 +388,7 @@ func SynthesizeClusterVersionFromEngines( // may not yet have picked up the final versions we're actually planning // to use. if minStoreVersion.Version.Less(minSupportedVersion) { - return cluster.ClusterVersion{}, errors.Errorf("store %s, last used with cockroach version v%s, "+ + return clusterversion.ClusterVersion{}, errors.Errorf("store %s, last used with cockroach version v%s, "+ "is too old for running version v%s (which requires data from v%s or later)", minStoreVersion.origin, minStoreVersion.Version, serverVersion, minSupportedVersion) } @@ -407,7 +407,7 @@ func SynthesizeClusterVersionFromEngines( // Version. // // If there aren't any stores, returns the minimum supported version of the binary. -func (ls *Stores) SynthesizeClusterVersion(ctx context.Context) (cluster.ClusterVersion, error) { +func (ls *Stores) SynthesizeClusterVersion(ctx context.Context) (clusterversion.ClusterVersion, error) { var engines []engine.Engine ls.storeMap.Range(func(_ int64, v unsafe.Pointer) bool { engines = append(engines, (*Store)(v).engine) @@ -415,7 +415,7 @@ func (ls *Stores) SynthesizeClusterVersion(ctx context.Context) (cluster.Cluster }) cv, err := SynthesizeClusterVersionFromEngines(ctx, engines, ls.minSupportedVersion, ls.serverVersion) if err != nil { - return cluster.ClusterVersion{}, err + return clusterversion.ClusterVersion{}, err } return cv, nil } @@ -425,7 +425,7 @@ func (ls *Stores) SynthesizeClusterVersion(ctx context.Context) (cluster.Cluster // error encountered writing to the stores. // // WriteClusterVersion makes no attempt to validate the supplied version. -func (ls *Stores) WriteClusterVersion(ctx context.Context, cv cluster.ClusterVersion) error { +func (ls *Stores) WriteClusterVersion(ctx context.Context, cv clusterversion.ClusterVersion) error { // Update all stores. engines := ls.engines() ls.storeMap.Range(func(_ int64, v unsafe.Pointer) bool { @@ -447,7 +447,7 @@ func (ls *Stores) engines() []engine.Engine { // OnClusterVersionChange is invoked when the running node receives a notification // indicating that the cluster version has changed. It checks the currently persisted // version and updates if it is older than the provided update. -func (ls *Stores) OnClusterVersionChange(ctx context.Context, cv cluster.ClusterVersion) error { +func (ls *Stores) OnClusterVersionChange(ctx context.Context, cv clusterversion.ClusterVersion) error { // Grab a lock to make sure that there aren't two interleaved invocations of // this method that result in clobbering of an update. ls.mu.Lock()