From 6e8840e8db5d28b8732d56f6e0f11125e20d5b98 Mon Sep 17 00:00:00 2001 From: nate Date: Mon, 10 Aug 2020 14:00:39 -0400 Subject: [PATCH] [dbnode] Add migration configuration and options (#2519) --- src/cmd/services/m3dbnode/config/bootstrap.go | 37 +++++++- .../services/m3dbnode/config/config_test.go | 1 + src/dbnode/persist/fs/migration/options.go | 71 +++++++++++++++ .../persist/fs/migration/options_test.go | 51 +++++++++++ src/dbnode/persist/fs/migration/types.go | 42 +++++++++ src/dbnode/persist/fs/migration/version.go | 87 +++++++++++++++++++ .../persist/fs/migration/version_test.go | 63 ++++++++++++++ .../bootstrap/bootstrapper/fs/options.go | 20 +++++ .../bootstrap/bootstrapper/fs/types.go | 7 ++ 9 files changed, 378 insertions(+), 1 deletion(-) create mode 100644 src/dbnode/persist/fs/migration/options.go create mode 100644 src/dbnode/persist/fs/migration/options_test.go create mode 100644 src/dbnode/persist/fs/migration/types.go create mode 100644 src/dbnode/persist/fs/migration/version.go create mode 100644 src/dbnode/persist/fs/migration/version_test.go diff --git a/src/cmd/services/m3dbnode/config/bootstrap.go b/src/cmd/services/m3dbnode/config/bootstrap.go index 754de6d4ce..0e0ed3f62a 100644 --- a/src/cmd/services/m3dbnode/config/bootstrap.go +++ b/src/cmd/services/m3dbnode/config/bootstrap.go @@ -27,6 +27,7 @@ import ( "github.com/m3db/m3/src/dbnode/client" "github.com/m3db/m3/src/dbnode/persist/fs" + "github.com/m3db/m3/src/dbnode/persist/fs/migration" "github.com/m3db/m3/src/dbnode/storage" "github.com/m3db/m3/src/dbnode/storage/bootstrap" "github.com/m3db/m3/src/dbnode/storage/bootstrap/bootstrapper" @@ -70,18 +71,51 @@ type BootstrapConfiguration struct { type BootstrapFilesystemConfiguration struct { // NumProcessorsPerCPU is the number of processors per CPU. NumProcessorsPerCPU float64 `yaml:"numProcessorsPerCPU" validate:"min=0.0"` + + // Migration configuration specifies what version, if any, existing data filesets should be migrated to + // if necessary. + Migration *BootstrapMigrationConfiguration `yaml:"migration"` } func (c BootstrapFilesystemConfiguration) numCPUs() int { return int(math.Ceil(float64(c.NumProcessorsPerCPU * float64(runtime.NumCPU())))) } +func (c BootstrapFilesystemConfiguration) migration() BootstrapMigrationConfiguration { + if cfg := c.Migration; cfg != nil { + return *cfg + } + return BootstrapMigrationConfiguration{} +} + func newDefaultBootstrapFilesystemConfiguration() BootstrapFilesystemConfiguration { return BootstrapFilesystemConfiguration{ NumProcessorsPerCPU: defaultNumProcessorsPerCPU, + Migration: &BootstrapMigrationConfiguration{}, } } +// BootstrapMigrationConfiguration specifies configuration for data migrations during bootstrapping. +type BootstrapMigrationConfiguration struct { + // TargetMigrationVersion indicates that we should attempt to upgrade filesets to + // what’s expected of the specified version. + TargetMigrationVersion migration.MigrationVersion `yaml:"targetMigrationVersion"` + + // Concurrency sets the number of concurrent workers performing migrations. + Concurrency int `yaml:"concurrency"` +} + +// NewOptions generates migration.Options from the configuration. +func (m BootstrapMigrationConfiguration) NewOptions() migration.Options { + opts := migration.NewOptions().SetTargetMigrationVersion(m.TargetMigrationVersion) + + if m.Concurrency > 0 { + opts = opts.SetConcurrency(m.Concurrency) + } + + return opts +} + // BootstrapCommitlogConfiguration specifies config for the commitlog bootstrapper. type BootstrapCommitlogConfiguration struct { // ReturnUnfulfilledForCorruptCommitLogFiles controls whether the commitlog bootstrapper @@ -182,7 +216,8 @@ func (bsc BootstrapConfiguration) New( SetCompactor(compactor). SetBoostrapDataNumProcessors(fsCfg.numCPUs()). SetRuntimeOptionsManager(opts.RuntimeOptionsManager()). - SetIdentifierPool(opts.IdentifierPool()) + SetIdentifierPool(opts.IdentifierPool()). + SetMigrationOptions(fsCfg.migration().NewOptions()) if err := validator.ValidateFilesystemBootstrapperOptions(fsbOpts); err != nil { return nil, err } diff --git a/src/cmd/services/m3dbnode/config/config_test.go b/src/cmd/services/m3dbnode/config/config_test.go index daa5d7674b..8e4580de37 100644 --- a/src/cmd/services/m3dbnode/config/config_test.go +++ b/src/cmd/services/m3dbnode/config/config_test.go @@ -419,6 +419,7 @@ func TestConfiguration(t *testing.T) { - noop-all fs: numProcessorsPerCPU: 0.42 + migration: null commitlog: returnUnfulfilledForCorruptCommitLogFiles: false peers: null diff --git a/src/dbnode/persist/fs/migration/options.go b/src/dbnode/persist/fs/migration/options.go new file mode 100644 index 0000000000..0b391f1640 --- /dev/null +++ b/src/dbnode/persist/fs/migration/options.go @@ -0,0 +1,71 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package migration + +import ( + "fmt" + "runtime" +) + +// defaultMigrationConcurrency is the default number of concurrent workers to perform migrations. +var defaultMigrationConcurrency = runtime.NumCPU() + +type options struct { + targetMigrationVersion MigrationVersion + concurrency int +} + +// NewOptions creates new migration options. +func NewOptions() Options { + return &options{ + concurrency: defaultMigrationConcurrency, + } +} + +func (o *options) Validate() error { + if err := ValidateMigrationVersion(o.targetMigrationVersion); err != nil { + return err + } + if o.concurrency < 1 { + return fmt.Errorf("concurrency value %d must be >= 1", o.concurrency) + } + return nil +} + +func (o *options) SetTargetMigrationVersion(value MigrationVersion) Options { + opts := *o + opts.targetMigrationVersion = value + return &opts +} + +func (o *options) TargetMigrationVersion() MigrationVersion { + return o.targetMigrationVersion +} + +func (o *options) SetConcurrency(value int) Options { + opts := *o + opts.concurrency = value + return &opts +} + +func (o *options) Concurrency() int { + return o.concurrency +} diff --git a/src/dbnode/persist/fs/migration/options_test.go b/src/dbnode/persist/fs/migration/options_test.go new file mode 100644 index 0000000000..73712bce7c --- /dev/null +++ b/src/dbnode/persist/fs/migration/options_test.go @@ -0,0 +1,51 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package migration + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestOptionsTargetMigrationVersion(t *testing.T) { + opts := NewOptions() + require.Equal(t, MigrationVersionNone, opts.TargetMigrationVersion()) + + opts = opts.SetTargetMigrationVersion(MigrationVersion_1_1) + require.Equal(t, MigrationVersion_1_1, opts.TargetMigrationVersion()) +} + +func TestOptionsConcurrency(t *testing.T) { + opts := NewOptions() + require.Equal(t, defaultMigrationConcurrency, opts.Concurrency()) + + opts = opts.SetConcurrency(100) + require.Equal(t, 100, opts.Concurrency()) +} + +func TestOptionsValidate(t *testing.T) { + opts := NewOptions() + require.NoError(t, opts.Validate()) + + require.Error(t, opts.SetTargetMigrationVersion(2).Validate()) + require.Error(t, opts.SetConcurrency(0).Validate()) +} diff --git a/src/dbnode/persist/fs/migration/types.go b/src/dbnode/persist/fs/migration/types.go new file mode 100644 index 0000000000..1b3aaea5aa --- /dev/null +++ b/src/dbnode/persist/fs/migration/types.go @@ -0,0 +1,42 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package migration + +// Options represents the options for migrations. +type Options interface { + // Validate validates migration options. + Validate() error + + // SetTargetMigrationVersion sets the target version for a migration + SetTargetMigrationVersion(value MigrationVersion) Options + + // TargetMigrationVersion is the target version for a migration. + TargetMigrationVersion() MigrationVersion + + // SetConcurrency sets the number of concurrent workers performing migrations. + SetConcurrency(value int) Options + + // Concurrency gets the number of concurrent workers performing migrations. + Concurrency() int +} + +// MigrationVersion is an enum that corresponds to the major and minor version number to migrate data files to. +type MigrationVersion uint diff --git a/src/dbnode/persist/fs/migration/version.go b/src/dbnode/persist/fs/migration/version.go new file mode 100644 index 0000000000..9eebc32984 --- /dev/null +++ b/src/dbnode/persist/fs/migration/version.go @@ -0,0 +1,87 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package migration + +import "fmt" + +const ( + // MigrationVersionNone indicates node should not attempt to perform any migrations. + MigrationVersionNone MigrationVersion = iota + // MigrationVersion_1_1 indicates node should attempt to migrate data files up to version 1.1. + MigrationVersion_1_1 +) + +var ( + validMigrationVersions = []MigrationVersion{ + MigrationVersionNone, + MigrationVersion_1_1, + } +) + +func (m *MigrationVersion) String() string { + switch *m { + case MigrationVersionNone: + return "none" + case MigrationVersion_1_1: + return "1.1" + default: + return "unknown" + } +} + +// ParseMigrationVersion parses a string for a MigrationVersion. +func ParseMigrationVersion(str string) (MigrationVersion, error) { + for _, valid := range validMigrationVersions { + if str == valid.String() { + return valid, nil + } + } + + return 0, fmt.Errorf("unrecognized migrate version: %v", str) +} + +// ValidateMigrationVersion validates a stored metrics type. +func ValidateMigrationVersion(m MigrationVersion) error { + for _, valid := range validMigrationVersions { + if valid == m { + return nil + } + } + + return fmt.Errorf("invalid migrate version '%v': should be one of %v", + m, validMigrationVersions) +} + +// UnmarshalYAML unmarshals a migrate version. +func (m *MigrationVersion) UnmarshalYAML(unmarshal func(interface{}) error) error { + var str string + if err := unmarshal(&str); err != nil { + return err + } + + if value, err := ParseMigrationVersion(str); err == nil { + *m = value + return nil + } + + return fmt.Errorf("invalid MigrationVersion '%s' valid types are: %v", + str, validMigrationVersions) +} diff --git a/src/dbnode/persist/fs/migration/version_test.go b/src/dbnode/persist/fs/migration/version_test.go new file mode 100644 index 0000000000..a5faeac91b --- /dev/null +++ b/src/dbnode/persist/fs/migration/version_test.go @@ -0,0 +1,63 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package migration + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + yaml "gopkg.in/yaml.v2" +) + +func TestParseMigrateVersion(t *testing.T) { + v, err := ParseMigrationVersion("none") + require.NoError(t, err) + require.Equal(t, MigrationVersionNone, v) + + v, err = ParseMigrationVersion("1.1") + require.NoError(t, err) + require.Equal(t, MigrationVersion_1_1, v) +} + +func TestValidateMigrateVersion(t *testing.T) { + err := ValidateMigrationVersion(MigrationVersion_1_1) + require.NoError(t, err) + + err = ValidateMigrationVersion(2) + require.Error(t, err) +} + +func TestUnmarshalYAML(t *testing.T) { + type config struct { + Version MigrationVersion `yaml:"version"` + } + + for _, value := range validMigrationVersions { + str := fmt.Sprintf("version: %s\n", value.String()) + var cfg config + require.NoError(t, yaml.Unmarshal([]byte(str), &cfg)) + require.Equal(t, value, cfg.Version) + } + + var cfg config + require.Error(t, yaml.Unmarshal([]byte("version: abc"), &cfg)) +} diff --git a/src/dbnode/storage/bootstrap/bootstrapper/fs/options.go b/src/dbnode/storage/bootstrap/bootstrapper/fs/options.go index 463ae32656..796aed3088 100644 --- a/src/dbnode/storage/bootstrap/bootstrapper/fs/options.go +++ b/src/dbnode/storage/bootstrap/bootstrapper/fs/options.go @@ -27,6 +27,7 @@ import ( "github.com/m3db/m3/src/dbnode/persist" "github.com/m3db/m3/src/dbnode/persist/fs" + "github.com/m3db/m3/src/dbnode/persist/fs/migration" "github.com/m3db/m3/src/dbnode/runtime" "github.com/m3db/m3/src/dbnode/storage/bootstrap/result" "github.com/m3db/m3/src/dbnode/storage/index" @@ -41,6 +42,7 @@ var ( errCompactorNotSet = errors.New("compactor not set") errIndexOptionsNotSet = errors.New("index options not set") errFilesystemOptionsNotSet = errors.New("filesystem options not set") + errMigrationOptionsNotSet = errors.New("migration options not set") // NB(r): Bootstrapping data doesn't use large amounts of memory // that won't be released, so its fine to do this as fast as possible. @@ -66,6 +68,7 @@ type options struct { bootstrapIndexNumProcessors int runtimeOptsMgr runtime.OptionsManager identifierPool ident.Pool + migrationOpts migration.Options } // NewOptions creates new bootstrap options @@ -83,6 +86,7 @@ func NewOptions() Options { bootstrapIndexNumProcessors: defaultBootstrapIndexNumProcessors, runtimeOptsMgr: runtime.NewOptionsManager(), identifierPool: idPool, + migrationOpts: migration.NewOptions(), } } @@ -99,6 +103,12 @@ func (o *options) Validate() error { if o.fsOpts == nil { return errFilesystemOptionsNotSet } + if o.migrationOpts == nil { + return errMigrationOptionsNotSet + } + if err := o.migrationOpts.Validate(); err != nil { + return err + } return nil } @@ -201,3 +211,13 @@ func (o *options) SetIdentifierPool(value ident.Pool) Options { func (o *options) IdentifierPool() ident.Pool { return o.identifierPool } + +func (o *options) SetMigrationOptions(value migration.Options) Options { + opts := *o + opts.migrationOpts = value + return &opts +} + +func (o *options) MigrationOptions() migration.Options { + return o.migrationOpts +} diff --git a/src/dbnode/storage/bootstrap/bootstrapper/fs/types.go b/src/dbnode/storage/bootstrap/bootstrapper/fs/types.go index 898cb2d682..d379c8c713 100644 --- a/src/dbnode/storage/bootstrap/bootstrapper/fs/types.go +++ b/src/dbnode/storage/bootstrap/bootstrapper/fs/types.go @@ -23,6 +23,7 @@ package fs import ( "github.com/m3db/m3/src/dbnode/persist" "github.com/m3db/m3/src/dbnode/persist/fs" + "github.com/m3db/m3/src/dbnode/persist/fs/migration" "github.com/m3db/m3/src/dbnode/runtime" "github.com/m3db/m3/src/dbnode/storage/bootstrap/result" "github.com/m3db/m3/src/dbnode/storage/index" @@ -101,4 +102,10 @@ type Options interface { // IndexOptions returns the indexing options. IndexOptions() index.Options + + // SetMigrationOptions sets the migration options. + SetMigrationOptions(value migration.Options) Options + + // MigrationOptions gets the migration options. + MigrationOptions() migration.Options }