From 65c8bb479da33de7640a247c5cf41a4ce91e7840 Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Fri, 25 Jun 2021 17:02:09 -0400 Subject: [PATCH 01/12] [dbnode] Add config option to set repair concurrency --- src/cmd/services/m3dbnode/config/config.go | 3 +++ src/dbnode/server/server.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/cmd/services/m3dbnode/config/config.go b/src/cmd/services/m3dbnode/config/config.go index 6380a16370..a52267f245 100644 --- a/src/cmd/services/m3dbnode/config/config.go +++ b/src/cmd/services/m3dbnode/config/config.go @@ -549,6 +549,9 @@ type RepairPolicy struct { // The repair check interval. CheckInterval time.Duration `yaml:"checkInterval"` + // Concurrency sets the repair shard concurrency if set. + Concurrency int `yaml:"concurrency"` + // Whether debug shadow comparisons are enabled. DebugShadowComparisonsEnabled bool `yaml:"debugShadowComparisonsEnabled"` diff --git a/src/dbnode/server/server.go b/src/dbnode/server/server.go index 461ad6ae23..a3d73227ec 100644 --- a/src/dbnode/server/server.go +++ b/src/dbnode/server/server.go @@ -933,6 +933,9 @@ func Run(runOpts RunOptions) { if cfg.Repair.CheckInterval > 0 { repairOpts = repairOpts.SetRepairCheckInterval(cfg.Repair.CheckInterval) } + if cfg.Repair.Concurrency > 0 { + repairOpts = repairOpts.SetRepairShardConcurrency(cfg.Repair.Concurrency) + } if cfg.Repair.DebugShadowComparisonsPercentage > 0 { // Set conditionally to avoid stomping on the default value of 1.0. From a7b2675b5c84afe9f1fbf29970d363845248587f Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Fri, 25 Jun 2021 17:46:58 -0400 Subject: [PATCH 02/12] Add option to set repair strategy and allow for full sweeps --- src/cmd/services/m3dbnode/config/config.go | 3 + .../services/m3dbnode/config/config_test.go | 2 + src/dbnode/server/server.go | 1 + src/dbnode/storage/repair.go | 23 +++++- src/dbnode/storage/repair/options.go | 15 ++++ src/dbnode/storage/repair/types.go | 66 +++++++++++++++++ src/dbnode/storage/repair_test.go | 73 ++++++++++++------- 7 files changed, 156 insertions(+), 27 deletions(-) diff --git a/src/cmd/services/m3dbnode/config/config.go b/src/cmd/services/m3dbnode/config/config.go index a52267f245..0974b8c336 100644 --- a/src/cmd/services/m3dbnode/config/config.go +++ b/src/cmd/services/m3dbnode/config/config.go @@ -540,6 +540,9 @@ type RepairPolicy struct { // Type is the type of repair to run. Type repair.Type `yaml:"type"` + // Strategy is the type of repair strategy to use. + Strategy repair.Strategy `yaml:"strategy"` + // Force the repair to run regardless of whether namespaces have repair enabled or not. Force bool `yaml:"force"` diff --git a/src/cmd/services/m3dbnode/config/config_test.go b/src/cmd/services/m3dbnode/config/config_test.go index f8c2192d4b..fc38fb6f25 100644 --- a/src/cmd/services/m3dbnode/config/config_test.go +++ b/src/cmd/services/m3dbnode/config/config_test.go @@ -454,9 +454,11 @@ func TestConfiguration(t *testing.T) { repair: enabled: false type: 0 + strategy: 0 force: false throttle: 2m0s checkInterval: 1m0s + concurrency: 0 debugShadowComparisonsEnabled: false debugShadowComparisonsPercentage: 0 replication: null diff --git a/src/dbnode/server/server.go b/src/dbnode/server/server.go index a3d73227ec..d1b9cd95eb 100644 --- a/src/dbnode/server/server.go +++ b/src/dbnode/server/server.go @@ -924,6 +924,7 @@ func Run(runOpts RunOptions) { if repairCfg := cfg.Repair; repairCfg != nil { repairOpts = repairOpts. SetType(repairCfg.Type). + SetStrategy(repairCfg.Strategy). SetForce(repairCfg.Force). SetResultOptions(rsOpts). SetDebugShadowComparisonsEnabled(cfg.Repair.DebugShadowComparisonsEnabled) diff --git a/src/dbnode/storage/repair.go b/src/dbnode/storage/repair.go index 216bb595ac..050f9d9087 100644 --- a/src/dbnode/storage/repair.go +++ b/src/dbnode/storage/repair.go @@ -666,6 +666,20 @@ func (r *dbRepairer) Repair() error { return err } + var ( + strategy = r.ropts.Strategy() + repairBlockStartShortCircuitRepair bool + ) + switch strategy { + case repair.DefaultStrategy: + repairBlockStartShortCircuitRepair = true + case repair.FullSweepStrategy: + repairBlockStartShortCircuitRepair = false + default: + // Unrecognized strategy. + return fmt.Errorf("unknown repair strategy: %v", strategy) + } + for _, n := range namespaces { repairRange := r.namespaceRepairTimeRange(n) blockSize := n.Options().RetentionOptions().BlockSize() @@ -681,6 +695,13 @@ func (r *dbRepairer) Repair() error { leastRecentlyRepairedBlockStartLastRepairTime xtime.UnixNano ) repairRange.IterateBackward(blockSize, func(blockStart xtime.UnixNano) bool { + // Update metrics around progress of repair. + blockStartUnixSeconds := blockStart.ToTime().Unix() + r.scope.Tagged(map[string]string{ + "namespace": n.ID().String(), + }).Gauge("timestamp-current-block-repair").Update(float64(blockStartUnixSeconds)) + + // Update state for later reporting of least recently repaired block. repairState, ok := r.repairStatesByNs.repairStates(n.ID(), blockStart) if ok && (leastRecentlyRepairedBlockStart.IsZero() || repairState.LastAttempt.Before(leastRecentlyRepairedBlockStartLastRepairTime)) { @@ -694,7 +715,7 @@ func (r *dbRepairer) Repair() error { // Failed or unrepair block from this point onwards. numUnrepairedBlocks++ - if hasRepairedABlockStart { + if hasRepairedABlockStart && repairBlockStartShortCircuitRepair { // Only want to repair one namespace/blockStart per call to Repair() // so once we've repaired a single blockStart we don't perform any // more actual repairs although we do keep iterating so that we can diff --git a/src/dbnode/storage/repair/options.go b/src/dbnode/storage/repair/options.go index f1f62bf5f5..2b240b05ec 100644 --- a/src/dbnode/storage/repair/options.go +++ b/src/dbnode/storage/repair/options.go @@ -31,6 +31,8 @@ import ( ) const ( + defaultRepairType = DefaultRepair + defaultRepairStrategy = DefaultStrategy // Allow repairs to progress when a single peer is down (I.E during single node failure // or deployments). defaultRepairConsistencyLevel = topology.ReadConsistencyLevelUnstrictMajority @@ -52,6 +54,7 @@ var ( type options struct { repairType Type + strategy Strategy force bool adminClients []client.AdminClient repairConsistencyLevel topology.ReadConsistencyLevel @@ -67,6 +70,8 @@ type options struct { // NewOptions creates new bootstrap options func NewOptions() Options { return &options{ + repairType: defaultRepairType, + strategy: defaultRepairStrategy, repairConsistencyLevel: defaultRepairConsistencyLevel, repairShardConcurrency: defaultRepairShardConcurrency, repairCheckInterval: defaultRepairCheckInterval, @@ -88,6 +93,16 @@ func (o *options) Type() Type { return o.repairType } +func (o *options) SetStrategy(value Strategy) Options { + opts := *o + opts.strategy = value + return &opts +} + +func (o *options) Strategy() Strategy { + return o.strategy +} + func (o *options) SetForce(value bool) Options { opts := *o opts.force = value diff --git a/src/dbnode/storage/repair/types.go b/src/dbnode/storage/repair/types.go index bdc2decfc2..49045f974d 100644 --- a/src/dbnode/storage/repair/types.go +++ b/src/dbnode/storage/repair/types.go @@ -82,6 +82,66 @@ func (t Type) String() string { return "unknown" } +// Strategy defines the repair strategy. +type Strategy uint + +const ( + // DefaultStrategy will compare iterating backwards then on repairing a + // block needing repair it will restart from the latest block start and + // work backwards again. + // This strategy is best at keeping most recent data repaired as quickly + // as possible but when turning on repair for the first time in a cluster + // you may want to do run repairs in full sweep for a while first. + DefaultStrategy Strategy = iota + // FullSweepStrategy will compare iterating backwards and repairing + // blocks needing repair until reaching the end of retention and then only + // once reaching the end of retention to repair does the repair restart + // evaluating blocks from the most recent block starts again. + // This mode may be more ideal in clusters that have never had repair + // enabled to ensure that historical data gets repaired at least once on + // a full sweep before switching back to the default strategy. + FullSweepStrategy +) + +var validStrategies = []Strategy{ + DefaultStrategy, + FullSweepStrategy, +} + +// UnmarshalYAML unmarshals an Type into a valid type from string. +func (t *Strategy) UnmarshalYAML(unmarshal func(interface{}) error) error { + var str string + if err := unmarshal(&str); err != nil { + return err + } + + // If unspecified, use default mode. + if str == "" { + *t = DefaultStrategy + return nil + } + + for _, valid := range validStrategies { + if str == valid.String() { + *t = valid + return nil + } + } + return fmt.Errorf("invalid repair Strategy '%s' valid types are: %s", + str, validStrategies) +} + +// String returns the bootstrap mode as a string +func (t Strategy) String() string { + switch t { + case DefaultStrategy: + return "default" + case FullSweepStrategy: + return "full_sweep" + } + return "unknown" +} + // ReplicaMetadataSlice captures a slice of block.ReplicaMetadata. type ReplicaMetadataSlice interface { // Add adds the metadata to the slice. @@ -272,6 +332,12 @@ type Options interface { // Type returns the type of repair to run. Type() Type + // SetStrategy sets the repair strategy. + SetStrategy(value Strategy) Options + + // Strategy returns the repair strategy. + Strategy() Strategy + // SetForce sets whether to force repairs to run for all namespaces. SetForce(value bool) Options diff --git a/src/dbnode/storage/repair_test.go b/src/dbnode/storage/repair_test.go index aa23c88ba6..e7cbb3e2b8 100644 --- a/src/dbnode/storage/repair_test.go +++ b/src/dbnode/storage/repair_test.go @@ -586,9 +586,6 @@ type expectedRepair struct { } func TestDatabaseRepairPrioritizationLogic(t *testing.T) { - ctrl := xtest.NewController(t) - defer ctrl.Finish() - var ( rOpts = retention.NewOptions(). SetRetentionPeriod(retention.NewOptions().BlockSize() * 2) @@ -607,22 +604,25 @@ func TestDatabaseRepairPrioritizationLogic(t *testing.T) { require.Equal(t, blockSize, flushTimeEnd.Sub(flushTimeStart)) testCases := []struct { - title string - repairState repairStatesByNs - expectedNS1Repair expectedRepair - expectedNS2Repair expectedRepair + title string + strategy repair.Strategy + repairState repairStatesByNs + expectedNS1Repairs []expectedRepair + expectedNS2Repairs []expectedRepair }{ { - title: "repairs most recent block if no repair state", - expectedNS1Repair: expectedRepair{ - expectedRepairRange: xtime.Range{Start: flushTimeEnd, End: flushTimeEnd.Add(blockSize)}, + title: "repairs most recent block if no repair state", + strategy: repair.DefaultStrategy, + expectedNS1Repairs: []expectedRepair{ + {expectedRepairRange: xtime.Range{Start: flushTimeEnd, End: flushTimeEnd.Add(blockSize)}}, }, - expectedNS2Repair: expectedRepair{ - expectedRepairRange: xtime.Range{Start: flushTimeEnd, End: flushTimeEnd.Add(blockSize)}, + expectedNS2Repairs: []expectedRepair{ + {expectedRepairRange: xtime.Range{Start: flushTimeEnd, End: flushTimeEnd.Add(blockSize)}}, }, }, { - title: "repairs next unrepaired block in reverse order if some (but not all) blocks have been repaired", + title: "repairs next unrepaired block in reverse order if some (but not all) blocks have been repaired", + strategy: repair.DefaultStrategy, repairState: repairStatesByNs{ "ns1": namespaceRepairStateByTime{ flushTimeEnd: repairState{ @@ -637,15 +637,16 @@ func TestDatabaseRepairPrioritizationLogic(t *testing.T) { }, }, }, - expectedNS1Repair: expectedRepair{ - expectedRepairRange: xtime.Range{Start: flushTimeStart, End: flushTimeStart.Add(blockSize)}, + expectedNS1Repairs: []expectedRepair{ + {expectedRepairRange: xtime.Range{Start: flushTimeStart, End: flushTimeStart.Add(blockSize)}}, }, - expectedNS2Repair: expectedRepair{ - expectedRepairRange: xtime.Range{Start: flushTimeStart, End: flushTimeStart.Add(blockSize)}, + expectedNS2Repairs: []expectedRepair{ + {expectedRepairRange: xtime.Range{Start: flushTimeStart, End: flushTimeStart.Add(blockSize)}}, }, }, { - title: "repairs least recently repaired block if all blocks have been repaired", + title: "repairs least recently repaired block if all blocks have been repaired", + strategy: repair.DefaultStrategy, repairState: repairStatesByNs{ "ns1": namespaceRepairStateByTime{ flushTimeStart: repairState{ @@ -668,11 +669,23 @@ func TestDatabaseRepairPrioritizationLogic(t *testing.T) { }, }, }, - expectedNS1Repair: expectedRepair{ - expectedRepairRange: xtime.Range{Start: flushTimeStart, End: flushTimeStart.Add(blockSize)}, + expectedNS1Repairs: []expectedRepair{ + {expectedRepairRange: xtime.Range{Start: flushTimeStart, End: flushTimeStart.Add(blockSize)}}, }, - expectedNS2Repair: expectedRepair{ - expectedRepairRange: xtime.Range{Start: flushTimeStart, End: flushTimeStart.Add(blockSize)}, + expectedNS2Repairs: []expectedRepair{ + {expectedRepairRange: xtime.Range{Start: flushTimeStart, End: flushTimeStart.Add(blockSize)}}, + }, + }, + { + title: "repairs all blocks block if no repair state with full sweep strategy", + strategy: repair.FullSweepStrategy, + expectedNS1Repairs: []expectedRepair{ + {expectedRepairRange: xtime.Range{Start: flushTimeEnd, End: flushTimeEnd.Add(blockSize)}}, + {expectedRepairRange: xtime.Range{Start: flushTimeStart, End: flushTimeStart.Add(blockSize)}}, + }, + expectedNS2Repairs: []expectedRepair{ + {expectedRepairRange: xtime.Range{Start: flushTimeEnd, End: flushTimeEnd.Add(blockSize)}}, + {expectedRepairRange: xtime.Range{Start: flushTimeStart, End: flushTimeStart.Add(blockSize)}}, }, }, } @@ -680,7 +693,11 @@ func TestDatabaseRepairPrioritizationLogic(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.title, func(t *testing.T) { - opts := DefaultTestOptions().SetRepairOptions(testRepairOptions(ctrl)) + ctrl := xtest.NewController(t) + defer ctrl.Finish() + + repairOpts := testRepairOptions(ctrl).SetStrategy(tc.strategy) + opts := DefaultTestOptions().SetRepairOptions(repairOpts) mockDatabase := NewMockdatabase(ctrl) databaseRepairer, err := newDatabaseRepairer(mockDatabase, opts) @@ -707,8 +724,12 @@ func TestDatabaseRepairPrioritizationLogic(t *testing.T) { ns1.EXPECT().ID().Return(ident.StringID("ns1")).AnyTimes() ns2.EXPECT().ID().Return(ident.StringID("ns2")).AnyTimes() - ns1.EXPECT().Repair(gomock.Any(), tc.expectedNS1Repair.expectedRepairRange, NamespaceRepairOptions{}) - ns2.EXPECT().Repair(gomock.Any(), tc.expectedNS2Repair.expectedRepairRange, NamespaceRepairOptions{}) + for _, expected := range tc.expectedNS1Repairs { + ns1.EXPECT().Repair(gomock.Any(), expected.expectedRepairRange, NamespaceRepairOptions{}) + } + for _, expected := range tc.expectedNS2Repairs { + ns2.EXPECT().Repair(gomock.Any(), expected.expectedRepairRange, NamespaceRepairOptions{}) + } mockDatabase.EXPECT().OwnedNamespaces().Return(namespaces, nil) require.Nil(t, repairer.Repair()) @@ -718,7 +739,7 @@ func TestDatabaseRepairPrioritizationLogic(t *testing.T) { // Database repairer repairs blocks in decreasing time ranges for each namespace. If database repairer fails to // repair a time range of a namespace then instead of skipping repair of all past time ranges of that namespace, test -// that database repaier tries to repair the past corrupt time range of that namespace. +// that database repairer tries to repair the past corrupt time range of that namespace. func TestDatabaseRepairSkipsPoisonShard(t *testing.T) { ctrl := xtest.NewController(t) defer ctrl.Finish() From 1f1e8ef949ca17af8eedaa2c16890967b74ebce5 Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Mon, 28 Jun 2021 16:40:24 -0400 Subject: [PATCH 03/12] Avoid divide by zero when compared blocks is zero --- src/dbnode/storage/repair/types.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/dbnode/storage/repair/types.go b/src/dbnode/storage/repair/types.go index 49045f974d..30e8d416eb 100644 --- a/src/dbnode/storage/repair/types.go +++ b/src/dbnode/storage/repair/types.go @@ -298,6 +298,10 @@ func (r PeerMetadataComparisonResults) Aggregate() AggregatePeerMetadataComparis result.ComparedMissingBlocks += elem.ComparedMissingBlocks result.ComparedExtraBlocks += elem.ComparedExtraBlocks } + if result.ComparedBlocks > 0 { + // Do not divide by zero and end up with a struct that cannot be JSON serialized. + return result + } result.ComparedDifferingPercent = float64(result.ComparedDifferingBlocks) / float64(result.ComparedBlocks) return result } From 55b39f0c64b18dd7224564087c5cf7cf96401db3 Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Thu, 1 Jul 2021 07:49:03 -0400 Subject: [PATCH 04/12] Propagate not just ID but tags as well --- src/dbnode/client/client_mock.go | 7 ++++--- src/dbnode/client/session.go | 4 ++-- src/dbnode/client/types.go | 2 +- src/dbnode/storage/repair.go | 10 ++++++---- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/dbnode/client/client_mock.go b/src/dbnode/client/client_mock.go index b87d4c1b81..b158c15539 100644 --- a/src/dbnode/client/client_mock.go +++ b/src/dbnode/client/client_mock.go @@ -722,13 +722,14 @@ func (m *MockPeerBlocksIter) EXPECT() *MockPeerBlocksIterMockRecorder { } // Current mocks base method. -func (m *MockPeerBlocksIter) Current() (topology.Host, ident.ID, block.DatabaseBlock) { +func (m *MockPeerBlocksIter) Current() (topology.Host, ident.ID, ident.Tags, block.DatabaseBlock) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Current") ret0, _ := ret[0].(topology.Host) ret1, _ := ret[1].(ident.ID) - ret2, _ := ret[2].(block.DatabaseBlock) - return ret0, ret1, ret2 + ret2, _ := ret[2].(ident.Tags) + ret3, _ := ret[3].(block.DatabaseBlock) + return ret0, ret1, ret2, ret3 } // Current indicates an expected call of Current. diff --git a/src/dbnode/client/session.go b/src/dbnode/client/session.go index 490117f5b8..6e952749dc 100644 --- a/src/dbnode/client/session.go +++ b/src/dbnode/client/session.go @@ -3769,8 +3769,8 @@ func newPeerBlocksIter( } } -func (it *peerBlocksIter) Current() (topology.Host, ident.ID, block.DatabaseBlock) { - return it.current.peer, it.current.id, it.current.block +func (it *peerBlocksIter) Current() (topology.Host, ident.ID, ident.Tags, block.DatabaseBlock) { + return it.current.peer, it.current.id, it.current.tags, it.current.block } func (it *peerBlocksIter) Err() error { diff --git a/src/dbnode/client/types.go b/src/dbnode/client/types.go index e3f1a232a6..950d73cf9b 100644 --- a/src/dbnode/client/types.go +++ b/src/dbnode/client/types.go @@ -230,7 +230,7 @@ type PeerBlocksIter interface { // Current returns the metadata, and block data for a single block replica. // These remain valid until Next() is called again. - Current() (topology.Host, ident.ID, block.DatabaseBlock) + Current() (topology.Host, ident.ID, ident.Tags, block.DatabaseBlock) // Err returns any error encountered. Err() error diff --git a/src/dbnode/storage/repair.go b/src/dbnode/storage/repair.go index 050f9d9087..45de4ca511 100644 --- a/src/dbnode/storage/repair.go +++ b/src/dbnode/storage/repair.go @@ -319,15 +319,17 @@ func (r shardRepairer) Repair( } for perSeriesReplicaIter.Next() { - _, id, block := perSeriesReplicaIter.Current() - // TODO(rartoul): Handle tags in both branches: https://github.com/m3db/m3/issues/1848 + _, id, tags, block := perSeriesReplicaIter.Current() if existing, ok := results.BlockAt(id, block.StartTime()); ok { + // Merge contents with existing block. if err := existing.Merge(block); err != nil { return repair.MetadataComparisonResult{}, err } - } else { - results.AddBlock(id, ident.Tags{}, block) + continue } + + // Add block for first time to results. + results.AddBlock(id, tags, block) } } From 0051edb0ce13d7f13cb7755c38a9a63f7608a3e6 Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Thu, 1 Jul 2021 07:50:02 -0400 Subject: [PATCH 05/12] Fix tests for client --- src/dbnode/client/session_fetch_bulk_blocks_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dbnode/client/session_fetch_bulk_blocks_test.go b/src/dbnode/client/session_fetch_bulk_blocks_test.go index e227e06c79..6582a88d36 100644 --- a/src/dbnode/client/session_fetch_bulk_blocks_test.go +++ b/src/dbnode/client/session_fetch_bulk_blocks_test.go @@ -703,7 +703,7 @@ func assertFetchBlocksFromPeersResult( } extraBlocks := []peerBlocksDatapoint{} for observedBlocksIter.Next() { - observedHost, observedID, observedBlock := observedBlocksIter.Current() + observedHost, observedID, _, observedBlock := observedBlocksIter.Current() // find which peer the current datapoint is for peerIdx := -1 From 0bc58257237cc4d2e02ec44c831053f27477f841 Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Thu, 1 Jul 2021 12:18:11 -0400 Subject: [PATCH 06/12] Add encoded tags to code path when fetching blocks from peers --- src/dbnode/client/session.go | 56 ++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/src/dbnode/client/session.go b/src/dbnode/client/session.go index 6e952749dc..845807fb7a 100644 --- a/src/dbnode/client/session.go +++ b/src/dbnode/client/session.go @@ -2416,11 +2416,50 @@ func (s *session) FetchBlocksFromPeers( if err != nil { return nil, err } + peersByHost := make(map[string]peer, len(peers.peers)) for _, peer := range peers.peers { peersByHost[peer.Host().ID()] = peer } + // If any metadata has tags then encode them up front so can + // return an error on tag encoding rather than logging error that would + // possibly get missed. + var ( + metadatasEncodedTags []checked.Bytes + anyTags bool + ) + for _, meta := range metadatas { + if len(meta.Tags.Values()) > 0 { + anyTags = true + break + } + } + if anyTags { + // NB(r): Allocate exact length so nil is used and each index + // references same index as the incoming metadatas being fetched. + metadatasEncodedTags = make([]checked.Bytes, len(metadatas)) + tagsIter := ident.NewTagsIterator(ident.Tags{}) + for idx, meta := range metadatas { + if len(meta.Tags.Values()) == 0 { + continue + } + + tagsIter.Reset(meta.Tags) + tagsEncoder := s.pools.tagEncoder.Get() + if err := tagsEncoder.Encode(tagsIter); err != nil { + return nil, err + } + + encodedTagsCheckedBytes, ok := tagsEncoder.Data() + if !ok { + return nil, fmt.Errorf("could not encode tags: id=%s", meta.ID.String()) + } + + metadatasEncodedTags[idx] = encodedTagsCheckedBytes + } + } + go func() { for atomic.LoadInt64(&complete) == 0 { progress.fetchBlocksFromPeers.Update(1) @@ -2431,7 +2470,7 @@ func (s *session) FetchBlocksFromPeers( metadataCh := make(chan receivedBlockMetadata, blockMetadataChBufSize) go func() { - for _, rb := range metadatas { + for idx, rb := range metadatas { peer, ok := peersByHost[rb.Host.ID()] if !ok { logger.Warn("replica requested from unknown peer, skipping", @@ -2441,9 +2480,20 @@ func (s *session) FetchBlocksFromPeers( ) continue } + + // Attach encoded tags if present. + var encodedTags checked.Bytes + if idx < len(metadatasEncodedTags) { + // Note: could still be nil if had no tags, but the slice + // was built so need to take ref to encoded tags if + // was encoded. + encodedTags = metadatasEncodedTags[idx] + } + metadataCh <- receivedBlockMetadata{ - id: rb.ID, - peer: peer, + id: rb.Metadata.ID, + encodedTags: encodedTags, + peer: peer, block: blockMetadata{ start: rb.Start, size: rb.Size, From 8d1ba802980eab24cd17591da61d3ea33764bbb2 Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Thu, 1 Jul 2021 13:29:28 -0400 Subject: [PATCH 07/12] Fix compared blocks check --- src/dbnode/storage/repair/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dbnode/storage/repair/types.go b/src/dbnode/storage/repair/types.go index 30e8d416eb..e0da116b15 100644 --- a/src/dbnode/storage/repair/types.go +++ b/src/dbnode/storage/repair/types.go @@ -298,7 +298,7 @@ func (r PeerMetadataComparisonResults) Aggregate() AggregatePeerMetadataComparis result.ComparedMissingBlocks += elem.ComparedMissingBlocks result.ComparedExtraBlocks += elem.ComparedExtraBlocks } - if result.ComparedBlocks > 0 { + if result.ComparedBlocks <= 0 { // Do not divide by zero and end up with a struct that cannot be JSON serialized. return result } From 63d7f7c0157a8088e3a1918f52b2d1f690a5fbeb Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Thu, 1 Jul 2021 13:39:15 -0400 Subject: [PATCH 08/12] Remove helper function --- src/dbnode/storage/repair/types.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/dbnode/storage/repair/types.go b/src/dbnode/storage/repair/types.go index e0da116b15..5283565147 100644 --- a/src/dbnode/storage/repair/types.go +++ b/src/dbnode/storage/repair/types.go @@ -279,12 +279,6 @@ type PeerMetadataComparisonResult struct { ComparedExtraBlocks int64 } -// ComparedDifferingPercent returns the percent, between range of -// [0.0, 1.0], of all the blocks in the comparison. -func (r PeerMetadataComparisonResult) ComparedDifferingPercent() float64 { - return float64(r.ComparedDifferingBlocks) / float64(r.ComparedBlocks) -} - // PeerMetadataComparisonResults is a slice of PeerMetadataComparisonResult. type PeerMetadataComparisonResults []PeerMetadataComparisonResult From e1cde22b62d2c0b01edca3473f54950f8522a372 Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Thu, 1 Jul 2021 13:41:54 -0400 Subject: [PATCH 09/12] Use direct log fields for logging the repair results --- src/dbnode/storage/namespace.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/dbnode/storage/namespace.go b/src/dbnode/storage/namespace.go index 4d3091801c..d4c05aa98e 100644 --- a/src/dbnode/storage/namespace.go +++ b/src/dbnode/storage/namespace.go @@ -1618,7 +1618,6 @@ func (n *dbNamespace) Repair( wg.Wait() aggregatePeerComparison := peerMetadataComparisons.Aggregate() - n.metrics.repairDifferingPercent.Update(aggregatePeerComparison.ComparedDifferingPercent) n.metrics.repairComparedBlocks.Inc(aggregatePeerComparison.ComparedBlocks) n.metrics.repairDifferingBlocks.Inc(aggregatePeerComparison.ComparedDifferingBlocks) @@ -1636,7 +1635,12 @@ func (n *dbNamespace) Repair( zap.Int64("numSizeDiffBlocks", numSizeDiffBlocks), zap.Int64("numChecksumDiffSeries", numChecksumDiffSeries), zap.Int64("numChecksumDiffBlocks", numChecksumDiffBlocks), - zap.Any("peerMetadataComparisons", aggregatePeerComparison), + zap.Float64("peerComparisonComparedDifferingPercent", aggregatePeerComparison.ComparedDifferingPercent), + zap.Int64("peerComparisonComparedBlocks", aggregatePeerComparison.ComparedBlocks), + zap.Int64("peerComparisonComparedDifferingBlocks", aggregatePeerComparison.ComparedDifferingBlocks), + zap.Int64("peerComparisonComparedMismatchBlocks", aggregatePeerComparison.ComparedMismatchBlocks), + zap.Int64("peerComparisonComparedMissingBlocks", aggregatePeerComparison.ComparedMissingBlocks), + zap.Int64("peerComparisonComparedExtraBlocks", aggregatePeerComparison.ComparedExtraBlocks), ) return multiErr.FinalError() From a795fe6809b6c736f1cf8eaa38cb208e0511dcec Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Thu, 1 Jul 2021 16:54:57 -0400 Subject: [PATCH 10/12] Fix integration test build and storage unit tests --- .../admin_session_fetch_blocks_test.go | 3 +- src/dbnode/storage/repair_test.go | 64 ++++++++++--------- 2 files changed, 36 insertions(+), 31 deletions(-) diff --git a/src/dbnode/integration/admin_session_fetch_blocks_test.go b/src/dbnode/integration/admin_session_fetch_blocks_test.go index 79e08de31c..d3cbdf3d7f 100644 --- a/src/dbnode/integration/admin_session_fetch_blocks_test.go +++ b/src/dbnode/integration/admin_session_fetch_blocks_test.go @@ -220,7 +220,7 @@ func testSetupToSeriesMaps( require.NotNil(t, blocksIter) for blocksIter.Next() { - _, id, blk := blocksIter.Current() + _, id, tags, blk := blocksIter.Current() ctx := context.NewBackground() reader, err := blk.Stream(ctx) require.NoError(t, err) @@ -242,6 +242,7 @@ func testSetupToSeriesMaps( seriesMapList := seriesMap[firstTS.Truncate(blockSize)] seriesMapList = append(seriesMapList, generate.Series{ ID: id, + Tags: tags, Data: datapoints, }) seriesMap[firstTS.Truncate(blockSize)] = seriesMapList diff --git a/src/dbnode/storage/repair_test.go b/src/dbnode/storage/repair_test.go index e7cbb3e2b8..09f367e20b 100644 --- a/src/dbnode/storage/repair_test.go +++ b/src/dbnode/storage/repair_test.go @@ -212,7 +212,7 @@ func testDatabaseShardRepairerRepair(t *testing.T, withLimit bool) { shard.EXPECT().ID().Return(shardID).AnyTimes() peerIter := client.NewMockPeerBlockMetadataIter(ctrl) - inputBlocks := []block.ReplicaMetadata{ + inBlocks := []block.ReplicaMetadata{ { Host: topology.NewHost("1", "addr1"), Metadata: block.NewMetadata(ident.StringID("foo"), ident.Tags{}, now.Add(30*time.Minute), sizes[0], &checksums[0], lastRead), @@ -230,11 +230,11 @@ func testDatabaseShardRepairerRepair(t *testing.T, withLimit bool) { gomock.InOrder( peerIter.EXPECT().Next().Return(true), - peerIter.EXPECT().Current().Return(inputBlocks[0].Host, inputBlocks[0].Metadata), + peerIter.EXPECT().Current().Return(inBlocks[0].Host, inBlocks[0].Metadata), peerIter.EXPECT().Next().Return(true), - peerIter.EXPECT().Current().Return(inputBlocks[1].Host, inputBlocks[1].Metadata), + peerIter.EXPECT().Current().Return(inBlocks[1].Host, inBlocks[1].Metadata), peerIter.EXPECT().Next().Return(true), - peerIter.EXPECT().Current().Return(inputBlocks[2].Host, inputBlocks[2].Metadata), + peerIter.EXPECT().Current().Return(inBlocks[2].Host, inBlocks[2].Metadata), peerIter.EXPECT().Next().Return(false), peerIter.EXPECT().Err().Return(nil), ) @@ -245,22 +245,24 @@ func testDatabaseShardRepairerRepair(t *testing.T, withLimit bool) { peerBlocksIter := client.NewMockPeerBlocksIter(ctrl) dbBlock1 := block.NewMockDatabaseBlock(ctrl) - dbBlock1.EXPECT().StartTime().Return(inputBlocks[2].Metadata.Start).AnyTimes() + dbBlock1.EXPECT().StartTime().Return(inBlocks[2].Metadata.Start).AnyTimes() dbBlock2 := block.NewMockDatabaseBlock(ctrl) - dbBlock2.EXPECT().StartTime().Return(inputBlocks[2].Metadata.Start).AnyTimes() + dbBlock2.EXPECT().StartTime().Return(inBlocks[2].Metadata.Start).AnyTimes() // Ensure merging logic works. dbBlock1.EXPECT().Merge(dbBlock2) gomock.InOrder( peerBlocksIter.EXPECT().Next().Return(true), - peerBlocksIter.EXPECT().Current().Return(inputBlocks[2].Host, inputBlocks[2].Metadata.ID, dbBlock1), + peerBlocksIter.EXPECT().Current(). + Return(inBlocks[2].Host, inBlocks[2].Metadata.ID, inBlocks[2].Metadata.Tags, dbBlock1), peerBlocksIter.EXPECT().Next().Return(true), - peerBlocksIter.EXPECT().Current().Return(inputBlocks[2].Host, inputBlocks[2].Metadata.ID, dbBlock2), + peerBlocksIter.EXPECT().Current(). + Return(inBlocks[2].Host, inBlocks[2].Metadata.ID, inBlocks[2].Metadata.Tags, dbBlock2), peerBlocksIter.EXPECT().Next().Return(false), ) nsMeta, err := namespace.NewMetadata(namespaceID, namespace.NewOptions()) require.NoError(t, err) session.EXPECT(). - FetchBlocksFromPeers(nsMeta, shardID, rpOpts.RepairConsistencyLevel(), inputBlocks[2:], gomock.Any()). + FetchBlocksFromPeers(nsMeta, shardID, rpOpts.RepairConsistencyLevel(), inBlocks[2:], gomock.Any()). Return(peerBlocksIter, nil) var ( @@ -302,7 +304,7 @@ func testDatabaseShardRepairerRepair(t *testing.T, withLimit bool) { expected := []block.ReplicaMetadata{ // Checksum difference for series "bar". {Host: topology.NewHost("0", "addr0"), Metadata: block.NewMetadata(ident.StringID("bar"), ident.Tags{}, now.Add(30*time.Minute), sizes[2], &checksums[2], lastRead)}, - {Host: topology.NewHost("1", "addr1"), Metadata: inputBlocks[2].Metadata}, + {Host: topology.NewHost("1", "addr1"), Metadata: inBlocks[2].Metadata}, } require.Equal(t, expected, currBlock.Metadata()) @@ -318,7 +320,7 @@ func testDatabaseShardRepairerRepair(t *testing.T, withLimit bool) { expected = []block.ReplicaMetadata{ // Size difference for series "foo". {Host: topology.NewHost("0", "addr0"), Metadata: block.NewMetadata(ident.StringID("foo"), ident.Tags{}, now.Add(time.Hour), sizes[1], &checksums[1], lastRead)}, - {Host: topology.NewHost("1", "addr1"), Metadata: inputBlocks[1].Metadata}, + {Host: topology.NewHost("1", "addr1"), Metadata: inBlocks[1].Metadata}, } require.Equal(t, expected, currBlock.Metadata()) } @@ -424,7 +426,7 @@ func TestDatabaseShardRepairerRepairMultiSession(t *testing.T) { shard.EXPECT().ID().Return(shardID).AnyTimes() shard.EXPECT().LoadBlocks(gomock.Any()).Return(nil) - inputBlocks := []block.ReplicaMetadata{ + inBlocks := []block.ReplicaMetadata{ { // Peer block size size[2] is different from origin block size size[0] Metadata: block.NewMetadata(ident.StringID("foo"), ident.Tags{}, now.Add(30*time.Minute), sizes[2], &checksums[0], lastRead), @@ -460,20 +462,20 @@ func TestDatabaseShardRepairerRepairMultiSession(t *testing.T) { session := mock.session // Make a copy of the input blocks where the host is set to the host for // the cluster associated with the current session. - inputBlocksForSession := make([]block.ReplicaMetadata, len(inputBlocks)) - copy(inputBlocksForSession, inputBlocks) - for j := range inputBlocksForSession { - inputBlocksForSession[j].Host = hosts[i] + inBlocksForSession := make([]block.ReplicaMetadata, len(inBlocks)) + copy(inBlocksForSession, inBlocks) + for j := range inBlocksForSession { + inBlocksForSession[j].Host = hosts[i] } peerIter := client.NewMockPeerBlockMetadataIter(ctrl) gomock.InOrder( peerIter.EXPECT().Next().Return(true), - peerIter.EXPECT().Current().Return(inputBlocksForSession[0].Host, inputBlocks[0].Metadata), + peerIter.EXPECT().Current().Return(inBlocksForSession[0].Host, inBlocks[0].Metadata), peerIter.EXPECT().Next().Return(true), - peerIter.EXPECT().Current().Return(inputBlocksForSession[1].Host, inputBlocks[1].Metadata), + peerIter.EXPECT().Current().Return(inBlocksForSession[1].Host, inBlocks[1].Metadata), peerIter.EXPECT().Next().Return(true), - peerIter.EXPECT().Current().Return(inputBlocksForSession[2].Host, inputBlocks[2].Metadata), + peerIter.EXPECT().Current().Return(inBlocksForSession[2].Host, inBlocks[2].Metadata), peerIter.EXPECT().Next().Return(false), peerIter.EXPECT().Err().Return(nil), ) @@ -484,23 +486,25 @@ func TestDatabaseShardRepairerRepairMultiSession(t *testing.T) { peerBlocksIter := client.NewMockPeerBlocksIter(ctrl) dbBlock1 := block.NewMockDatabaseBlock(ctrl) - dbBlock1.EXPECT().StartTime().Return(inputBlocksForSession[2].Metadata.Start).AnyTimes() + dbBlock1.EXPECT().StartTime().Return(inBlocksForSession[2].Metadata.Start).AnyTimes() dbBlock2 := block.NewMockDatabaseBlock(ctrl) - dbBlock2.EXPECT().StartTime().Return(inputBlocksForSession[2].Metadata.Start).AnyTimes() + dbBlock2.EXPECT().StartTime().Return(inBlocksForSession[2].Metadata.Start).AnyTimes() // Ensure merging logic works. Nede AnyTimes() because the Merge() will only be called on dbBlock1 // for the first session (all subsequent blocks from other sessions will get merged into dbBlock1 // from the first session.) dbBlock1.EXPECT().Merge(dbBlock2).AnyTimes() gomock.InOrder( peerBlocksIter.EXPECT().Next().Return(true), - peerBlocksIter.EXPECT().Current().Return(inputBlocksForSession[2].Host, inputBlocks[2].Metadata.ID, dbBlock1), + peerBlocksIter.EXPECT().Current(). + Return(inBlocksForSession[2].Host, inBlocks[2].Metadata.ID, inBlocks[2].Metadata.Tags, dbBlock1), peerBlocksIter.EXPECT().Next().Return(true), - peerBlocksIter.EXPECT().Current().Return(inputBlocksForSession[2].Host, inputBlocks[2].Metadata.ID, dbBlock2), + peerBlocksIter.EXPECT().Current(). + Return(inBlocksForSession[2].Host, inBlocks[2].Metadata.ID, inBlocks[2].Metadata.Tags, dbBlock2), peerBlocksIter.EXPECT().Next().Return(false), ) require.NoError(t, err) session.EXPECT(). - FetchBlocksFromPeers(nsMeta, shardID, rpOpts.RepairConsistencyLevel(), inputBlocksForSession[2:], gomock.Any()). + FetchBlocksFromPeers(nsMeta, shardID, rpOpts.RepairConsistencyLevel(), inBlocksForSession[2:], gomock.Any()). Return(peerBlocksIter, nil) } @@ -529,8 +533,8 @@ func TestDatabaseShardRepairerRepairMultiSession(t *testing.T) { expected := []block.ReplicaMetadata{ // Checksum difference for series "bar". {Host: origin, Metadata: block.NewMetadata(ident.StringID("bar"), ident.Tags{}, now.Add(30*time.Minute), sizes[2], &checksums[2], lastRead)}, - {Host: hosts[0], Metadata: inputBlocks[2].Metadata}, - {Host: hosts[1], Metadata: inputBlocks[2].Metadata}, + {Host: hosts[0], Metadata: inBlocks[2].Metadata}, + {Host: hosts[1], Metadata: inBlocks[2].Metadata}, } require.Equal(t, expected, currBlock.Metadata()) @@ -547,8 +551,8 @@ func TestDatabaseShardRepairerRepairMultiSession(t *testing.T) { expected = []block.ReplicaMetadata{ // Size difference for series "foo". {Host: origin, Metadata: block.NewMetadata(ident.StringID("foo"), ident.Tags{}, now.Add(30*time.Minute), sizes[0], &checksums[0], lastRead)}, - {Host: hosts[0], Metadata: inputBlocks[0].Metadata}, - {Host: hosts[1], Metadata: inputBlocks[0].Metadata}, + {Host: hosts[0], Metadata: inBlocks[0].Metadata}, + {Host: hosts[1], Metadata: inBlocks[0].Metadata}, } require.Equal(t, expected, currBlock.Metadata()) // Validate second block @@ -558,8 +562,8 @@ func TestDatabaseShardRepairerRepairMultiSession(t *testing.T) { expected = []block.ReplicaMetadata{ // Size difference for series "foo". {Host: origin, Metadata: block.NewMetadata(ident.StringID("foo"), ident.Tags{}, now.Add(time.Hour), sizes[1], &checksums[1], lastRead)}, - {Host: hosts[0], Metadata: inputBlocks[1].Metadata}, - {Host: hosts[1], Metadata: inputBlocks[1].Metadata}, + {Host: hosts[0], Metadata: inBlocks[1].Metadata}, + {Host: hosts[1], Metadata: inBlocks[1].Metadata}, } require.Equal(t, expected, currBlock.Metadata()) From a1b87c19417d2416d300bc22da9073c70bde288e Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Thu, 1 Jul 2021 16:58:24 -0400 Subject: [PATCH 11/12] Fix lint --- go.mod | 1 + go.sum | 5 +++++ src/dbnode/storage/repair.go | 15 ++++++--------- src/dbnode/storage/repair/types.go | 6 ++++-- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 99fc77c73b..124bba30bb 100644 --- a/go.mod +++ b/go.mod @@ -73,6 +73,7 @@ require ( github.com/prometheus/client_golang v1.10.0 github.com/prometheus/common v0.20.0 github.com/prometheus/prometheus v1.8.2-0.20210331101223-3cafc58827d1 + github.com/quasilyte/go-ruleguard/dsl/fluent v0.0.0-20201222093424-5d7e62a465d3 // indirect github.com/rakyll/statik v0.1.6 github.com/remeh/sizedwaitgroup v1.0.0 // indirect github.com/rhysd/go-github-selfupdate v1.2.2 // indirect diff --git a/go.sum b/go.sum index c1f00a6118..16e8289b1e 100644 --- a/go.sum +++ b/go.sum @@ -1051,8 +1051,13 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= github.com/quasilyte/go-ruleguard v0.3.0 h1:A3OfpsK2ynOTbz/KMi62qWzignjGCOZVChATSf4P+A0= github.com/quasilyte/go-ruleguard v0.3.0/go.mod h1:p2miAhLp6fERzFNbcuQ4bevXs8rgK//uCHsUDkumITg= +github.com/quasilyte/go-ruleguard v0.3.7 h1:fcKxvLDaTvWtzw3eFPtWQ9/a8u1+fILOT1nIm/gVzdg= github.com/quasilyte/go-ruleguard/dsl v0.0.0-20210106184943-e47d54850b18/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= +github.com/quasilyte/go-ruleguard/dsl v0.0.0-20210115110123-c73ee1cbff1f h1:e+uECJCDesYxvHKYsB1xWY/WpReTQoN6F14Nhny2Vik= github.com/quasilyte/go-ruleguard/dsl v0.0.0-20210115110123-c73ee1cbff1f/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= +github.com/quasilyte/go-ruleguard/dsl v0.3.6 h1:W2wnISJifyda0x/RXq15Qjrsu9iOhX2gy4+Ku+owylw= +github.com/quasilyte/go-ruleguard/dsl/fluent v0.0.0-20201222093424-5d7e62a465d3 h1:eL7x4/zMnlquMxYe7V078BD7MGskZ0daGln+SJCVzuY= +github.com/quasilyte/go-ruleguard/dsl/fluent v0.0.0-20201222093424-5d7e62a465d3/go.mod h1:P7JlQWFT7jDcFZMtUPQbtGzzzxva3rBn6oIF+LPwFcM= github.com/quasilyte/go-ruleguard/rules v0.0.0-20201231183845-9e62ed36efe1/go.mod h1:7JTjp89EGyU1d6XfBiXihJNG37wB2VRkd125Q1u7Plc= github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 h1:L8QM9bvf68pVdQ3bCFZMDmnt9yqcMBro1pC7F+IPYMY= github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0= diff --git a/src/dbnode/storage/repair.go b/src/dbnode/storage/repair.go index 45de4ca511..f9472ab51d 100644 --- a/src/dbnode/storage/repair.go +++ b/src/dbnode/storage/repair.go @@ -695,13 +695,14 @@ func (r *dbRepairer) Repair() error { hasRepairedABlockStart = false leastRecentlyRepairedBlockStart xtime.UnixNano leastRecentlyRepairedBlockStartLastRepairTime xtime.UnixNano + namespaceScope = r.scope.Tagged(map[string]string{ + "namespace": n.ID().String(), + }) ) repairRange.IterateBackward(blockSize, func(blockStart xtime.UnixNano) bool { // Update metrics around progress of repair. blockStartUnixSeconds := blockStart.ToTime().Unix() - r.scope.Tagged(map[string]string{ - "namespace": n.ID().String(), - }).Gauge("timestamp-current-block-repair").Update(float64(blockStartUnixSeconds)) + namespaceScope.Gauge("timestamp-current-block-repair").Update(float64(blockStartUnixSeconds)) // Update state for later reporting of least recently repaired block. repairState, ok := r.repairStatesByNs.repairStates(n.ID(), blockStart) @@ -735,15 +736,11 @@ func (r *dbRepairer) Repair() error { }) // Update metrics with statistics about repair status. - r.scope.Tagged(map[string]string{ - "namespace": n.ID().String(), - }).Gauge("num-unrepaired-blocks").Update(float64(numUnrepairedBlocks)) + namespaceScope.Gauge("num-unrepaired-blocks").Update(float64(numUnrepairedBlocks)) secondsSinceLastRepair := xtime.ToUnixNano(r.nowFn()). Sub(leastRecentlyRepairedBlockStartLastRepairTime).Seconds() - r.scope.Tagged(map[string]string{ - "namespace": n.ID().String(), - }).Gauge("max-seconds-since-last-block-repair").Update(secondsSinceLastRepair) + namespaceScope.Gauge("max-seconds-since-last-block-repair").Update(secondsSinceLastRepair) if hasRepairedABlockStart { // Previous loop performed a repair which means we've hit our limit of repairing diff --git a/src/dbnode/storage/repair/types.go b/src/dbnode/storage/repair/types.go index 5283565147..d65cabe66c 100644 --- a/src/dbnode/storage/repair/types.go +++ b/src/dbnode/storage/repair/types.go @@ -78,8 +78,9 @@ func (t Type) String() string { return "default" case OnlyCompareRepair: return "only_compare" + default: + return "unknown" } - return "unknown" } // Strategy defines the repair strategy. @@ -138,8 +139,9 @@ func (t Strategy) String() string { return "default" case FullSweepStrategy: return "full_sweep" + default: + return "unknown" } - return "unknown" } // ReplicaMetadataSlice captures a slice of block.ReplicaMetadata. From 1a681d344316de2580ab8bafe8a8a40a68bd2861 Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Thu, 1 Jul 2021 16:59:06 -0400 Subject: [PATCH 12/12] Revert go.mod/go.sum changes from lint --- go.mod | 1 - go.sum | 5 ----- 2 files changed, 6 deletions(-) diff --git a/go.mod b/go.mod index 124bba30bb..99fc77c73b 100644 --- a/go.mod +++ b/go.mod @@ -73,7 +73,6 @@ require ( github.com/prometheus/client_golang v1.10.0 github.com/prometheus/common v0.20.0 github.com/prometheus/prometheus v1.8.2-0.20210331101223-3cafc58827d1 - github.com/quasilyte/go-ruleguard/dsl/fluent v0.0.0-20201222093424-5d7e62a465d3 // indirect github.com/rakyll/statik v0.1.6 github.com/remeh/sizedwaitgroup v1.0.0 // indirect github.com/rhysd/go-github-selfupdate v1.2.2 // indirect diff --git a/go.sum b/go.sum index 16e8289b1e..c1f00a6118 100644 --- a/go.sum +++ b/go.sum @@ -1051,13 +1051,8 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= github.com/quasilyte/go-ruleguard v0.3.0 h1:A3OfpsK2ynOTbz/KMi62qWzignjGCOZVChATSf4P+A0= github.com/quasilyte/go-ruleguard v0.3.0/go.mod h1:p2miAhLp6fERzFNbcuQ4bevXs8rgK//uCHsUDkumITg= -github.com/quasilyte/go-ruleguard v0.3.7 h1:fcKxvLDaTvWtzw3eFPtWQ9/a8u1+fILOT1nIm/gVzdg= github.com/quasilyte/go-ruleguard/dsl v0.0.0-20210106184943-e47d54850b18/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= -github.com/quasilyte/go-ruleguard/dsl v0.0.0-20210115110123-c73ee1cbff1f h1:e+uECJCDesYxvHKYsB1xWY/WpReTQoN6F14Nhny2Vik= github.com/quasilyte/go-ruleguard/dsl v0.0.0-20210115110123-c73ee1cbff1f/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= -github.com/quasilyte/go-ruleguard/dsl v0.3.6 h1:W2wnISJifyda0x/RXq15Qjrsu9iOhX2gy4+Ku+owylw= -github.com/quasilyte/go-ruleguard/dsl/fluent v0.0.0-20201222093424-5d7e62a465d3 h1:eL7x4/zMnlquMxYe7V078BD7MGskZ0daGln+SJCVzuY= -github.com/quasilyte/go-ruleguard/dsl/fluent v0.0.0-20201222093424-5d7e62a465d3/go.mod h1:P7JlQWFT7jDcFZMtUPQbtGzzzxva3rBn6oIF+LPwFcM= github.com/quasilyte/go-ruleguard/rules v0.0.0-20201231183845-9e62ed36efe1/go.mod h1:7JTjp89EGyU1d6XfBiXihJNG37wB2VRkd125Q1u7Plc= github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 h1:L8QM9bvf68pVdQ3bCFZMDmnt9yqcMBro1pC7F+IPYMY= github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0=