From 6c086cf954825f415babe3ef1a49dcc0f73aec94 Mon Sep 17 00:00:00 2001 From: Justin Chan Date: Mon, 22 Oct 2018 17:11:25 -0400 Subject: [PATCH] Remove CacheAllMetadata caching policy (#1110) * Remove CacheAllMetadata caching policy --- .buildkite/pipeline.yml | 7 - src/cmd/services/m3dbnode/config/bootstrap.go | 20 - .../services/m3dbnode/config/config_test.go | 1 - src/dbnode/client/client_mock.go | 48 +- src/dbnode/client/session.go | 285 +-- .../client/session_fetch_bulk_blocks_test.go | 261 +-- src/dbnode/client/types.go | 3 - src/dbnode/generated/thrift/rpc.thrift | 37 - src/dbnode/generated/thrift/rpc/rpc.go | 1974 ++--------------- src/dbnode/generated/thrift/rpc/rpc_mock.go | 13 - src/dbnode/generated/thrift/rpc/tchan-rpc.go | 50 - .../admin_session_fetch_blocks_test.go | 4 +- src/dbnode/integration/client.go | 3 +- src/dbnode/integration/integration.go | 34 +- .../peers_bootstrap_high_concurrency_test.go | 14 +- .../integration/peers_bootstrap_index_test.go | 5 +- .../peers_bootstrap_merge_local_test.go | 14 +- .../peers_bootstrap_merge_peer_blocks_test.go | 9 +- .../peers_bootstrap_node_down_test.go | 9 +- .../peers_bootstrap_none_available_test.go | 11 +- .../peers_bootstrap_select_best_test.go | 9 +- .../peers_bootstrap_simple_test.go | 9 +- .../peers_bootstrap_single_node_test.go | 3 +- ...eers_bootstrap_v2_high_concurrency_test.go | 37 - .../peers_bootstrap_v2_merge_local_test.go | 37 - ...ers_bootstrap_v2_merge_peer_blocks_test.go | 37 - .../peers_bootstrap_v2_node_down_test.go | 37 - .../peers_bootstrap_v2_select_best_test.go | 37 - .../peers_bootstrap_v2_simple_test.go | 37 - .../server/tchannelthrift/node/service.go | 134 -- .../tchannelthrift/node/service_test.go | 145 -- .../network/server/tchannelthrift/options.go | 48 - .../network/server/tchannelthrift/pool.go | 146 -- .../network/server/tchannelthrift/types.go | 24 - src/dbnode/server/server.go | 15 - src/dbnode/storage/block/block.go | 136 +- src/dbnode/storage/block/block_mock.go | 46 +- src/dbnode/storage/block/types.go | 19 +- src/dbnode/storage/block/wired_list.go | 12 +- src/dbnode/storage/block/wired_list_test.go | 2 +- .../bootstrap/bootstrapper/fs/source.go | 32 +- .../bootstrap/bootstrapper/peers/options.go | 53 +- .../bootstrap/bootstrapper/peers/source.go | 35 +- .../bootstrapper/peers/source_data_test.go | 48 +- .../bootstrapper/peers/source_index_test.go | 18 +- .../bootstrap/bootstrapper/peers/types.go | 8 - src/dbnode/storage/database.go | 19 - src/dbnode/storage/database_test.go | 72 - src/dbnode/storage/namespace.go | 21 - .../storage/namespace/convert_prop_test.go | 1 - src/dbnode/storage/namespace_test.go | 67 - src/dbnode/storage/repair.go | 4 +- src/dbnode/storage/repair_test.go | 4 +- src/dbnode/storage/series/buffer.go | 15 - src/dbnode/storage/series/policy.go | 9 +- src/dbnode/storage/series/reader.go | 4 - src/dbnode/storage/series/series.go | 52 +- src/dbnode/storage/series/series_test.go | 109 +- .../series_wired_list_interaction_test.go | 1 - src/dbnode/storage/shard.go | 46 +- .../shard_fetch_blocks_metadata_test.go | 58 - src/dbnode/storage/shard_race_prop_test.go | 14 +- src/dbnode/storage/storage_mock.go | 56 - src/dbnode/storage/types.go | 32 - 64 files changed, 418 insertions(+), 4132 deletions(-) delete mode 100644 src/dbnode/integration/peers_bootstrap_v2_high_concurrency_test.go delete mode 100644 src/dbnode/integration/peers_bootstrap_v2_merge_local_test.go delete mode 100644 src/dbnode/integration/peers_bootstrap_v2_merge_peer_blocks_test.go delete mode 100644 src/dbnode/integration/peers_bootstrap_v2_node_down_test.go delete mode 100644 src/dbnode/integration/peers_bootstrap_v2_select_best_test.go delete mode 100644 src/dbnode/integration/peers_bootstrap_v2_simple_test.go diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 7bf61f320c..f926a098aa 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -44,13 +44,6 @@ steps: gopath-checkout#v1.0.1: import: github.com/m3db/m3 <<: *common - - label: "Integration (dbnode All Metadata) %n" - parallelism: 2 - command: make clean install-vendor test-ci-integration-dbnode cache_policy=all_metadata - plugins: - docker-compose#v2.5.1: - run: app - <<: *common - name: "Integration (dbnode Recently Read) %n" parallelism: 2 command: make clean install-vendor test-ci-integration-dbnode cache_policy=recently_read diff --git a/src/cmd/services/m3dbnode/config/bootstrap.go b/src/cmd/services/m3dbnode/config/bootstrap.go index 354791c3d9..bf9782609e 100644 --- a/src/cmd/services/m3dbnode/config/bootstrap.go +++ b/src/cmd/services/m3dbnode/config/bootstrap.go @@ -53,9 +53,6 @@ type BootstrapConfiguration struct { // Filesystem bootstrapper configuration. Filesystem *BootstrapFilesystemConfiguration `yaml:"fs"` - // Peers bootstrapper configuration. - Peers *BootstrapPeersConfiguration `yaml:"peers"` - // Commitlog bootstrapper configuration. Commitlog *BootstrapCommitlogConfiguration `yaml:"commitlog"` @@ -72,28 +69,12 @@ func (bsc BootstrapConfiguration) fsNumProcessors() int { return int(math.Ceil(float64(runtime.NumCPU()) * np)) } -// TODO: Remove once v1 endpoint no longer required. -func (bsc BootstrapConfiguration) peersFetchBlocksMetadataEndpointVersion() client.FetchBlocksMetadataEndpointVersion { - version := client.FetchBlocksMetadataEndpointDefault - if peersCfg := bsc.Peers; peersCfg != nil { - version = peersCfg.FetchBlocksMetadataEndpointVersion - } - return version -} - // BootstrapFilesystemConfiguration specifies config for the fs bootstrapper. type BootstrapFilesystemConfiguration struct { // NumProcessorsPerCPU is the number of processors per CPU. NumProcessorsPerCPU float64 `yaml:"numProcessorsPerCPU" validate:"min=0.0"` } -// BootstrapPeersConfiguration specifies config for the peers bootstrapper. -type BootstrapPeersConfiguration struct { - // FetchBlocksMetadataEndpointVersion is the endpoint to use when fetching blocks metadata. - // TODO: Remove once v1 endpoint no longer required. - FetchBlocksMetadataEndpointVersion client.FetchBlocksMetadataEndpointVersion `yaml:"fetchBlocksMetadataEndpointVersion"` -} - // BootstrapCommitlogConfiguration specifies config for the commitlog bootstrapper. type BootstrapCommitlogConfiguration struct { // ReturnUnfulfilledForCorruptCommitlogFiles controls whether the commitlog bootstrapper @@ -171,7 +152,6 @@ func (bsc BootstrapConfiguration) New( SetAdminClient(adminClient). SetPersistManager(opts.PersistManager()). SetDatabaseBlockRetrieverManager(opts.DatabaseBlockRetrieverManager()). - SetFetchBlocksMetadataEndpointVersion(bsc.peersFetchBlocksMetadataEndpointVersion()). SetRuntimeOptionsManager(opts.RuntimeOptionsManager()) bs, err = peers.NewPeersBootstrapperProvider(pOpts, bs) if err != nil { diff --git a/src/cmd/services/m3dbnode/config/config_test.go b/src/cmd/services/m3dbnode/config/config_test.go index a79c6c526c..86cee751d5 100644 --- a/src/cmd/services/m3dbnode/config/config_test.go +++ b/src/cmd/services/m3dbnode/config/config_test.go @@ -367,7 +367,6 @@ db: - noop-all fs: numProcessorsPerCPU: 0.125 - peers: null commitlog: null cacheSeriesMetadata: null blockRetrieve: null diff --git a/src/dbnode/client/client_mock.go b/src/dbnode/client/client_mock.go index 180b6e00a3..27e5c8f5e1 100644 --- a/src/dbnode/client/client_mock.go +++ b/src/dbnode/client/client_mock.go @@ -742,42 +742,42 @@ func (mr *MockAdminSessionMockRecorder) Truncate(namespace interface{}) *gomock. } // FetchBootstrapBlocksFromPeers mocks base method -func (m *MockAdminSession) FetchBootstrapBlocksFromPeers(namespace namespace.Metadata, shard uint32, start, end time.Time, opts result.Options, version FetchBlocksMetadataEndpointVersion) (result.ShardResult, error) { - ret := m.ctrl.Call(m, "FetchBootstrapBlocksFromPeers", namespace, shard, start, end, opts, version) +func (m *MockAdminSession) FetchBootstrapBlocksFromPeers(namespace namespace.Metadata, shard uint32, start, end time.Time, opts result.Options) (result.ShardResult, error) { + ret := m.ctrl.Call(m, "FetchBootstrapBlocksFromPeers", namespace, shard, start, end, opts) ret0, _ := ret[0].(result.ShardResult) ret1, _ := ret[1].(error) return ret0, ret1 } // FetchBootstrapBlocksFromPeers indicates an expected call of FetchBootstrapBlocksFromPeers -func (mr *MockAdminSessionMockRecorder) FetchBootstrapBlocksFromPeers(namespace, shard, start, end, opts, version interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchBootstrapBlocksFromPeers", reflect.TypeOf((*MockAdminSession)(nil).FetchBootstrapBlocksFromPeers), namespace, shard, start, end, opts, version) +func (mr *MockAdminSessionMockRecorder) FetchBootstrapBlocksFromPeers(namespace, shard, start, end, opts interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchBootstrapBlocksFromPeers", reflect.TypeOf((*MockAdminSession)(nil).FetchBootstrapBlocksFromPeers), namespace, shard, start, end, opts) } // FetchBootstrapBlocksMetadataFromPeers mocks base method -func (m *MockAdminSession) FetchBootstrapBlocksMetadataFromPeers(namespace ident.ID, shard uint32, start, end time.Time, result result.Options, version FetchBlocksMetadataEndpointVersion) (PeerBlockMetadataIter, error) { - ret := m.ctrl.Call(m, "FetchBootstrapBlocksMetadataFromPeers", namespace, shard, start, end, result, version) +func (m *MockAdminSession) FetchBootstrapBlocksMetadataFromPeers(namespace ident.ID, shard uint32, start, end time.Time, result result.Options) (PeerBlockMetadataIter, error) { + ret := m.ctrl.Call(m, "FetchBootstrapBlocksMetadataFromPeers", namespace, shard, start, end, result) ret0, _ := ret[0].(PeerBlockMetadataIter) ret1, _ := ret[1].(error) return ret0, ret1 } // FetchBootstrapBlocksMetadataFromPeers indicates an expected call of FetchBootstrapBlocksMetadataFromPeers -func (mr *MockAdminSessionMockRecorder) FetchBootstrapBlocksMetadataFromPeers(namespace, shard, start, end, result, version interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchBootstrapBlocksMetadataFromPeers", reflect.TypeOf((*MockAdminSession)(nil).FetchBootstrapBlocksMetadataFromPeers), namespace, shard, start, end, result, version) +func (mr *MockAdminSessionMockRecorder) FetchBootstrapBlocksMetadataFromPeers(namespace, shard, start, end, result interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchBootstrapBlocksMetadataFromPeers", reflect.TypeOf((*MockAdminSession)(nil).FetchBootstrapBlocksMetadataFromPeers), namespace, shard, start, end, result) } // FetchBlocksMetadataFromPeers mocks base method -func (m *MockAdminSession) FetchBlocksMetadataFromPeers(namespace ident.ID, shard uint32, start, end time.Time, consistencyLevel topology.ReadConsistencyLevel, result result.Options, version FetchBlocksMetadataEndpointVersion) (PeerBlockMetadataIter, error) { - ret := m.ctrl.Call(m, "FetchBlocksMetadataFromPeers", namespace, shard, start, end, consistencyLevel, result, version) +func (m *MockAdminSession) FetchBlocksMetadataFromPeers(namespace ident.ID, shard uint32, start, end time.Time, consistencyLevel topology.ReadConsistencyLevel, result result.Options) (PeerBlockMetadataIter, error) { + ret := m.ctrl.Call(m, "FetchBlocksMetadataFromPeers", namespace, shard, start, end, consistencyLevel, result) ret0, _ := ret[0].(PeerBlockMetadataIter) ret1, _ := ret[1].(error) return ret0, ret1 } // FetchBlocksMetadataFromPeers indicates an expected call of FetchBlocksMetadataFromPeers -func (mr *MockAdminSessionMockRecorder) FetchBlocksMetadataFromPeers(namespace, shard, start, end, consistencyLevel, result, version interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchBlocksMetadataFromPeers", reflect.TypeOf((*MockAdminSession)(nil).FetchBlocksMetadataFromPeers), namespace, shard, start, end, consistencyLevel, result, version) +func (mr *MockAdminSessionMockRecorder) FetchBlocksMetadataFromPeers(namespace, shard, start, end, consistencyLevel, result interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchBlocksMetadataFromPeers", reflect.TypeOf((*MockAdminSession)(nil).FetchBlocksMetadataFromPeers), namespace, shard, start, end, consistencyLevel, result) } // FetchBlocksFromPeers mocks base method @@ -3237,42 +3237,42 @@ func (mr *MockclientSessionMockRecorder) Truncate(namespace interface{}) *gomock } // FetchBootstrapBlocksFromPeers mocks base method -func (m *MockclientSession) FetchBootstrapBlocksFromPeers(namespace namespace.Metadata, shard uint32, start, end time.Time, opts result.Options, version FetchBlocksMetadataEndpointVersion) (result.ShardResult, error) { - ret := m.ctrl.Call(m, "FetchBootstrapBlocksFromPeers", namespace, shard, start, end, opts, version) +func (m *MockclientSession) FetchBootstrapBlocksFromPeers(namespace namespace.Metadata, shard uint32, start, end time.Time, opts result.Options) (result.ShardResult, error) { + ret := m.ctrl.Call(m, "FetchBootstrapBlocksFromPeers", namespace, shard, start, end, opts) ret0, _ := ret[0].(result.ShardResult) ret1, _ := ret[1].(error) return ret0, ret1 } // FetchBootstrapBlocksFromPeers indicates an expected call of FetchBootstrapBlocksFromPeers -func (mr *MockclientSessionMockRecorder) FetchBootstrapBlocksFromPeers(namespace, shard, start, end, opts, version interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchBootstrapBlocksFromPeers", reflect.TypeOf((*MockclientSession)(nil).FetchBootstrapBlocksFromPeers), namespace, shard, start, end, opts, version) +func (mr *MockclientSessionMockRecorder) FetchBootstrapBlocksFromPeers(namespace, shard, start, end, opts interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchBootstrapBlocksFromPeers", reflect.TypeOf((*MockclientSession)(nil).FetchBootstrapBlocksFromPeers), namespace, shard, start, end, opts) } // FetchBootstrapBlocksMetadataFromPeers mocks base method -func (m *MockclientSession) FetchBootstrapBlocksMetadataFromPeers(namespace ident.ID, shard uint32, start, end time.Time, result result.Options, version FetchBlocksMetadataEndpointVersion) (PeerBlockMetadataIter, error) { - ret := m.ctrl.Call(m, "FetchBootstrapBlocksMetadataFromPeers", namespace, shard, start, end, result, version) +func (m *MockclientSession) FetchBootstrapBlocksMetadataFromPeers(namespace ident.ID, shard uint32, start, end time.Time, result result.Options) (PeerBlockMetadataIter, error) { + ret := m.ctrl.Call(m, "FetchBootstrapBlocksMetadataFromPeers", namespace, shard, start, end, result) ret0, _ := ret[0].(PeerBlockMetadataIter) ret1, _ := ret[1].(error) return ret0, ret1 } // FetchBootstrapBlocksMetadataFromPeers indicates an expected call of FetchBootstrapBlocksMetadataFromPeers -func (mr *MockclientSessionMockRecorder) FetchBootstrapBlocksMetadataFromPeers(namespace, shard, start, end, result, version interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchBootstrapBlocksMetadataFromPeers", reflect.TypeOf((*MockclientSession)(nil).FetchBootstrapBlocksMetadataFromPeers), namespace, shard, start, end, result, version) +func (mr *MockclientSessionMockRecorder) FetchBootstrapBlocksMetadataFromPeers(namespace, shard, start, end, result interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchBootstrapBlocksMetadataFromPeers", reflect.TypeOf((*MockclientSession)(nil).FetchBootstrapBlocksMetadataFromPeers), namespace, shard, start, end, result) } // FetchBlocksMetadataFromPeers mocks base method -func (m *MockclientSession) FetchBlocksMetadataFromPeers(namespace ident.ID, shard uint32, start, end time.Time, consistencyLevel topology.ReadConsistencyLevel, result result.Options, version FetchBlocksMetadataEndpointVersion) (PeerBlockMetadataIter, error) { - ret := m.ctrl.Call(m, "FetchBlocksMetadataFromPeers", namespace, shard, start, end, consistencyLevel, result, version) +func (m *MockclientSession) FetchBlocksMetadataFromPeers(namespace ident.ID, shard uint32, start, end time.Time, consistencyLevel topology.ReadConsistencyLevel, result result.Options) (PeerBlockMetadataIter, error) { + ret := m.ctrl.Call(m, "FetchBlocksMetadataFromPeers", namespace, shard, start, end, consistencyLevel, result) ret0, _ := ret[0].(PeerBlockMetadataIter) ret1, _ := ret[1].(error) return ret0, ret1 } // FetchBlocksMetadataFromPeers indicates an expected call of FetchBlocksMetadataFromPeers -func (mr *MockclientSessionMockRecorder) FetchBlocksMetadataFromPeers(namespace, shard, start, end, consistencyLevel, result, version interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchBlocksMetadataFromPeers", reflect.TypeOf((*MockclientSession)(nil).FetchBlocksMetadataFromPeers), namespace, shard, start, end, consistencyLevel, result, version) +func (mr *MockclientSessionMockRecorder) FetchBlocksMetadataFromPeers(namespace, shard, start, end, consistencyLevel, result interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchBlocksMetadataFromPeers", reflect.TypeOf((*MockclientSession)(nil).FetchBlocksMetadataFromPeers), namespace, shard, start, end, consistencyLevel, result) } // FetchBlocksFromPeers mocks base method diff --git a/src/dbnode/client/session.go b/src/dbnode/client/session.go index 24193c5d12..3767943af0 100644 --- a/src/dbnode/client/session.go +++ b/src/dbnode/client/session.go @@ -25,7 +25,6 @@ import ( "errors" "fmt" "math" - "reflect" "sort" "strings" "sync" @@ -80,26 +79,7 @@ const ( resultTypeRaw = "raw" ) -// FetchBlocksMetadataEndpointVersion represents an endpoint version for the -// fetch blocks metadata endpoint -type FetchBlocksMetadataEndpointVersion int - -const ( - // FetchBlocksMetadataEndpointDefault represents to use the default fetch blocks metadata endpoint - FetchBlocksMetadataEndpointDefault FetchBlocksMetadataEndpointVersion = iota - // FetchBlocksMetadataEndpointV1 represents v1 of the fetch blocks metadata endpoint - FetchBlocksMetadataEndpointV1 - // FetchBlocksMetadataEndpointV2 represents v2 of the fetch blocks metadata endpoint - FetchBlocksMetadataEndpointV2 = FetchBlocksMetadataEndpointDefault -) - var ( - validFetchBlocksMetadataEndpoints = []FetchBlocksMetadataEndpointVersion{ - FetchBlocksMetadataEndpointV1, - FetchBlocksMetadataEndpointV2, - } - errFetchBlocksMetadataEndpointVersionUnspecified = errors.New( - "fetch blocks metadata endpoint version unspecified") errUnknownWriteAttemptType = errors.New( "unknown write attempt type specified, internal error") ) @@ -122,9 +102,6 @@ var ( errSessionInvalidConnectClusterConnectConsistencyLevel = errors.New("session has invalid connect consistency level specified") // errSessionHasNoHostQueueForHost is raised when host queue requested for a missing host errSessionHasNoHostQueueForHost = errors.New("session has no host queue for host") - // errInvalidFetchBlocksMetadataVersion is raised when an invalid fetch blocks - // metadata endpoint version is provided - errInvalidFetchBlocksMetadataVersion = errors.New("invalid fetch blocks metadata endpoint version") // errUnableToEncodeTags is raised when the server is unable to encode provided tags // to be sent over the wire. errUnableToEncodeTags = errors.New("unable to include tags") @@ -1707,11 +1684,10 @@ func (s *session) FetchBootstrapBlocksMetadataFromPeers( shard uint32, start, end time.Time, resultOpts result.Options, - version FetchBlocksMetadataEndpointVersion, ) (PeerBlockMetadataIter, error) { level := newSessionBootstrapRuntimeReadConsistencyLevel(s) return s.fetchBlocksMetadataFromPeers(namespace, - shard, start, end, level, resultOpts, version) + shard, start, end, level, resultOpts) } func (s *session) FetchBlocksMetadataFromPeers( @@ -1720,11 +1696,10 @@ func (s *session) FetchBlocksMetadataFromPeers( start, end time.Time, consistencyLevel topology.ReadConsistencyLevel, resultOpts result.Options, - version FetchBlocksMetadataEndpointVersion, ) (PeerBlockMetadataIter, error) { level := newStaticRuntimeReadConsistencyLevel(consistencyLevel) return s.fetchBlocksMetadataFromPeers(namespace, - shard, start, end, level, resultOpts, version) + shard, start, end, level, resultOpts) } func (s *session) fetchBlocksMetadataFromPeers( @@ -1733,7 +1708,6 @@ func (s *session) fetchBlocksMetadataFromPeers( start, end time.Time, level runtimeReadConsistencyLevel, resultOpts result.Options, - version FetchBlocksMetadataEndpointVersion, ) (PeerBlockMetadataIter, error) { peers, err := s.peersForShard(shard) if err != nil { @@ -1749,7 +1723,7 @@ func (s *session) fetchBlocksMetadataFromPeers( ) go func() { errCh <- s.streamBlocksMetadataFromPeers(namespace, shard, - peers, start, end, level, metadataCh, resultOpts, m, version) + peers, start, end, level, metadataCh, resultOpts, m) close(metadataCh) close(errCh) }() @@ -1766,12 +1740,7 @@ func (s *session) FetchBootstrapBlocksFromPeers( shard uint32, start, end time.Time, opts result.Options, - version FetchBlocksMetadataEndpointVersion, ) (result.ShardResult, error) { - if !IsValidFetchBlocksMetadataEndpoint(version) { - return nil, errInvalidFetchBlocksMetadataVersion - } - var ( result = newBulkBlocksResult(s.opts, opts, s.pools.tagDecoder, s.pools.id) @@ -1810,7 +1779,7 @@ func (s *session) FetchBootstrapBlocksFromPeers( errCh := make(chan error, 1) go func() { errCh <- s.streamBlocksMetadataFromPeers(nsMetadata.ID(), shard, - peers, start, end, level, metadataCh, opts, progress, version) + peers, start, end, level, metadataCh, opts, progress) close(metadataCh) }() @@ -1920,7 +1889,6 @@ func (s *session) streamBlocksMetadataFromPeers( metadataCh chan<- receivedBlockMetadata, resultOpts result.Options, progress *streamFromPeersMetrics, - version FetchBlocksMetadataEndpointVersion, ) error { var ( wg sync.WaitGroup @@ -1988,18 +1956,8 @@ func (s *session) streamBlocksMetadataFromPeers( } for condition() { var err error - switch version { - case FetchBlocksMetadataEndpointV1: - currPageToken, err = s.streamBlocksMetadataFromPeer(namespace, shardID, - peer, start, end, currPageToken, metadataCh, progress) - case FetchBlocksMetadataEndpointV2: - currPageToken, err = s.streamBlocksMetadataFromPeerV2(namespace, shardID, - peer, start, end, currPageToken, metadataCh, resultOpts, progress) - default: - // Should never happen - we validate the version before this function is - // ever called - err = xerrors.NewNonRetryableError(errInvalidFetchBlocksMetadataVersion) - } + currPageToken, err = s.streamBlocksMetadataFromPeer(namespace, shardID, + peer, start, end, currPageToken, metadataCh, resultOpts, progress) // Set error or success if err is nil errs.setError(idx, err) @@ -2028,178 +1986,12 @@ func (s *session) streamBlocksMetadataFromPeers( atomic.LoadInt32(&responded), int32(len(errors)), errors) } -// pageToken is just an opaque type that needs to be downcasted to expected -// page token type, this makes it easy to use the page token across the two -// versions -// TODO(r): Delete this once we delete the V1 code path -type pageToken interface{} - -// TODO(rartoul): Delete this once we delete the V1 code path -func (s *session) streamBlocksMetadataFromPeer( - namespace ident.ID, - shard uint32, - peer peer, - start, end time.Time, - startPageToken pageToken, - ch chan<- receivedBlockMetadata, - progress *streamFromPeersMetrics, -) (pageToken, error) { - var pageToken *int64 - if startPageToken != nil { - var ok bool - pageToken, ok = startPageToken.(*int64) - if !ok { - err := fmt.Errorf("unexpected start page token type: %s", - reflect.TypeOf(startPageToken).Elem().String()) - return nil, xerrors.NewNonRetryableError(err) - } - } - - var ( - optionIncludeSizes = true - optionIncludeChecksums = true - optionIncludeLastRead = true - moreResults = true - - // Only used for logs - peerStr = peer.Host().ID() - metadataCountByBlock = map[xtime.UnixNano]int64{} - ) - - // Only used for logs - defer func() { - for block, numMetadata := range metadataCountByBlock { - s.log.WithFields( - xlog.NewField("shard", shard), - xlog.NewField("peer", peerStr), - xlog.NewField("numMetadata", numMetadata), - xlog.NewField("block", block), - ).Debug("finished streaming blocks metadata from peer") - } - }() - - // Declare before loop to avoid redeclaring each iteration - attemptFn := func(client rpc.TChanNode) error { - tctx, _ := thrift.NewContext(s.streamBlocksMetadataBatchTimeout) - req := rpc.NewFetchBlocksMetadataRawRequest() - req.NameSpace = namespace.Bytes() - req.Shard = int32(shard) - req.RangeStart = start.UnixNano() - req.RangeEnd = end.UnixNano() - req.Limit = int64(s.streamBlocksBatchSize) - req.PageToken = pageToken - req.IncludeSizes = &optionIncludeSizes - req.IncludeChecksums = &optionIncludeChecksums - req.IncludeLastRead = &optionIncludeLastRead - - progress.metadataFetchBatchCall.Inc(1) - result, err := client.FetchBlocksMetadataRaw(tctx, req) - if err != nil { - progress.metadataFetchBatchError.Inc(1) - return err - } - - progress.metadataFetchBatchSuccess.Inc(1) - progress.metadataReceived.Inc(int64(len(result.Elements))) +type pageToken []byte - if result.NextPageToken != nil { - // Create space on the heap for the page token and take it's - // address to avoid having to keep the entire result around just - // for the page token - resultPageToken := *result.NextPageToken - pageToken = &resultPageToken - } else { - // No further results - moreResults = false - } - - for _, elem := range result.Elements { - blockID := ident.BinaryID(checked.NewBytes(elem.ID, nil)) - for _, b := range elem.Blocks { - blockStart := time.Unix(0, b.Start) - - // Error occurred retrieving block metadata, use default values - if b.Err != nil { - progress.metadataFetchBatchBlockErr.Inc(1) - s.log.WithFields( - xlog.NewField("shard", shard), - xlog.NewField("peer", peerStr), - xlog.NewField("block", blockStart), - xlog.NewField("error", err), - ).Error("error occurred retrieving block metadata") - // Enqueue with a zeroed checksum which triggers a fanout fetch - ch <- receivedBlockMetadata{ - peer: peer, - id: blockID, - block: blockMetadata{ - start: blockStart, - }, - } - continue - } - - var size int64 - if b.Size != nil { - size = *b.Size - } - - var pChecksum *uint32 - if b.Checksum != nil { - value := uint32(*b.Checksum) - pChecksum = &value - } - - var lastRead time.Time - if b.LastRead != nil { - value, err := convert.ToTime(*b.LastRead, b.LastReadTimeType) - if err == nil { - lastRead = value - } - } - - ch <- receivedBlockMetadata{ - peer: peer, - id: blockID, - block: blockMetadata{ - start: blockStart, - size: size, - checksum: pChecksum, - lastRead: lastRead, - }, - } - - // Only used for logs - metadataCountByBlock[xtime.ToUnixNano(blockStart)]++ - } - } - - return nil - } - - // NB(r): split the following methods up so they don't allocate - // a closure per fetch blocks call - var attemptErr error - checkedAttemptFn := func(client rpc.TChanNode) { - attemptErr = attemptFn(client) - } - - fetchFn := func() error { - borrowErr := peer.BorrowConnection(checkedAttemptFn) - return xerrors.FirstError(borrowErr, attemptErr) - } - - for moreResults { - if err := s.streamBlocksRetrier.Attempt(fetchFn); err != nil { - return pageToken, err - } - } - return nil, nil -} - -// streamBlocksMetadataFromPeerV2 has several heap allocated anonymous +// streamBlocksMetadataFromPeer has several heap allocated anonymous // function, however, they're only allocated once per peer/shard combination // for the entire peer bootstrapping process so performance is acceptable -func (s *session) streamBlocksMetadataFromPeerV2( +func (s *session) streamBlocksMetadataFromPeer( namespace ident.ID, shard uint32, peer peer, @@ -2209,17 +2001,6 @@ func (s *session) streamBlocksMetadataFromPeerV2( resultOpts result.Options, progress *streamFromPeersMetrics, ) (pageToken, error) { - var pageToken []byte - if startPageToken != nil { - var ok bool - pageToken, ok = startPageToken.([]byte) - if !ok { - err := fmt.Errorf("unexpected start page token type: %s", - reflect.TypeOf(startPageToken).Elem().String()) - return nil, xerrors.NewNonRetryableError(err) - } - } - var ( optionIncludeSizes = true optionIncludeChecksums = true @@ -2252,7 +2033,7 @@ func (s *session) streamBlocksMetadataFromPeerV2( req.RangeStart = start.UnixNano() req.RangeEnd = end.UnixNano() req.Limit = int64(s.streamBlocksBatchSize) - req.PageToken = pageToken + req.PageToken = startPageToken req.IncludeSizes = &optionIncludeSizes req.IncludeChecksums = &optionIncludeChecksums req.IncludeLastRead = &optionIncludeLastRead @@ -2270,7 +2051,7 @@ func (s *session) streamBlocksMetadataFromPeerV2( if result.NextPageToken != nil { // Reset pageToken + copy new pageToken into previously allocated memory, // extending as necessary - pageToken = append(pageToken[:0], result.NextPageToken...) + startPageToken = append(startPageToken[:0], result.NextPageToken...) } else { // No further results moreResults = false @@ -2363,7 +2144,7 @@ func (s *session) streamBlocksMetadataFromPeerV2( for moreResults { if err := s.streamBlocksRetrier.Attempt(fetchFn); err != nil { - return pageToken, err + return startPageToken, err } } return nil, nil @@ -3872,48 +3653,6 @@ type idAndBlockStart struct { blockStart int64 } -// IsValidFetchBlocksMetadataEndpoint returns a bool indicating whether the -// specified endpointVersion is valid -func IsValidFetchBlocksMetadataEndpoint( - endpointVersion FetchBlocksMetadataEndpointVersion, -) bool { - for _, version := range validFetchBlocksMetadataEndpoints { - if version == endpointVersion { - return true - } - } - return false -} - -// UnmarshalYAML unmarshals an FetchBlocksMetadataEndpointVersion into a valid type from string. -func (v *FetchBlocksMetadataEndpointVersion) UnmarshalYAML(unmarshal func(interface{}) error) error { - var str string - if err := unmarshal(&str); err != nil { - return err - } - if str == "" { - return errFetchBlocksMetadataEndpointVersionUnspecified - } - for _, valid := range validFetchBlocksMetadataEndpoints { - if str == valid.String() { - *v = valid - return nil - } - } - return fmt.Errorf("invalid FetchBlocksMetadataEndpointVersion '%s' valid types are: %v", - str, validFetchBlocksMetadataEndpoints) -} - -func (v FetchBlocksMetadataEndpointVersion) String() string { - switch v { - case FetchBlocksMetadataEndpointV1: - return "v1" - case FetchBlocksMetadataEndpointV2: - return "v2" - } - return "unknown" -} - func newTagsFromEncodedTags( seriesID ident.ID, encodedTags checked.Bytes, diff --git a/src/dbnode/client/session_fetch_bulk_blocks_test.go b/src/dbnode/client/session_fetch_bulk_blocks_test.go index 8c4232fce3..6a47c7c296 100644 --- a/src/dbnode/client/session_fetch_bulk_blocks_test.go +++ b/src/dbnode/client/session_fetch_bulk_blocks_test.go @@ -131,141 +131,6 @@ func testPeers(v []peer) peers { return peers{peers: v, majorityReplicas: topology.Majority(len(v))} } -// TODO(rartoul): Delete when we delete the V1 code path -func TestFetchBootstrapBlocksAllPeersSucceed(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - opts := newSessionTestAdminOptions() - s, err := newSession(opts) - require.NoError(t, err) - session := s.(*session) - - mockHostQueues, mockClients := mockHostQueuesAndClientsForFetchBootstrapBlocks(ctrl, opts) - session.newHostQueueFn = mockHostQueues.newHostQueueFn() - - // Don't drain the peer blocks queue, explicitly drain ourselves to - // avoid unpredictable batches being retrieved from peers - var ( - qs []*peerBlocksQueue - qsMutex sync.RWMutex - ) - session.newPeerBlocksQueueFn = func( - peer peer, - maxQueueSize int, - _ time.Duration, - workers xsync.WorkerPool, - processFn processFn, - ) *peerBlocksQueue { - qsMutex.Lock() - defer qsMutex.Unlock() - q := newPeerBlocksQueue(peer, maxQueueSize, 0, workers, processFn) - qs = append(qs, q) - return q - } - - require.NoError(t, session.Open()) - - batchSize := opts.FetchSeriesBlocksBatchSize() - - start := time.Now().Truncate(blockSize).Add(blockSize * -(24 - 1)) - - blocks := []testBlocks{ - { - id: fooID, - blocks: []testBlock{ - { - start: start.Add(blockSize * 1), - segments: &testBlockSegments{merged: &testBlockSegment{ - head: []byte{1, 2}, - tail: []byte{3}, - }}, - }, - }, - }, - { - id: barID, - blocks: []testBlock{ - { - start: start.Add(blockSize * 2), - segments: &testBlockSegments{merged: &testBlockSegment{ - head: []byte{4, 5}, - tail: []byte{6}, - }}, - }, - }, - }, - { - id: bazID, - blocks: []testBlock{ - { - start: start.Add(blockSize * 3), - segments: &testBlockSegments{merged: &testBlockSegment{ - head: []byte{7, 8}, - tail: []byte{9}, - }}, - }, - }, - }, - } - - // Expect the fetch metadata calls - metadataResult := resultMetadataFromBlocks(blocks) - // Skip the first client which is the client for the origin - mockClients[1:].expectFetchMetadataAndReturn(metadataResult, opts, false) - - // Expect the fetch blocks calls - participating := len(mockClients) - 1 - blocksExpectedReqs, blocksResult := expectedReqsAndResultFromBlocks(t, - blocks, batchSize, participating, - func(blockIdx int) (clientIdx int) { - // Round robin to match the best peer selection algorithm - return blockIdx % participating - }) - // Skip the first client which is the client for the origin - for i, client := range mockClients[1:] { - expectFetchBlocksAndReturn(client, blocksExpectedReqs[i], blocksResult[i]) - } - - // Make sure peer selection is round robin to match our expected - // peer fetch calls - session.pickBestPeerFn = newRoundRobinPickBestPeerFn() - - // Fetch blocks - go func() { - // Trigger peer queues to drain explicitly when all work enqueued - for { - qsMutex.RLock() - assigned := 0 - for _, q := range qs { - assigned += int(atomic.LoadUint64(&q.assigned)) - } - qsMutex.RUnlock() - if assigned == len(blocks) { - qsMutex.Lock() - defer qsMutex.Unlock() - for _, q := range qs { - q.drain() - } - return - } - time.Sleep(10 * time.Millisecond) - } - }() - rangeStart := start - rangeEnd := start.Add(blockSize * (24 - 1)) - bootstrapOpts := newResultTestOptions() - result, err := session.FetchBootstrapBlocksFromPeers( - testsNsMetadata(t), 0, rangeStart, rangeEnd, bootstrapOpts, FetchBlocksMetadataEndpointV1) - assert.NoError(t, err) - assert.NotNil(t, result) - - // Assert result - assertFetchBootstrapBlocksResult(t, blocks, result) - - assert.NoError(t, session.Close()) -} - func newRoundRobinPickBestPeerFn() pickBestPeerFn { calls := int32(0) return func( @@ -360,7 +225,7 @@ func TestFetchBootstrapBlocksAllPeersSucceedV2(t *testing.T) { // Expect the fetch metadata calls metadataResult := resultMetadataFromBlocks(blocks) // Skip the first client which is the client for the origin - mockClients[1:].expectFetchMetadataAndReturn(metadataResult, opts, true) + mockClients[1:].expectFetchMetadataAndReturn(metadataResult, opts) // Expect the fetch blocks calls participating := len(mockClients) - 1 @@ -404,7 +269,7 @@ func TestFetchBootstrapBlocksAllPeersSucceedV2(t *testing.T) { rangeEnd := start.Add(blockSize * (24 - 1)) bootstrapOpts := newResultTestOptions() result, err := session.FetchBootstrapBlocksFromPeers( - testsNsMetadata(t), 0, rangeStart, rangeEnd, bootstrapOpts, FetchBlocksMetadataEndpointV2) + testsNsMetadata(t), 0, rangeStart, rangeEnd, bootstrapOpts) assert.NoError(t, err) assert.NotNil(t, result) @@ -1914,14 +1779,9 @@ type MockTChanNodes []*rpc.MockTChanNode func (c MockTChanNodes) expectFetchMetadataAndReturn( result []testBlocksMetadata, opts AdminOptions, - isV2 bool, ) { for _, client := range c { - if isV2 { - expectFetchMetadataAndReturnV2(client, result, opts) - } else { - expectFetchMetadataAndReturn(client, result, opts) - } + expectFetchMetadataAndReturn(client, result, opts) } } @@ -2089,7 +1949,6 @@ func expectedReqsAndResultFromBlocks( return clientsExpectReqs, clientsBlocksResult } -// TODO(rartoul): Delete when we delete the V1 code path func expectFetchMetadataAndReturn( client *rpc.MockTChanNode, result []testBlocksMetadata, @@ -2099,60 +1958,6 @@ func expectFetchMetadataAndReturn( totalCalls := int(math.Ceil(float64(len(result)) / float64(batchSize))) includeSizes := true - var calls []*gomock.Call - for i := 0; i < totalCalls; i++ { - var ( - ret = &rpc.FetchBlocksMetadataRawResult_{} - beginIdx = i * batchSize - nextIdx = int64(0) - ) - for j := beginIdx; j < len(result) && j < beginIdx+batchSize; j++ { - elem := &rpc.BlocksMetadata{} - elem.ID = result[j].id.Bytes() - for k := 0; k < len(result[j].blocks); k++ { - bl := &rpc.BlockMetadata{} - bl.Start = result[j].blocks[k].start.UnixNano() - bl.Size = result[j].blocks[k].size - if result[j].blocks[k].checksum != nil { - checksum := int64(*result[j].blocks[k].checksum) - bl.Checksum = &checksum - } - elem.Blocks = append(elem.Blocks, bl) - } - ret.Elements = append(ret.Elements, elem) - nextIdx = int64(j) + 1 - } - if i != totalCalls-1 { - // Include next page token if not last page - ret.NextPageToken = &nextIdx - } - - matcher := &fetchMetadataReqMatcher{ - shard: 0, - limit: int64(batchSize), - includeSizes: &includeSizes, - } - if i != 0 { - expectPageToken := int64(beginIdx) - matcher.pageToken = &expectPageToken - } - - call := client.EXPECT().FetchBlocksMetadataRaw(gomock.Any(), matcher).Return(ret, nil) - calls = append(calls, call) - } - - gomock.InOrder(calls...) -} - -func expectFetchMetadataAndReturnV2( - client *rpc.MockTChanNode, - result []testBlocksMetadata, - opts AdminOptions, -) { - batchSize := opts.FetchSeriesBlocksBatchSize() - totalCalls := int(math.Ceil(float64(len(result)) / float64(batchSize))) - includeSizes := true - var calls []*gomock.Call for i := 0; i < totalCalls; i++ { var ( @@ -2185,7 +1990,7 @@ func expectFetchMetadataAndReturnV2( isV2: true, } if i != 0 { - matcher.pageTokenV2 = []byte(fmt.Sprintf("token_%d", i)) + matcher.pageToken = []byte(fmt.Sprintf("token_%d", i)) } call := client.EXPECT().FetchBlocksMetadataRawV2(gomock.Any(), matcher).Return(ret, nil) @@ -2198,64 +2003,12 @@ func expectFetchMetadataAndReturnV2( type fetchMetadataReqMatcher struct { shard int32 limit int64 - pageToken *int64 - pageTokenV2 []byte + pageToken []byte includeSizes *bool isV2 bool } func (m *fetchMetadataReqMatcher) Matches(x interface{}) bool { - if m.isV2 { - return m.matchesV2(x) - } - return m.matchesV1(x) -} - -// TODO(rartoul): Delete when we delete the V1 code path -func (m *fetchMetadataReqMatcher) matchesV1(x interface{}) bool { - req, ok := x.(*rpc.FetchBlocksMetadataRawRequest) - if !ok { - return false - } - - if m.shard != req.Shard { - return false - } - - if m.limit != req.Limit { - return false - } - - if m.pageToken == nil { - if req.PageToken != nil { - return false - } - } else { - if req.PageToken == nil { - return false - } - if *req.PageToken != *m.pageToken { - return false - } - } - - if m.includeSizes == nil { - if req.IncludeSizes != nil { - return false - } - } else { - if req.IncludeSizes == nil { - return false - } - if *req.IncludeSizes != *m.includeSizes { - return false - } - } - - return true -} - -func (m *fetchMetadataReqMatcher) matchesV2(x interface{}) bool { req, ok := x.(*rpc.FetchBlocksMetadataRawV2Request) if !ok { return false @@ -2269,7 +2022,7 @@ func (m *fetchMetadataReqMatcher) matchesV2(x interface{}) bool { return false } - if m.pageTokenV2 == nil { + if m.pageToken == nil { if req.PageToken != nil { return false } @@ -2277,7 +2030,7 @@ func (m *fetchMetadataReqMatcher) matchesV2(x interface{}) bool { if req.PageToken == nil { return false } - if !bytes.Equal(req.PageToken, m.pageTokenV2) { + if !bytes.Equal(req.PageToken, m.pageToken) { return false } } diff --git a/src/dbnode/client/types.go b/src/dbnode/client/types.go index 13ef76a812..3464b7c501 100644 --- a/src/dbnode/client/types.go +++ b/src/dbnode/client/types.go @@ -170,7 +170,6 @@ type AdminSession interface { shard uint32, start, end time.Time, opts result.Options, - version FetchBlocksMetadataEndpointVersion, ) (result.ShardResult, error) // FetchBootstrapBlocksMetadataFromPeers will fetch the blocks metadata from @@ -180,7 +179,6 @@ type AdminSession interface { shard uint32, start, end time.Time, result result.Options, - version FetchBlocksMetadataEndpointVersion, ) (PeerBlockMetadataIter, error) // FetchBlocksMetadataFromPeers will fetch the blocks metadata from @@ -191,7 +189,6 @@ type AdminSession interface { start, end time.Time, consistencyLevel topology.ReadConsistencyLevel, result result.Options, - version FetchBlocksMetadataEndpointVersion, ) (PeerBlockMetadataIter, error) // FetchBlocksFromPeers will fetch the required blocks from the diff --git a/src/dbnode/generated/thrift/rpc.thrift b/src/dbnode/generated/thrift/rpc.thrift index 87ea9a8731..3e1e5b0c6d 100644 --- a/src/dbnode/generated/thrift/rpc.thrift +++ b/src/dbnode/generated/thrift/rpc.thrift @@ -53,8 +53,6 @@ service Node { FetchBatchRawResult fetchBatchRaw(1: FetchBatchRawRequest req) throws (1: Error err) FetchBlocksRawResult fetchBlocksRaw(1: FetchBlocksRawRequest req) throws (1: Error err) - // TODO(rartoul): Delete this once we delete the V1 code path - FetchBlocksMetadataRawResult fetchBlocksMetadataRaw(1: FetchBlocksMetadataRawRequest req) throws (1: Error err) FetchBlocksMetadataRawV2Result fetchBlocksMetadataRawV2(1: FetchBlocksMetadataRawV2Request req) throws (1: Error err) void writeBatchRaw(1: WriteBatchRawRequest req) throws (1: WriteBatchRawErrors err) void writeTaggedBatchRaw(1: WriteTaggedBatchRawRequest req) throws (1: WriteBatchRawErrors err) @@ -191,41 +189,6 @@ struct Tag { 2: required string value } -// TODO(rartoul): Delete this once we delete the V1 code path -struct FetchBlocksMetadataRawRequest { - 1: required binary nameSpace - 2: required i32 shard - 3: required i64 rangeStart - 4: required i64 rangeEnd - 5: required i64 limit - 6: optional i64 pageToken - 7: optional bool includeSizes - 8: optional bool includeChecksums - 9: optional bool includeLastRead -} - -// TODO(rartoul): Delete this once we delete the V1 code path -struct FetchBlocksMetadataRawResult { - 1: required list elements - 2: optional i64 nextPageToken -} - -// TODO(rartoul): Delete this once we delete the V1 code path -struct BlocksMetadata { - 1: required binary id - 2: required list blocks -} - -// TODO(rartoul): Delete this once we delete the V1 code path -struct BlockMetadata { - 1: optional Error err - 2: required i64 start - 3: optional i64 size - 4: optional i64 checksum - 5: optional i64 lastRead - 6: optional TimeType lastReadTimeType = TimeType.UNIX_SECONDS -} - struct FetchBlocksMetadataRawV2Request { 1: required binary nameSpace 2: required i32 shard diff --git a/src/dbnode/generated/thrift/rpc/rpc.go b/src/dbnode/generated/thrift/rpc/rpc.go index f179472d83..00603dec31 100644 --- a/src/dbnode/generated/thrift/rpc/rpc.go +++ b/src/dbnode/generated/thrift/rpc/rpc.go @@ -4298,1103 +4298,6 @@ func (p *Tag) String() string { return fmt.Sprintf("Tag(%+v)", *p) } -// Attributes: -// - NameSpace -// - Shard -// - RangeStart -// - RangeEnd -// - Limit -// - PageToken -// - IncludeSizes -// - IncludeChecksums -// - IncludeLastRead -type FetchBlocksMetadataRawRequest struct { - NameSpace []byte `thrift:"nameSpace,1,required" db:"nameSpace" json:"nameSpace"` - Shard int32 `thrift:"shard,2,required" db:"shard" json:"shard"` - RangeStart int64 `thrift:"rangeStart,3,required" db:"rangeStart" json:"rangeStart"` - RangeEnd int64 `thrift:"rangeEnd,4,required" db:"rangeEnd" json:"rangeEnd"` - Limit int64 `thrift:"limit,5,required" db:"limit" json:"limit"` - PageToken *int64 `thrift:"pageToken,6" db:"pageToken" json:"pageToken,omitempty"` - IncludeSizes *bool `thrift:"includeSizes,7" db:"includeSizes" json:"includeSizes,omitempty"` - IncludeChecksums *bool `thrift:"includeChecksums,8" db:"includeChecksums" json:"includeChecksums,omitempty"` - IncludeLastRead *bool `thrift:"includeLastRead,9" db:"includeLastRead" json:"includeLastRead,omitempty"` -} - -func NewFetchBlocksMetadataRawRequest() *FetchBlocksMetadataRawRequest { - return &FetchBlocksMetadataRawRequest{} -} - -func (p *FetchBlocksMetadataRawRequest) GetNameSpace() []byte { - return p.NameSpace -} - -func (p *FetchBlocksMetadataRawRequest) GetShard() int32 { - return p.Shard -} - -func (p *FetchBlocksMetadataRawRequest) GetRangeStart() int64 { - return p.RangeStart -} - -func (p *FetchBlocksMetadataRawRequest) GetRangeEnd() int64 { - return p.RangeEnd -} - -func (p *FetchBlocksMetadataRawRequest) GetLimit() int64 { - return p.Limit -} - -var FetchBlocksMetadataRawRequest_PageToken_DEFAULT int64 - -func (p *FetchBlocksMetadataRawRequest) GetPageToken() int64 { - if !p.IsSetPageToken() { - return FetchBlocksMetadataRawRequest_PageToken_DEFAULT - } - return *p.PageToken -} - -var FetchBlocksMetadataRawRequest_IncludeSizes_DEFAULT bool - -func (p *FetchBlocksMetadataRawRequest) GetIncludeSizes() bool { - if !p.IsSetIncludeSizes() { - return FetchBlocksMetadataRawRequest_IncludeSizes_DEFAULT - } - return *p.IncludeSizes -} - -var FetchBlocksMetadataRawRequest_IncludeChecksums_DEFAULT bool - -func (p *FetchBlocksMetadataRawRequest) GetIncludeChecksums() bool { - if !p.IsSetIncludeChecksums() { - return FetchBlocksMetadataRawRequest_IncludeChecksums_DEFAULT - } - return *p.IncludeChecksums -} - -var FetchBlocksMetadataRawRequest_IncludeLastRead_DEFAULT bool - -func (p *FetchBlocksMetadataRawRequest) GetIncludeLastRead() bool { - if !p.IsSetIncludeLastRead() { - return FetchBlocksMetadataRawRequest_IncludeLastRead_DEFAULT - } - return *p.IncludeLastRead -} -func (p *FetchBlocksMetadataRawRequest) IsSetPageToken() bool { - return p.PageToken != nil -} - -func (p *FetchBlocksMetadataRawRequest) IsSetIncludeSizes() bool { - return p.IncludeSizes != nil -} - -func (p *FetchBlocksMetadataRawRequest) IsSetIncludeChecksums() bool { - return p.IncludeChecksums != nil -} - -func (p *FetchBlocksMetadataRawRequest) IsSetIncludeLastRead() bool { - return p.IncludeLastRead != nil -} - -func (p *FetchBlocksMetadataRawRequest) Read(iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetNameSpace bool = false - var issetShard bool = false - var issetRangeStart bool = false - var issetRangeEnd bool = false - var issetLimit bool = false - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin() - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { - break - } - switch fieldId { - case 1: - if err := p.ReadField1(iprot); err != nil { - return err - } - issetNameSpace = true - case 2: - if err := p.ReadField2(iprot); err != nil { - return err - } - issetShard = true - case 3: - if err := p.ReadField3(iprot); err != nil { - return err - } - issetRangeStart = true - case 4: - if err := p.ReadField4(iprot); err != nil { - return err - } - issetRangeEnd = true - case 5: - if err := p.ReadField5(iprot); err != nil { - return err - } - issetLimit = true - case 6: - if err := p.ReadField6(iprot); err != nil { - return err - } - case 7: - if err := p.ReadField7(iprot); err != nil { - return err - } - case 8: - if err := p.ReadField8(iprot); err != nil { - return err - } - case 9: - if err := p.ReadField9(iprot); err != nil { - return err - } - default: - if err := iprot.Skip(fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetNameSpace { - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field NameSpace is not set")) - } - if !issetShard { - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Shard is not set")) - } - if !issetRangeStart { - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field RangeStart is not set")) - } - if !issetRangeEnd { - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field RangeEnd is not set")) - } - if !issetLimit { - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Limit is not set")) - } - return nil -} - -func (p *FetchBlocksMetadataRawRequest) ReadField1(iprot thrift.TProtocol) error { - if v, err := iprot.ReadBinary(); err != nil { - return thrift.PrependError("error reading field 1: ", err) - } else { - p.NameSpace = v - } - return nil -} - -func (p *FetchBlocksMetadataRawRequest) ReadField2(iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(); err != nil { - return thrift.PrependError("error reading field 2: ", err) - } else { - p.Shard = v - } - return nil -} - -func (p *FetchBlocksMetadataRawRequest) ReadField3(iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(); err != nil { - return thrift.PrependError("error reading field 3: ", err) - } else { - p.RangeStart = v - } - return nil -} - -func (p *FetchBlocksMetadataRawRequest) ReadField4(iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(); err != nil { - return thrift.PrependError("error reading field 4: ", err) - } else { - p.RangeEnd = v - } - return nil -} - -func (p *FetchBlocksMetadataRawRequest) ReadField5(iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(); err != nil { - return thrift.PrependError("error reading field 5: ", err) - } else { - p.Limit = v - } - return nil -} - -func (p *FetchBlocksMetadataRawRequest) ReadField6(iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(); err != nil { - return thrift.PrependError("error reading field 6: ", err) - } else { - p.PageToken = &v - } - return nil -} - -func (p *FetchBlocksMetadataRawRequest) ReadField7(iprot thrift.TProtocol) error { - if v, err := iprot.ReadBool(); err != nil { - return thrift.PrependError("error reading field 7: ", err) - } else { - p.IncludeSizes = &v - } - return nil -} - -func (p *FetchBlocksMetadataRawRequest) ReadField8(iprot thrift.TProtocol) error { - if v, err := iprot.ReadBool(); err != nil { - return thrift.PrependError("error reading field 8: ", err) - } else { - p.IncludeChecksums = &v - } - return nil -} - -func (p *FetchBlocksMetadataRawRequest) ReadField9(iprot thrift.TProtocol) error { - if v, err := iprot.ReadBool(); err != nil { - return thrift.PrependError("error reading field 9: ", err) - } else { - p.IncludeLastRead = &v - } - return nil -} - -func (p *FetchBlocksMetadataRawRequest) Write(oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin("FetchBlocksMetadataRawRequest"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) - } - if p != nil { - if err := p.writeField1(oprot); err != nil { - return err - } - if err := p.writeField2(oprot); err != nil { - return err - } - if err := p.writeField3(oprot); err != nil { - return err - } - if err := p.writeField4(oprot); err != nil { - return err - } - if err := p.writeField5(oprot); err != nil { - return err - } - if err := p.writeField6(oprot); err != nil { - return err - } - if err := p.writeField7(oprot); err != nil { - return err - } - if err := p.writeField8(oprot); err != nil { - return err - } - if err := p.writeField9(oprot); err != nil { - return err - } - } - if err := oprot.WriteFieldStop(); err != nil { - return thrift.PrependError("write field stop error: ", err) - } - if err := oprot.WriteStructEnd(); err != nil { - return thrift.PrependError("write struct stop error: ", err) - } - return nil -} - -func (p *FetchBlocksMetadataRawRequest) writeField1(oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin("nameSpace", thrift.STRING, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:nameSpace: ", p), err) - } - if err := oprot.WriteBinary(p.NameSpace); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.nameSpace (1) field write error: ", p), err) - } - if err := oprot.WriteFieldEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:nameSpace: ", p), err) - } - return err -} - -func (p *FetchBlocksMetadataRawRequest) writeField2(oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin("shard", thrift.I32, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:shard: ", p), err) - } - if err := oprot.WriteI32(int32(p.Shard)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.shard (2) field write error: ", p), err) - } - if err := oprot.WriteFieldEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:shard: ", p), err) - } - return err -} - -func (p *FetchBlocksMetadataRawRequest) writeField3(oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin("rangeStart", thrift.I64, 3); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:rangeStart: ", p), err) - } - if err := oprot.WriteI64(int64(p.RangeStart)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.rangeStart (3) field write error: ", p), err) - } - if err := oprot.WriteFieldEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 3:rangeStart: ", p), err) - } - return err -} - -func (p *FetchBlocksMetadataRawRequest) writeField4(oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin("rangeEnd", thrift.I64, 4); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:rangeEnd: ", p), err) - } - if err := oprot.WriteI64(int64(p.RangeEnd)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.rangeEnd (4) field write error: ", p), err) - } - if err := oprot.WriteFieldEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 4:rangeEnd: ", p), err) - } - return err -} - -func (p *FetchBlocksMetadataRawRequest) writeField5(oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin("limit", thrift.I64, 5); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:limit: ", p), err) - } - if err := oprot.WriteI64(int64(p.Limit)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.limit (5) field write error: ", p), err) - } - if err := oprot.WriteFieldEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 5:limit: ", p), err) - } - return err -} - -func (p *FetchBlocksMetadataRawRequest) writeField6(oprot thrift.TProtocol) (err error) { - if p.IsSetPageToken() { - if err := oprot.WriteFieldBegin("pageToken", thrift.I64, 6); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 6:pageToken: ", p), err) - } - if err := oprot.WriteI64(int64(*p.PageToken)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.pageToken (6) field write error: ", p), err) - } - if err := oprot.WriteFieldEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 6:pageToken: ", p), err) - } - } - return err -} - -func (p *FetchBlocksMetadataRawRequest) writeField7(oprot thrift.TProtocol) (err error) { - if p.IsSetIncludeSizes() { - if err := oprot.WriteFieldBegin("includeSizes", thrift.BOOL, 7); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 7:includeSizes: ", p), err) - } - if err := oprot.WriteBool(bool(*p.IncludeSizes)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.includeSizes (7) field write error: ", p), err) - } - if err := oprot.WriteFieldEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 7:includeSizes: ", p), err) - } - } - return err -} - -func (p *FetchBlocksMetadataRawRequest) writeField8(oprot thrift.TProtocol) (err error) { - if p.IsSetIncludeChecksums() { - if err := oprot.WriteFieldBegin("includeChecksums", thrift.BOOL, 8); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 8:includeChecksums: ", p), err) - } - if err := oprot.WriteBool(bool(*p.IncludeChecksums)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.includeChecksums (8) field write error: ", p), err) - } - if err := oprot.WriteFieldEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 8:includeChecksums: ", p), err) - } - } - return err -} - -func (p *FetchBlocksMetadataRawRequest) writeField9(oprot thrift.TProtocol) (err error) { - if p.IsSetIncludeLastRead() { - if err := oprot.WriteFieldBegin("includeLastRead", thrift.BOOL, 9); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 9:includeLastRead: ", p), err) - } - if err := oprot.WriteBool(bool(*p.IncludeLastRead)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.includeLastRead (9) field write error: ", p), err) - } - if err := oprot.WriteFieldEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 9:includeLastRead: ", p), err) - } - } - return err -} - -func (p *FetchBlocksMetadataRawRequest) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("FetchBlocksMetadataRawRequest(%+v)", *p) -} - -// Attributes: -// - Elements -// - NextPageToken -type FetchBlocksMetadataRawResult_ struct { - Elements []*BlocksMetadata `thrift:"elements,1,required" db:"elements" json:"elements"` - NextPageToken *int64 `thrift:"nextPageToken,2" db:"nextPageToken" json:"nextPageToken,omitempty"` -} - -func NewFetchBlocksMetadataRawResult_() *FetchBlocksMetadataRawResult_ { - return &FetchBlocksMetadataRawResult_{} -} - -func (p *FetchBlocksMetadataRawResult_) GetElements() []*BlocksMetadata { - return p.Elements -} - -var FetchBlocksMetadataRawResult__NextPageToken_DEFAULT int64 - -func (p *FetchBlocksMetadataRawResult_) GetNextPageToken() int64 { - if !p.IsSetNextPageToken() { - return FetchBlocksMetadataRawResult__NextPageToken_DEFAULT - } - return *p.NextPageToken -} -func (p *FetchBlocksMetadataRawResult_) IsSetNextPageToken() bool { - return p.NextPageToken != nil -} - -func (p *FetchBlocksMetadataRawResult_) Read(iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetElements bool = false - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin() - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { - break - } - switch fieldId { - case 1: - if err := p.ReadField1(iprot); err != nil { - return err - } - issetElements = true - case 2: - if err := p.ReadField2(iprot); err != nil { - return err - } - default: - if err := iprot.Skip(fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetElements { - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Elements is not set")) - } - return nil -} - -func (p *FetchBlocksMetadataRawResult_) ReadField1(iprot thrift.TProtocol) error { - _, size, err := iprot.ReadListBegin() - if err != nil { - return thrift.PrependError("error reading list begin: ", err) - } - tSlice := make([]*BlocksMetadata, 0, size) - p.Elements = tSlice - for i := 0; i < size; i++ { - _elem13 := &BlocksMetadata{} - if err := _elem13.Read(iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem13), err) - } - p.Elements = append(p.Elements, _elem13) - } - if err := iprot.ReadListEnd(); err != nil { - return thrift.PrependError("error reading list end: ", err) - } - return nil -} - -func (p *FetchBlocksMetadataRawResult_) ReadField2(iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(); err != nil { - return thrift.PrependError("error reading field 2: ", err) - } else { - p.NextPageToken = &v - } - return nil -} - -func (p *FetchBlocksMetadataRawResult_) Write(oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin("FetchBlocksMetadataRawResult"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) - } - if p != nil { - if err := p.writeField1(oprot); err != nil { - return err - } - if err := p.writeField2(oprot); err != nil { - return err - } - } - if err := oprot.WriteFieldStop(); err != nil { - return thrift.PrependError("write field stop error: ", err) - } - if err := oprot.WriteStructEnd(); err != nil { - return thrift.PrependError("write struct stop error: ", err) - } - return nil -} - -func (p *FetchBlocksMetadataRawResult_) writeField1(oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin("elements", thrift.LIST, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:elements: ", p), err) - } - if err := oprot.WriteListBegin(thrift.STRUCT, len(p.Elements)); err != nil { - return thrift.PrependError("error writing list begin: ", err) - } - for _, v := range p.Elements { - if err := v.Write(oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", v), err) - } - } - if err := oprot.WriteListEnd(); err != nil { - return thrift.PrependError("error writing list end: ", err) - } - if err := oprot.WriteFieldEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:elements: ", p), err) - } - return err -} - -func (p *FetchBlocksMetadataRawResult_) writeField2(oprot thrift.TProtocol) (err error) { - if p.IsSetNextPageToken() { - if err := oprot.WriteFieldBegin("nextPageToken", thrift.I64, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:nextPageToken: ", p), err) - } - if err := oprot.WriteI64(int64(*p.NextPageToken)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.nextPageToken (2) field write error: ", p), err) - } - if err := oprot.WriteFieldEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:nextPageToken: ", p), err) - } - } - return err -} - -func (p *FetchBlocksMetadataRawResult_) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("FetchBlocksMetadataRawResult_(%+v)", *p) -} - -// Attributes: -// - ID -// - Blocks -type BlocksMetadata struct { - ID []byte `thrift:"id,1,required" db:"id" json:"id"` - Blocks []*BlockMetadata `thrift:"blocks,2,required" db:"blocks" json:"blocks"` -} - -func NewBlocksMetadata() *BlocksMetadata { - return &BlocksMetadata{} -} - -func (p *BlocksMetadata) GetID() []byte { - return p.ID -} - -func (p *BlocksMetadata) GetBlocks() []*BlockMetadata { - return p.Blocks -} -func (p *BlocksMetadata) Read(iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetID bool = false - var issetBlocks bool = false - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin() - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { - break - } - switch fieldId { - case 1: - if err := p.ReadField1(iprot); err != nil { - return err - } - issetID = true - case 2: - if err := p.ReadField2(iprot); err != nil { - return err - } - issetBlocks = true - default: - if err := iprot.Skip(fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetID { - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field ID is not set")) - } - if !issetBlocks { - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Blocks is not set")) - } - return nil -} - -func (p *BlocksMetadata) ReadField1(iprot thrift.TProtocol) error { - if v, err := iprot.ReadBinary(); err != nil { - return thrift.PrependError("error reading field 1: ", err) - } else { - p.ID = v - } - return nil -} - -func (p *BlocksMetadata) ReadField2(iprot thrift.TProtocol) error { - _, size, err := iprot.ReadListBegin() - if err != nil { - return thrift.PrependError("error reading list begin: ", err) - } - tSlice := make([]*BlockMetadata, 0, size) - p.Blocks = tSlice - for i := 0; i < size; i++ { - _elem14 := &BlockMetadata{ - LastReadTimeType: 0, - } - if err := _elem14.Read(iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem14), err) - } - p.Blocks = append(p.Blocks, _elem14) - } - if err := iprot.ReadListEnd(); err != nil { - return thrift.PrependError("error reading list end: ", err) - } - return nil -} - -func (p *BlocksMetadata) Write(oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin("BlocksMetadata"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) - } - if p != nil { - if err := p.writeField1(oprot); err != nil { - return err - } - if err := p.writeField2(oprot); err != nil { - return err - } - } - if err := oprot.WriteFieldStop(); err != nil { - return thrift.PrependError("write field stop error: ", err) - } - if err := oprot.WriteStructEnd(); err != nil { - return thrift.PrependError("write struct stop error: ", err) - } - return nil -} - -func (p *BlocksMetadata) writeField1(oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin("id", thrift.STRING, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:id: ", p), err) - } - if err := oprot.WriteBinary(p.ID); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.id (1) field write error: ", p), err) - } - if err := oprot.WriteFieldEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:id: ", p), err) - } - return err -} - -func (p *BlocksMetadata) writeField2(oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin("blocks", thrift.LIST, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:blocks: ", p), err) - } - if err := oprot.WriteListBegin(thrift.STRUCT, len(p.Blocks)); err != nil { - return thrift.PrependError("error writing list begin: ", err) - } - for _, v := range p.Blocks { - if err := v.Write(oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", v), err) - } - } - if err := oprot.WriteListEnd(); err != nil { - return thrift.PrependError("error writing list end: ", err) - } - if err := oprot.WriteFieldEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:blocks: ", p), err) - } - return err -} - -func (p *BlocksMetadata) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("BlocksMetadata(%+v)", *p) -} - -// Attributes: -// - Err -// - Start -// - Size -// - Checksum -// - LastRead -// - LastReadTimeType -type BlockMetadata struct { - Err *Error `thrift:"err,1" db:"err" json:"err,omitempty"` - Start int64 `thrift:"start,2,required" db:"start" json:"start"` - Size *int64 `thrift:"size,3" db:"size" json:"size,omitempty"` - Checksum *int64 `thrift:"checksum,4" db:"checksum" json:"checksum,omitempty"` - LastRead *int64 `thrift:"lastRead,5" db:"lastRead" json:"lastRead,omitempty"` - LastReadTimeType TimeType `thrift:"lastReadTimeType,6" db:"lastReadTimeType" json:"lastReadTimeType,omitempty"` -} - -func NewBlockMetadata() *BlockMetadata { - return &BlockMetadata{ - LastReadTimeType: 0, - } -} - -var BlockMetadata_Err_DEFAULT *Error - -func (p *BlockMetadata) GetErr() *Error { - if !p.IsSetErr() { - return BlockMetadata_Err_DEFAULT - } - return p.Err -} - -func (p *BlockMetadata) GetStart() int64 { - return p.Start -} - -var BlockMetadata_Size_DEFAULT int64 - -func (p *BlockMetadata) GetSize() int64 { - if !p.IsSetSize() { - return BlockMetadata_Size_DEFAULT - } - return *p.Size -} - -var BlockMetadata_Checksum_DEFAULT int64 - -func (p *BlockMetadata) GetChecksum() int64 { - if !p.IsSetChecksum() { - return BlockMetadata_Checksum_DEFAULT - } - return *p.Checksum -} - -var BlockMetadata_LastRead_DEFAULT int64 - -func (p *BlockMetadata) GetLastRead() int64 { - if !p.IsSetLastRead() { - return BlockMetadata_LastRead_DEFAULT - } - return *p.LastRead -} - -var BlockMetadata_LastReadTimeType_DEFAULT TimeType = 0 - -func (p *BlockMetadata) GetLastReadTimeType() TimeType { - return p.LastReadTimeType -} -func (p *BlockMetadata) IsSetErr() bool { - return p.Err != nil -} - -func (p *BlockMetadata) IsSetSize() bool { - return p.Size != nil -} - -func (p *BlockMetadata) IsSetChecksum() bool { - return p.Checksum != nil -} - -func (p *BlockMetadata) IsSetLastRead() bool { - return p.LastRead != nil -} - -func (p *BlockMetadata) IsSetLastReadTimeType() bool { - return p.LastReadTimeType != BlockMetadata_LastReadTimeType_DEFAULT -} - -func (p *BlockMetadata) Read(iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetStart bool = false - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin() - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { - break - } - switch fieldId { - case 1: - if err := p.ReadField1(iprot); err != nil { - return err - } - case 2: - if err := p.ReadField2(iprot); err != nil { - return err - } - issetStart = true - case 3: - if err := p.ReadField3(iprot); err != nil { - return err - } - case 4: - if err := p.ReadField4(iprot); err != nil { - return err - } - case 5: - if err := p.ReadField5(iprot); err != nil { - return err - } - case 6: - if err := p.ReadField6(iprot); err != nil { - return err - } - default: - if err := iprot.Skip(fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetStart { - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Start is not set")) - } - return nil -} - -func (p *BlockMetadata) ReadField1(iprot thrift.TProtocol) error { - p.Err = &Error{ - Type: 0, - } - if err := p.Err.Read(iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.Err), err) - } - return nil -} - -func (p *BlockMetadata) ReadField2(iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(); err != nil { - return thrift.PrependError("error reading field 2: ", err) - } else { - p.Start = v - } - return nil -} - -func (p *BlockMetadata) ReadField3(iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(); err != nil { - return thrift.PrependError("error reading field 3: ", err) - } else { - p.Size = &v - } - return nil -} - -func (p *BlockMetadata) ReadField4(iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(); err != nil { - return thrift.PrependError("error reading field 4: ", err) - } else { - p.Checksum = &v - } - return nil -} - -func (p *BlockMetadata) ReadField5(iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(); err != nil { - return thrift.PrependError("error reading field 5: ", err) - } else { - p.LastRead = &v - } - return nil -} - -func (p *BlockMetadata) ReadField6(iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(); err != nil { - return thrift.PrependError("error reading field 6: ", err) - } else { - temp := TimeType(v) - p.LastReadTimeType = temp - } - return nil -} - -func (p *BlockMetadata) Write(oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin("BlockMetadata"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) - } - if p != nil { - if err := p.writeField1(oprot); err != nil { - return err - } - if err := p.writeField2(oprot); err != nil { - return err - } - if err := p.writeField3(oprot); err != nil { - return err - } - if err := p.writeField4(oprot); err != nil { - return err - } - if err := p.writeField5(oprot); err != nil { - return err - } - if err := p.writeField6(oprot); err != nil { - return err - } - } - if err := oprot.WriteFieldStop(); err != nil { - return thrift.PrependError("write field stop error: ", err) - } - if err := oprot.WriteStructEnd(); err != nil { - return thrift.PrependError("write struct stop error: ", err) - } - return nil -} - -func (p *BlockMetadata) writeField1(oprot thrift.TProtocol) (err error) { - if p.IsSetErr() { - if err := oprot.WriteFieldBegin("err", thrift.STRUCT, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:err: ", p), err) - } - if err := p.Err.Write(oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.Err), err) - } - if err := oprot.WriteFieldEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:err: ", p), err) - } - } - return err -} - -func (p *BlockMetadata) writeField2(oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin("start", thrift.I64, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:start: ", p), err) - } - if err := oprot.WriteI64(int64(p.Start)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.start (2) field write error: ", p), err) - } - if err := oprot.WriteFieldEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:start: ", p), err) - } - return err -} - -func (p *BlockMetadata) writeField3(oprot thrift.TProtocol) (err error) { - if p.IsSetSize() { - if err := oprot.WriteFieldBegin("size", thrift.I64, 3); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:size: ", p), err) - } - if err := oprot.WriteI64(int64(*p.Size)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.size (3) field write error: ", p), err) - } - if err := oprot.WriteFieldEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 3:size: ", p), err) - } - } - return err -} - -func (p *BlockMetadata) writeField4(oprot thrift.TProtocol) (err error) { - if p.IsSetChecksum() { - if err := oprot.WriteFieldBegin("checksum", thrift.I64, 4); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:checksum: ", p), err) - } - if err := oprot.WriteI64(int64(*p.Checksum)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.checksum (4) field write error: ", p), err) - } - if err := oprot.WriteFieldEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 4:checksum: ", p), err) - } - } - return err -} - -func (p *BlockMetadata) writeField5(oprot thrift.TProtocol) (err error) { - if p.IsSetLastRead() { - if err := oprot.WriteFieldBegin("lastRead", thrift.I64, 5); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:lastRead: ", p), err) - } - if err := oprot.WriteI64(int64(*p.LastRead)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.lastRead (5) field write error: ", p), err) - } - if err := oprot.WriteFieldEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 5:lastRead: ", p), err) - } - } - return err -} - -func (p *BlockMetadata) writeField6(oprot thrift.TProtocol) (err error) { - if p.IsSetLastReadTimeType() { - if err := oprot.WriteFieldBegin("lastReadTimeType", thrift.I32, 6); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 6:lastReadTimeType: ", p), err) - } - if err := oprot.WriteI32(int32(p.LastReadTimeType)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.lastReadTimeType (6) field write error: ", p), err) - } - if err := oprot.WriteFieldEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 6:lastReadTimeType: ", p), err) - } - } - return err -} - -func (p *BlockMetadata) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("BlockMetadata(%+v)", *p) -} - // Attributes: // - NameSpace // - Shard @@ -5911,13 +4814,13 @@ func (p *FetchBlocksMetadataRawV2Result_) ReadField1(iprot thrift.TProtocol) err tSlice := make([]*BlockMetadataV2, 0, size) p.Elements = tSlice for i := 0; i < size; i++ { - _elem15 := &BlockMetadataV2{ + _elem13 := &BlockMetadataV2{ LastReadTimeType: 0, } - if err := _elem15.Read(iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem15), err) + if err := _elem13.Read(iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem13), err) } - p.Elements = append(p.Elements, _elem15) + p.Elements = append(p.Elements, _elem13) } if err := iprot.ReadListEnd(); err != nil { return thrift.PrependError("error reading list end: ", err) @@ -6495,11 +5398,11 @@ func (p *WriteBatchRawRequest) ReadField2(iprot thrift.TProtocol) error { tSlice := make([]*WriteBatchRawRequestElement, 0, size) p.Elements = tSlice for i := 0; i < size; i++ { - _elem16 := &WriteBatchRawRequestElement{} - if err := _elem16.Read(iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem16), err) + _elem14 := &WriteBatchRawRequestElement{} + if err := _elem14.Read(iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem14), err) } - p.Elements = append(p.Elements, _elem16) + p.Elements = append(p.Elements, _elem14) } if err := iprot.ReadListEnd(); err != nil { return thrift.PrependError("error reading list end: ", err) @@ -6802,11 +5705,11 @@ func (p *WriteTaggedBatchRawRequest) ReadField2(iprot thrift.TProtocol) error { tSlice := make([]*WriteTaggedBatchRawRequestElement, 0, size) p.Elements = tSlice for i := 0; i < size; i++ { - _elem17 := &WriteTaggedBatchRawRequestElement{} - if err := _elem17.Read(iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem17), err) + _elem15 := &WriteTaggedBatchRawRequestElement{} + if err := _elem15.Read(iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem15), err) } - p.Elements = append(p.Elements, _elem17) + p.Elements = append(p.Elements, _elem15) } if err := iprot.ReadListEnd(); err != nil { return thrift.PrependError("error reading list end: ", err) @@ -9327,11 +8230,11 @@ func (p *QueryResult_) ReadField1(iprot thrift.TProtocol) error { tSlice := make([]*QueryResultElement, 0, size) p.Results = tSlice for i := 0; i < size; i++ { - _elem18 := &QueryResultElement{} - if err := _elem18.Read(iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem18), err) + _elem16 := &QueryResultElement{} + if err := _elem16.Read(iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem16), err) } - p.Results = append(p.Results, _elem18) + p.Results = append(p.Results, _elem16) } if err := iprot.ReadListEnd(); err != nil { return thrift.PrependError("error reading list end: ", err) @@ -9509,11 +8412,11 @@ func (p *QueryResultElement) ReadField2(iprot thrift.TProtocol) error { tSlice := make([]*Tag, 0, size) p.Tags = tSlice for i := 0; i < size; i++ { - _elem19 := &Tag{} - if err := _elem19.Read(iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem19), err) + _elem17 := &Tag{} + if err := _elem17.Read(iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem17), err) } - p.Tags = append(p.Tags, _elem19) + p.Tags = append(p.Tags, _elem17) } if err := iprot.ReadListEnd(); err != nil { return thrift.PrependError("error reading list end: ", err) @@ -9529,13 +8432,13 @@ func (p *QueryResultElement) ReadField3(iprot thrift.TProtocol) error { tSlice := make([]*Datapoint, 0, size) p.Datapoints = tSlice for i := 0; i < size; i++ { - _elem20 := &Datapoint{ + _elem18 := &Datapoint{ TimestampTimeType: 0, } - if err := _elem20.Read(iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem20), err) + if err := _elem18.Read(iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem18), err) } - p.Datapoints = append(p.Datapoints, _elem20) + p.Datapoints = append(p.Datapoints, _elem18) } if err := iprot.ReadListEnd(); err != nil { return thrift.PrependError("error reading list end: ", err) @@ -10074,11 +8977,11 @@ func (p *ConjunctionQuery) ReadField1(iprot thrift.TProtocol) error { tSlice := make([]*Query, 0, size) p.Queries = tSlice for i := 0; i < size; i++ { - _elem21 := &Query{} - if err := _elem21.Read(iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem21), err) + _elem19 := &Query{} + if err := _elem19.Read(iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem19), err) } - p.Queries = append(p.Queries, _elem21) + p.Queries = append(p.Queries, _elem19) } if err := iprot.ReadListEnd(); err != nil { return thrift.PrependError("error reading list end: ", err) @@ -10192,11 +9095,11 @@ func (p *DisjunctionQuery) ReadField1(iprot thrift.TProtocol) error { tSlice := make([]*Query, 0, size) p.Queries = tSlice for i := 0; i < size; i++ { - _elem22 := &Query{} - if err := _elem22.Read(iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem22), err) + _elem20 := &Query{} + if err := _elem20.Read(iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem20), err) } - p.Queries = append(p.Queries, _elem22) + p.Queries = append(p.Queries, _elem20) } if err := iprot.ReadListEnd(); err != nil { return thrift.PrependError("error reading list end: ", err) @@ -10557,9 +9460,6 @@ type Node interface { FetchBlocksRaw(req *FetchBlocksRawRequest) (r *FetchBlocksRawResult_, err error) // Parameters: // - Req - FetchBlocksMetadataRaw(req *FetchBlocksMetadataRawRequest) (r *FetchBlocksMetadataRawResult_, err error) - // Parameters: - // - Req FetchBlocksMetadataRawV2(req *FetchBlocksMetadataRawV2Request) (r *FetchBlocksMetadataRawV2Result_, err error) // Parameters: // - Req @@ -10667,16 +9567,16 @@ func (p *NodeClient) recvQuery() (value *QueryResult_, err error) { return } if mTypeId == thrift.EXCEPTION { - error23 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error24 error - error24, err = error23.Read(iprot) + error21 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error22 error + error22, err = error21.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error24 + err = error22 return } if mTypeId != thrift.REPLY { @@ -10748,16 +9648,16 @@ func (p *NodeClient) recvFetch() (value *FetchResult_, err error) { return } if mTypeId == thrift.EXCEPTION { - error25 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error26 error - error26, err = error25.Read(iprot) + error23 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error24 error + error24, err = error23.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error26 + err = error24 return } if mTypeId != thrift.REPLY { @@ -10829,16 +9729,16 @@ func (p *NodeClient) recvFetchTagged() (value *FetchTaggedResult_, err error) { return } if mTypeId == thrift.EXCEPTION { - error27 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error28 error - error28, err = error27.Read(iprot) + error25 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error26 error + error26, err = error25.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error28 + err = error26 return } if mTypeId != thrift.REPLY { @@ -10910,103 +9810,23 @@ func (p *NodeClient) recvWrite() (err error) { return } if mTypeId == thrift.EXCEPTION { - error29 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error30 error - error30, err = error29.Read(iprot) - if err != nil { - return - } - if err = iprot.ReadMessageEnd(); err != nil { - return - } - err = error30 - return - } - if mTypeId != thrift.REPLY { - err = thrift.NewTApplicationException(thrift.INVALID_MESSAGE_TYPE_EXCEPTION, "write failed: invalid message type") - return - } - result := NodeWriteResult{} - if err = result.Read(iprot); err != nil { - return - } - if err = iprot.ReadMessageEnd(); err != nil { - return - } - if result.Err != nil { - err = result.Err - return - } - return -} - -// Parameters: -// - Req -func (p *NodeClient) WriteTagged(req *WriteTaggedRequest) (err error) { - if err = p.sendWriteTagged(req); err != nil { - return - } - return p.recvWriteTagged() -} - -func (p *NodeClient) sendWriteTagged(req *WriteTaggedRequest) (err error) { - oprot := p.OutputProtocol - if oprot == nil { - oprot = p.ProtocolFactory.GetProtocol(p.Transport) - p.OutputProtocol = oprot - } - p.SeqId++ - if err = oprot.WriteMessageBegin("writeTagged", thrift.CALL, p.SeqId); err != nil { - return - } - args := NodeWriteTaggedArgs{ - Req: req, - } - if err = args.Write(oprot); err != nil { - return - } - if err = oprot.WriteMessageEnd(); err != nil { - return - } - return oprot.Flush() -} - -func (p *NodeClient) recvWriteTagged() (err error) { - iprot := p.InputProtocol - if iprot == nil { - iprot = p.ProtocolFactory.GetProtocol(p.Transport) - p.InputProtocol = iprot - } - method, mTypeId, seqId, err := iprot.ReadMessageBegin() - if err != nil { - return - } - if method != "writeTagged" { - err = thrift.NewTApplicationException(thrift.WRONG_METHOD_NAME, "writeTagged failed: wrong method name") - return - } - if p.SeqId != seqId { - err = thrift.NewTApplicationException(thrift.BAD_SEQUENCE_ID, "writeTagged failed: out of sequence response") - return - } - if mTypeId == thrift.EXCEPTION { - error31 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error32 error - error32, err = error31.Read(iprot) + error27 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error28 error + error28, err = error27.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error32 + err = error28 return } if mTypeId != thrift.REPLY { - err = thrift.NewTApplicationException(thrift.INVALID_MESSAGE_TYPE_EXCEPTION, "writeTagged failed: invalid message type") + err = thrift.NewTApplicationException(thrift.INVALID_MESSAGE_TYPE_EXCEPTION, "write failed: invalid message type") return } - result := NodeWriteTaggedResult{} + result := NodeWriteResult{} if err = result.Read(iprot); err != nil { return } @@ -11022,24 +9842,24 @@ func (p *NodeClient) recvWriteTagged() (err error) { // Parameters: // - Req -func (p *NodeClient) FetchBatchRaw(req *FetchBatchRawRequest) (r *FetchBatchRawResult_, err error) { - if err = p.sendFetchBatchRaw(req); err != nil { +func (p *NodeClient) WriteTagged(req *WriteTaggedRequest) (err error) { + if err = p.sendWriteTagged(req); err != nil { return } - return p.recvFetchBatchRaw() + return p.recvWriteTagged() } -func (p *NodeClient) sendFetchBatchRaw(req *FetchBatchRawRequest) (err error) { +func (p *NodeClient) sendWriteTagged(req *WriteTaggedRequest) (err error) { oprot := p.OutputProtocol if oprot == nil { oprot = p.ProtocolFactory.GetProtocol(p.Transport) p.OutputProtocol = oprot } p.SeqId++ - if err = oprot.WriteMessageBegin("fetchBatchRaw", thrift.CALL, p.SeqId); err != nil { + if err = oprot.WriteMessageBegin("writeTagged", thrift.CALL, p.SeqId); err != nil { return } - args := NodeFetchBatchRawArgs{ + args := NodeWriteTaggedArgs{ Req: req, } if err = args.Write(oprot); err != nil { @@ -11051,7 +9871,7 @@ func (p *NodeClient) sendFetchBatchRaw(req *FetchBatchRawRequest) (err error) { return oprot.Flush() } -func (p *NodeClient) recvFetchBatchRaw() (value *FetchBatchRawResult_, err error) { +func (p *NodeClient) recvWriteTagged() (err error) { iprot := p.InputProtocol if iprot == nil { iprot = p.ProtocolFactory.GetProtocol(p.Transport) @@ -11061,32 +9881,32 @@ func (p *NodeClient) recvFetchBatchRaw() (value *FetchBatchRawResult_, err error if err != nil { return } - if method != "fetchBatchRaw" { - err = thrift.NewTApplicationException(thrift.WRONG_METHOD_NAME, "fetchBatchRaw failed: wrong method name") + if method != "writeTagged" { + err = thrift.NewTApplicationException(thrift.WRONG_METHOD_NAME, "writeTagged failed: wrong method name") return } if p.SeqId != seqId { - err = thrift.NewTApplicationException(thrift.BAD_SEQUENCE_ID, "fetchBatchRaw failed: out of sequence response") + err = thrift.NewTApplicationException(thrift.BAD_SEQUENCE_ID, "writeTagged failed: out of sequence response") return } if mTypeId == thrift.EXCEPTION { - error33 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error34 error - error34, err = error33.Read(iprot) + error29 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error30 error + error30, err = error29.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error34 + err = error30 return } if mTypeId != thrift.REPLY { - err = thrift.NewTApplicationException(thrift.INVALID_MESSAGE_TYPE_EXCEPTION, "fetchBatchRaw failed: invalid message type") + err = thrift.NewTApplicationException(thrift.INVALID_MESSAGE_TYPE_EXCEPTION, "writeTagged failed: invalid message type") return } - result := NodeFetchBatchRawResult{} + result := NodeWriteTaggedResult{} if err = result.Read(iprot); err != nil { return } @@ -11097,30 +9917,29 @@ func (p *NodeClient) recvFetchBatchRaw() (value *FetchBatchRawResult_, err error err = result.Err return } - value = result.GetSuccess() return } // Parameters: // - Req -func (p *NodeClient) FetchBlocksRaw(req *FetchBlocksRawRequest) (r *FetchBlocksRawResult_, err error) { - if err = p.sendFetchBlocksRaw(req); err != nil { +func (p *NodeClient) FetchBatchRaw(req *FetchBatchRawRequest) (r *FetchBatchRawResult_, err error) { + if err = p.sendFetchBatchRaw(req); err != nil { return } - return p.recvFetchBlocksRaw() + return p.recvFetchBatchRaw() } -func (p *NodeClient) sendFetchBlocksRaw(req *FetchBlocksRawRequest) (err error) { +func (p *NodeClient) sendFetchBatchRaw(req *FetchBatchRawRequest) (err error) { oprot := p.OutputProtocol if oprot == nil { oprot = p.ProtocolFactory.GetProtocol(p.Transport) p.OutputProtocol = oprot } p.SeqId++ - if err = oprot.WriteMessageBegin("fetchBlocksRaw", thrift.CALL, p.SeqId); err != nil { + if err = oprot.WriteMessageBegin("fetchBatchRaw", thrift.CALL, p.SeqId); err != nil { return } - args := NodeFetchBlocksRawArgs{ + args := NodeFetchBatchRawArgs{ Req: req, } if err = args.Write(oprot); err != nil { @@ -11132,7 +9951,7 @@ func (p *NodeClient) sendFetchBlocksRaw(req *FetchBlocksRawRequest) (err error) return oprot.Flush() } -func (p *NodeClient) recvFetchBlocksRaw() (value *FetchBlocksRawResult_, err error) { +func (p *NodeClient) recvFetchBatchRaw() (value *FetchBatchRawResult_, err error) { iprot := p.InputProtocol if iprot == nil { iprot = p.ProtocolFactory.GetProtocol(p.Transport) @@ -11142,32 +9961,32 @@ func (p *NodeClient) recvFetchBlocksRaw() (value *FetchBlocksRawResult_, err err if err != nil { return } - if method != "fetchBlocksRaw" { - err = thrift.NewTApplicationException(thrift.WRONG_METHOD_NAME, "fetchBlocksRaw failed: wrong method name") + if method != "fetchBatchRaw" { + err = thrift.NewTApplicationException(thrift.WRONG_METHOD_NAME, "fetchBatchRaw failed: wrong method name") return } if p.SeqId != seqId { - err = thrift.NewTApplicationException(thrift.BAD_SEQUENCE_ID, "fetchBlocksRaw failed: out of sequence response") + err = thrift.NewTApplicationException(thrift.BAD_SEQUENCE_ID, "fetchBatchRaw failed: out of sequence response") return } if mTypeId == thrift.EXCEPTION { - error35 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error36 error - error36, err = error35.Read(iprot) + error31 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error32 error + error32, err = error31.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error36 + err = error32 return } if mTypeId != thrift.REPLY { - err = thrift.NewTApplicationException(thrift.INVALID_MESSAGE_TYPE_EXCEPTION, "fetchBlocksRaw failed: invalid message type") + err = thrift.NewTApplicationException(thrift.INVALID_MESSAGE_TYPE_EXCEPTION, "fetchBatchRaw failed: invalid message type") return } - result := NodeFetchBlocksRawResult{} + result := NodeFetchBatchRawResult{} if err = result.Read(iprot); err != nil { return } @@ -11184,24 +10003,24 @@ func (p *NodeClient) recvFetchBlocksRaw() (value *FetchBlocksRawResult_, err err // Parameters: // - Req -func (p *NodeClient) FetchBlocksMetadataRaw(req *FetchBlocksMetadataRawRequest) (r *FetchBlocksMetadataRawResult_, err error) { - if err = p.sendFetchBlocksMetadataRaw(req); err != nil { +func (p *NodeClient) FetchBlocksRaw(req *FetchBlocksRawRequest) (r *FetchBlocksRawResult_, err error) { + if err = p.sendFetchBlocksRaw(req); err != nil { return } - return p.recvFetchBlocksMetadataRaw() + return p.recvFetchBlocksRaw() } -func (p *NodeClient) sendFetchBlocksMetadataRaw(req *FetchBlocksMetadataRawRequest) (err error) { +func (p *NodeClient) sendFetchBlocksRaw(req *FetchBlocksRawRequest) (err error) { oprot := p.OutputProtocol if oprot == nil { oprot = p.ProtocolFactory.GetProtocol(p.Transport) p.OutputProtocol = oprot } p.SeqId++ - if err = oprot.WriteMessageBegin("fetchBlocksMetadataRaw", thrift.CALL, p.SeqId); err != nil { + if err = oprot.WriteMessageBegin("fetchBlocksRaw", thrift.CALL, p.SeqId); err != nil { return } - args := NodeFetchBlocksMetadataRawArgs{ + args := NodeFetchBlocksRawArgs{ Req: req, } if err = args.Write(oprot); err != nil { @@ -11213,7 +10032,7 @@ func (p *NodeClient) sendFetchBlocksMetadataRaw(req *FetchBlocksMetadataRawReque return oprot.Flush() } -func (p *NodeClient) recvFetchBlocksMetadataRaw() (value *FetchBlocksMetadataRawResult_, err error) { +func (p *NodeClient) recvFetchBlocksRaw() (value *FetchBlocksRawResult_, err error) { iprot := p.InputProtocol if iprot == nil { iprot = p.ProtocolFactory.GetProtocol(p.Transport) @@ -11223,32 +10042,32 @@ func (p *NodeClient) recvFetchBlocksMetadataRaw() (value *FetchBlocksMetadataRaw if err != nil { return } - if method != "fetchBlocksMetadataRaw" { - err = thrift.NewTApplicationException(thrift.WRONG_METHOD_NAME, "fetchBlocksMetadataRaw failed: wrong method name") + if method != "fetchBlocksRaw" { + err = thrift.NewTApplicationException(thrift.WRONG_METHOD_NAME, "fetchBlocksRaw failed: wrong method name") return } if p.SeqId != seqId { - err = thrift.NewTApplicationException(thrift.BAD_SEQUENCE_ID, "fetchBlocksMetadataRaw failed: out of sequence response") + err = thrift.NewTApplicationException(thrift.BAD_SEQUENCE_ID, "fetchBlocksRaw failed: out of sequence response") return } if mTypeId == thrift.EXCEPTION { - error37 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error38 error - error38, err = error37.Read(iprot) + error33 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error34 error + error34, err = error33.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error38 + err = error34 return } if mTypeId != thrift.REPLY { - err = thrift.NewTApplicationException(thrift.INVALID_MESSAGE_TYPE_EXCEPTION, "fetchBlocksMetadataRaw failed: invalid message type") + err = thrift.NewTApplicationException(thrift.INVALID_MESSAGE_TYPE_EXCEPTION, "fetchBlocksRaw failed: invalid message type") return } - result := NodeFetchBlocksMetadataRawResult{} + result := NodeFetchBlocksRawResult{} if err = result.Read(iprot); err != nil { return } @@ -11313,16 +10132,16 @@ func (p *NodeClient) recvFetchBlocksMetadataRawV2() (value *FetchBlocksMetadataR return } if mTypeId == thrift.EXCEPTION { - error39 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error40 error - error40, err = error39.Read(iprot) + error35 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error36 error + error36, err = error35.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error40 + err = error36 return } if mTypeId != thrift.REPLY { @@ -11394,16 +10213,16 @@ func (p *NodeClient) recvWriteBatchRaw() (err error) { return } if mTypeId == thrift.EXCEPTION { - error41 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error42 error - error42, err = error41.Read(iprot) + error37 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error38 error + error38, err = error37.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error42 + err = error38 return } if mTypeId != thrift.REPLY { @@ -11474,16 +10293,16 @@ func (p *NodeClient) recvWriteTaggedBatchRaw() (err error) { return } if mTypeId == thrift.EXCEPTION { - error43 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error44 error - error44, err = error43.Read(iprot) + error39 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error40 error + error40, err = error39.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error44 + err = error40 return } if mTypeId != thrift.REPLY { @@ -11550,16 +10369,16 @@ func (p *NodeClient) recvRepair() (err error) { return } if mTypeId == thrift.EXCEPTION { - error45 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error46 error - error46, err = error45.Read(iprot) + error41 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error42 error + error42, err = error41.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error46 + err = error42 return } if mTypeId != thrift.REPLY { @@ -11630,16 +10449,16 @@ func (p *NodeClient) recvTruncate() (value *TruncateResult_, err error) { return } if mTypeId == thrift.EXCEPTION { - error47 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error48 error - error48, err = error47.Read(iprot) + error43 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error44 error + error44, err = error43.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error48 + err = error44 return } if mTypeId != thrift.REPLY { @@ -11707,16 +10526,16 @@ func (p *NodeClient) recvHealth() (value *NodeHealthResult_, err error) { return } if mTypeId == thrift.EXCEPTION { - error49 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error50 error - error50, err = error49.Read(iprot) + error45 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error46 error + error46, err = error45.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error50 + err = error46 return } if mTypeId != thrift.REPLY { @@ -11784,16 +10603,16 @@ func (p *NodeClient) recvBootstrapped() (value *NodeBootstrappedResult_, err err return } if mTypeId == thrift.EXCEPTION { - error51 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error52 error - error52, err = error51.Read(iprot) + error47 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error48 error + error48, err = error47.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error52 + err = error48 return } if mTypeId != thrift.REPLY { @@ -11861,16 +10680,16 @@ func (p *NodeClient) recvGetPersistRateLimit() (value *NodePersistRateLimitResul return } if mTypeId == thrift.EXCEPTION { - error53 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error54 error - error54, err = error53.Read(iprot) + error49 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error50 error + error50, err = error49.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error54 + err = error50 return } if mTypeId != thrift.REPLY { @@ -11942,16 +10761,16 @@ func (p *NodeClient) recvSetPersistRateLimit() (value *NodePersistRateLimitResul return } if mTypeId == thrift.EXCEPTION { - error55 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error56 error - error56, err = error55.Read(iprot) + error51 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error52 error + error52, err = error51.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error56 + err = error52 return } if mTypeId != thrift.REPLY { @@ -12019,16 +10838,16 @@ func (p *NodeClient) recvGetWriteNewSeriesAsync() (value *NodeWriteNewSeriesAsyn return } if mTypeId == thrift.EXCEPTION { - error57 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error58 error - error58, err = error57.Read(iprot) + error53 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error54 error + error54, err = error53.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error58 + err = error54 return } if mTypeId != thrift.REPLY { @@ -12100,16 +10919,16 @@ func (p *NodeClient) recvSetWriteNewSeriesAsync() (value *NodeWriteNewSeriesAsyn return } if mTypeId == thrift.EXCEPTION { - error59 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error60 error - error60, err = error59.Read(iprot) + error55 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error56 error + error56, err = error55.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error60 + err = error56 return } if mTypeId != thrift.REPLY { @@ -12177,16 +10996,16 @@ func (p *NodeClient) recvGetWriteNewSeriesBackoffDuration() (value *NodeWriteNew return } if mTypeId == thrift.EXCEPTION { - error61 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error62 error - error62, err = error61.Read(iprot) + error57 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error58 error + error58, err = error57.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error62 + err = error58 return } if mTypeId != thrift.REPLY { @@ -12258,16 +11077,16 @@ func (p *NodeClient) recvSetWriteNewSeriesBackoffDuration() (value *NodeWriteNew return } if mTypeId == thrift.EXCEPTION { - error63 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error64 error - error64, err = error63.Read(iprot) + error59 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error60 error + error60, err = error59.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error64 + err = error60 return } if mTypeId != thrift.REPLY { @@ -12335,16 +11154,16 @@ func (p *NodeClient) recvGetWriteNewSeriesLimitPerShardPerSecond() (value *NodeW return } if mTypeId == thrift.EXCEPTION { - error65 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error66 error - error66, err = error65.Read(iprot) + error61 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error62 error + error62, err = error61.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error66 + err = error62 return } if mTypeId != thrift.REPLY { @@ -12416,16 +11235,16 @@ func (p *NodeClient) recvSetWriteNewSeriesLimitPerShardPerSecond() (value *NodeW return } if mTypeId == thrift.EXCEPTION { - error67 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error68 error - error68, err = error67.Read(iprot) + error63 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error64 error + error64, err = error63.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error68 + err = error64 return } if mTypeId != thrift.REPLY { @@ -12467,31 +11286,30 @@ func (p *NodeProcessor) ProcessorMap() map[string]thrift.TProcessorFunction { func NewNodeProcessor(handler Node) *NodeProcessor { - self69 := &NodeProcessor{handler: handler, processorMap: make(map[string]thrift.TProcessorFunction)} - self69.processorMap["query"] = &nodeProcessorQuery{handler: handler} - self69.processorMap["fetch"] = &nodeProcessorFetch{handler: handler} - self69.processorMap["fetchTagged"] = &nodeProcessorFetchTagged{handler: handler} - self69.processorMap["write"] = &nodeProcessorWrite{handler: handler} - self69.processorMap["writeTagged"] = &nodeProcessorWriteTagged{handler: handler} - self69.processorMap["fetchBatchRaw"] = &nodeProcessorFetchBatchRaw{handler: handler} - self69.processorMap["fetchBlocksRaw"] = &nodeProcessorFetchBlocksRaw{handler: handler} - self69.processorMap["fetchBlocksMetadataRaw"] = &nodeProcessorFetchBlocksMetadataRaw{handler: handler} - self69.processorMap["fetchBlocksMetadataRawV2"] = &nodeProcessorFetchBlocksMetadataRawV2{handler: handler} - self69.processorMap["writeBatchRaw"] = &nodeProcessorWriteBatchRaw{handler: handler} - self69.processorMap["writeTaggedBatchRaw"] = &nodeProcessorWriteTaggedBatchRaw{handler: handler} - self69.processorMap["repair"] = &nodeProcessorRepair{handler: handler} - self69.processorMap["truncate"] = &nodeProcessorTruncate{handler: handler} - self69.processorMap["health"] = &nodeProcessorHealth{handler: handler} - self69.processorMap["bootstrapped"] = &nodeProcessorBootstrapped{handler: handler} - self69.processorMap["getPersistRateLimit"] = &nodeProcessorGetPersistRateLimit{handler: handler} - self69.processorMap["setPersistRateLimit"] = &nodeProcessorSetPersistRateLimit{handler: handler} - self69.processorMap["getWriteNewSeriesAsync"] = &nodeProcessorGetWriteNewSeriesAsync{handler: handler} - self69.processorMap["setWriteNewSeriesAsync"] = &nodeProcessorSetWriteNewSeriesAsync{handler: handler} - self69.processorMap["getWriteNewSeriesBackoffDuration"] = &nodeProcessorGetWriteNewSeriesBackoffDuration{handler: handler} - self69.processorMap["setWriteNewSeriesBackoffDuration"] = &nodeProcessorSetWriteNewSeriesBackoffDuration{handler: handler} - self69.processorMap["getWriteNewSeriesLimitPerShardPerSecond"] = &nodeProcessorGetWriteNewSeriesLimitPerShardPerSecond{handler: handler} - self69.processorMap["setWriteNewSeriesLimitPerShardPerSecond"] = &nodeProcessorSetWriteNewSeriesLimitPerShardPerSecond{handler: handler} - return self69 + self65 := &NodeProcessor{handler: handler, processorMap: make(map[string]thrift.TProcessorFunction)} + self65.processorMap["query"] = &nodeProcessorQuery{handler: handler} + self65.processorMap["fetch"] = &nodeProcessorFetch{handler: handler} + self65.processorMap["fetchTagged"] = &nodeProcessorFetchTagged{handler: handler} + self65.processorMap["write"] = &nodeProcessorWrite{handler: handler} + self65.processorMap["writeTagged"] = &nodeProcessorWriteTagged{handler: handler} + self65.processorMap["fetchBatchRaw"] = &nodeProcessorFetchBatchRaw{handler: handler} + self65.processorMap["fetchBlocksRaw"] = &nodeProcessorFetchBlocksRaw{handler: handler} + self65.processorMap["fetchBlocksMetadataRawV2"] = &nodeProcessorFetchBlocksMetadataRawV2{handler: handler} + self65.processorMap["writeBatchRaw"] = &nodeProcessorWriteBatchRaw{handler: handler} + self65.processorMap["writeTaggedBatchRaw"] = &nodeProcessorWriteTaggedBatchRaw{handler: handler} + self65.processorMap["repair"] = &nodeProcessorRepair{handler: handler} + self65.processorMap["truncate"] = &nodeProcessorTruncate{handler: handler} + self65.processorMap["health"] = &nodeProcessorHealth{handler: handler} + self65.processorMap["bootstrapped"] = &nodeProcessorBootstrapped{handler: handler} + self65.processorMap["getPersistRateLimit"] = &nodeProcessorGetPersistRateLimit{handler: handler} + self65.processorMap["setPersistRateLimit"] = &nodeProcessorSetPersistRateLimit{handler: handler} + self65.processorMap["getWriteNewSeriesAsync"] = &nodeProcessorGetWriteNewSeriesAsync{handler: handler} + self65.processorMap["setWriteNewSeriesAsync"] = &nodeProcessorSetWriteNewSeriesAsync{handler: handler} + self65.processorMap["getWriteNewSeriesBackoffDuration"] = &nodeProcessorGetWriteNewSeriesBackoffDuration{handler: handler} + self65.processorMap["setWriteNewSeriesBackoffDuration"] = &nodeProcessorSetWriteNewSeriesBackoffDuration{handler: handler} + self65.processorMap["getWriteNewSeriesLimitPerShardPerSecond"] = &nodeProcessorGetWriteNewSeriesLimitPerShardPerSecond{handler: handler} + self65.processorMap["setWriteNewSeriesLimitPerShardPerSecond"] = &nodeProcessorSetWriteNewSeriesLimitPerShardPerSecond{handler: handler} + return self65 } func (p *NodeProcessor) Process(iprot, oprot thrift.TProtocol) (success bool, err thrift.TException) { @@ -12504,12 +11322,12 @@ func (p *NodeProcessor) Process(iprot, oprot thrift.TProtocol) (success bool, er } iprot.Skip(thrift.STRUCT) iprot.ReadMessageEnd() - x70 := thrift.NewTApplicationException(thrift.UNKNOWN_METHOD, "Unknown function "+name) + x66 := thrift.NewTApplicationException(thrift.UNKNOWN_METHOD, "Unknown function "+name) oprot.WriteMessageBegin(name, thrift.EXCEPTION, seqId) - x70.Write(oprot) + x66.Write(oprot) oprot.WriteMessageEnd() oprot.Flush() - return false, x70 + return false, x66 } @@ -12776,65 +11594,12 @@ type nodeProcessorFetchBatchRaw struct { handler Node } -func (p *nodeProcessorFetchBatchRaw) Process(seqId int32, iprot, oprot thrift.TProtocol) (success bool, err thrift.TException) { - args := NodeFetchBatchRawArgs{} - if err = args.Read(iprot); err != nil { - iprot.ReadMessageEnd() - x := thrift.NewTApplicationException(thrift.PROTOCOL_ERROR, err.Error()) - oprot.WriteMessageBegin("fetchBatchRaw", thrift.EXCEPTION, seqId) - x.Write(oprot) - oprot.WriteMessageEnd() - oprot.Flush() - return false, err - } - - iprot.ReadMessageEnd() - result := NodeFetchBatchRawResult{} - var retval *FetchBatchRawResult_ - var err2 error - if retval, err2 = p.handler.FetchBatchRaw(args.Req); err2 != nil { - switch v := err2.(type) { - case *Error: - result.Err = v - default: - x := thrift.NewTApplicationException(thrift.INTERNAL_ERROR, "Internal error processing fetchBatchRaw: "+err2.Error()) - oprot.WriteMessageBegin("fetchBatchRaw", thrift.EXCEPTION, seqId) - x.Write(oprot) - oprot.WriteMessageEnd() - oprot.Flush() - return true, err2 - } - } else { - result.Success = retval - } - if err2 = oprot.WriteMessageBegin("fetchBatchRaw", thrift.REPLY, seqId); err2 != nil { - err = err2 - } - if err2 = result.Write(oprot); err == nil && err2 != nil { - err = err2 - } - if err2 = oprot.WriteMessageEnd(); err == nil && err2 != nil { - err = err2 - } - if err2 = oprot.Flush(); err == nil && err2 != nil { - err = err2 - } - if err != nil { - return - } - return true, err -} - -type nodeProcessorFetchBlocksRaw struct { - handler Node -} - -func (p *nodeProcessorFetchBlocksRaw) Process(seqId int32, iprot, oprot thrift.TProtocol) (success bool, err thrift.TException) { - args := NodeFetchBlocksRawArgs{} +func (p *nodeProcessorFetchBatchRaw) Process(seqId int32, iprot, oprot thrift.TProtocol) (success bool, err thrift.TException) { + args := NodeFetchBatchRawArgs{} if err = args.Read(iprot); err != nil { iprot.ReadMessageEnd() x := thrift.NewTApplicationException(thrift.PROTOCOL_ERROR, err.Error()) - oprot.WriteMessageBegin("fetchBlocksRaw", thrift.EXCEPTION, seqId) + oprot.WriteMessageBegin("fetchBatchRaw", thrift.EXCEPTION, seqId) x.Write(oprot) oprot.WriteMessageEnd() oprot.Flush() @@ -12842,16 +11607,16 @@ func (p *nodeProcessorFetchBlocksRaw) Process(seqId int32, iprot, oprot thrift.T } iprot.ReadMessageEnd() - result := NodeFetchBlocksRawResult{} - var retval *FetchBlocksRawResult_ + result := NodeFetchBatchRawResult{} + var retval *FetchBatchRawResult_ var err2 error - if retval, err2 = p.handler.FetchBlocksRaw(args.Req); err2 != nil { + if retval, err2 = p.handler.FetchBatchRaw(args.Req); err2 != nil { switch v := err2.(type) { case *Error: result.Err = v default: - x := thrift.NewTApplicationException(thrift.INTERNAL_ERROR, "Internal error processing fetchBlocksRaw: "+err2.Error()) - oprot.WriteMessageBegin("fetchBlocksRaw", thrift.EXCEPTION, seqId) + x := thrift.NewTApplicationException(thrift.INTERNAL_ERROR, "Internal error processing fetchBatchRaw: "+err2.Error()) + oprot.WriteMessageBegin("fetchBatchRaw", thrift.EXCEPTION, seqId) x.Write(oprot) oprot.WriteMessageEnd() oprot.Flush() @@ -12860,7 +11625,7 @@ func (p *nodeProcessorFetchBlocksRaw) Process(seqId int32, iprot, oprot thrift.T } else { result.Success = retval } - if err2 = oprot.WriteMessageBegin("fetchBlocksRaw", thrift.REPLY, seqId); err2 != nil { + if err2 = oprot.WriteMessageBegin("fetchBatchRaw", thrift.REPLY, seqId); err2 != nil { err = err2 } if err2 = result.Write(oprot); err == nil && err2 != nil { @@ -12878,16 +11643,16 @@ func (p *nodeProcessorFetchBlocksRaw) Process(seqId int32, iprot, oprot thrift.T return true, err } -type nodeProcessorFetchBlocksMetadataRaw struct { +type nodeProcessorFetchBlocksRaw struct { handler Node } -func (p *nodeProcessorFetchBlocksMetadataRaw) Process(seqId int32, iprot, oprot thrift.TProtocol) (success bool, err thrift.TException) { - args := NodeFetchBlocksMetadataRawArgs{} +func (p *nodeProcessorFetchBlocksRaw) Process(seqId int32, iprot, oprot thrift.TProtocol) (success bool, err thrift.TException) { + args := NodeFetchBlocksRawArgs{} if err = args.Read(iprot); err != nil { iprot.ReadMessageEnd() x := thrift.NewTApplicationException(thrift.PROTOCOL_ERROR, err.Error()) - oprot.WriteMessageBegin("fetchBlocksMetadataRaw", thrift.EXCEPTION, seqId) + oprot.WriteMessageBegin("fetchBlocksRaw", thrift.EXCEPTION, seqId) x.Write(oprot) oprot.WriteMessageEnd() oprot.Flush() @@ -12895,16 +11660,16 @@ func (p *nodeProcessorFetchBlocksMetadataRaw) Process(seqId int32, iprot, oprot } iprot.ReadMessageEnd() - result := NodeFetchBlocksMetadataRawResult{} - var retval *FetchBlocksMetadataRawResult_ + result := NodeFetchBlocksRawResult{} + var retval *FetchBlocksRawResult_ var err2 error - if retval, err2 = p.handler.FetchBlocksMetadataRaw(args.Req); err2 != nil { + if retval, err2 = p.handler.FetchBlocksRaw(args.Req); err2 != nil { switch v := err2.(type) { case *Error: result.Err = v default: - x := thrift.NewTApplicationException(thrift.INTERNAL_ERROR, "Internal error processing fetchBlocksMetadataRaw: "+err2.Error()) - oprot.WriteMessageBegin("fetchBlocksMetadataRaw", thrift.EXCEPTION, seqId) + x := thrift.NewTApplicationException(thrift.INTERNAL_ERROR, "Internal error processing fetchBlocksRaw: "+err2.Error()) + oprot.WriteMessageBegin("fetchBlocksRaw", thrift.EXCEPTION, seqId) x.Write(oprot) oprot.WriteMessageEnd() oprot.Flush() @@ -12913,7 +11678,7 @@ func (p *nodeProcessorFetchBlocksMetadataRaw) Process(seqId int32, iprot, oprot } else { result.Success = retval } - if err2 = oprot.WriteMessageBegin("fetchBlocksMetadataRaw", thrift.REPLY, seqId); err2 != nil { + if err2 = oprot.WriteMessageBegin("fetchBlocksRaw", thrift.REPLY, seqId); err2 != nil { err = err2 } if err2 = result.Write(oprot); err == nil && err2 != nil { @@ -15398,257 +14163,6 @@ func (p *NodeFetchBlocksRawResult) String() string { return fmt.Sprintf("NodeFetchBlocksRawResult(%+v)", *p) } -// Attributes: -// - Req -type NodeFetchBlocksMetadataRawArgs struct { - Req *FetchBlocksMetadataRawRequest `thrift:"req,1" db:"req" json:"req"` -} - -func NewNodeFetchBlocksMetadataRawArgs() *NodeFetchBlocksMetadataRawArgs { - return &NodeFetchBlocksMetadataRawArgs{} -} - -var NodeFetchBlocksMetadataRawArgs_Req_DEFAULT *FetchBlocksMetadataRawRequest - -func (p *NodeFetchBlocksMetadataRawArgs) GetReq() *FetchBlocksMetadataRawRequest { - if !p.IsSetReq() { - return NodeFetchBlocksMetadataRawArgs_Req_DEFAULT - } - return p.Req -} -func (p *NodeFetchBlocksMetadataRawArgs) IsSetReq() bool { - return p.Req != nil -} - -func (p *NodeFetchBlocksMetadataRawArgs) Read(iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin() - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { - break - } - switch fieldId { - case 1: - if err := p.ReadField1(iprot); err != nil { - return err - } - default: - if err := iprot.Skip(fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil -} - -func (p *NodeFetchBlocksMetadataRawArgs) ReadField1(iprot thrift.TProtocol) error { - p.Req = &FetchBlocksMetadataRawRequest{} - if err := p.Req.Read(iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.Req), err) - } - return nil -} - -func (p *NodeFetchBlocksMetadataRawArgs) Write(oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin("fetchBlocksMetadataRaw_args"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) - } - if p != nil { - if err := p.writeField1(oprot); err != nil { - return err - } - } - if err := oprot.WriteFieldStop(); err != nil { - return thrift.PrependError("write field stop error: ", err) - } - if err := oprot.WriteStructEnd(); err != nil { - return thrift.PrependError("write struct stop error: ", err) - } - return nil -} - -func (p *NodeFetchBlocksMetadataRawArgs) writeField1(oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin("req", thrift.STRUCT, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:req: ", p), err) - } - if err := p.Req.Write(oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.Req), err) - } - if err := oprot.WriteFieldEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:req: ", p), err) - } - return err -} - -func (p *NodeFetchBlocksMetadataRawArgs) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("NodeFetchBlocksMetadataRawArgs(%+v)", *p) -} - -// Attributes: -// - Success -// - Err -type NodeFetchBlocksMetadataRawResult struct { - Success *FetchBlocksMetadataRawResult_ `thrift:"success,0" db:"success" json:"success,omitempty"` - Err *Error `thrift:"err,1" db:"err" json:"err,omitempty"` -} - -func NewNodeFetchBlocksMetadataRawResult() *NodeFetchBlocksMetadataRawResult { - return &NodeFetchBlocksMetadataRawResult{} -} - -var NodeFetchBlocksMetadataRawResult_Success_DEFAULT *FetchBlocksMetadataRawResult_ - -func (p *NodeFetchBlocksMetadataRawResult) GetSuccess() *FetchBlocksMetadataRawResult_ { - if !p.IsSetSuccess() { - return NodeFetchBlocksMetadataRawResult_Success_DEFAULT - } - return p.Success -} - -var NodeFetchBlocksMetadataRawResult_Err_DEFAULT *Error - -func (p *NodeFetchBlocksMetadataRawResult) GetErr() *Error { - if !p.IsSetErr() { - return NodeFetchBlocksMetadataRawResult_Err_DEFAULT - } - return p.Err -} -func (p *NodeFetchBlocksMetadataRawResult) IsSetSuccess() bool { - return p.Success != nil -} - -func (p *NodeFetchBlocksMetadataRawResult) IsSetErr() bool { - return p.Err != nil -} - -func (p *NodeFetchBlocksMetadataRawResult) Read(iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin() - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { - break - } - switch fieldId { - case 0: - if err := p.ReadField0(iprot); err != nil { - return err - } - case 1: - if err := p.ReadField1(iprot); err != nil { - return err - } - default: - if err := iprot.Skip(fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil -} - -func (p *NodeFetchBlocksMetadataRawResult) ReadField0(iprot thrift.TProtocol) error { - p.Success = &FetchBlocksMetadataRawResult_{} - if err := p.Success.Read(iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.Success), err) - } - return nil -} - -func (p *NodeFetchBlocksMetadataRawResult) ReadField1(iprot thrift.TProtocol) error { - p.Err = &Error{ - Type: 0, - } - if err := p.Err.Read(iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.Err), err) - } - return nil -} - -func (p *NodeFetchBlocksMetadataRawResult) Write(oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin("fetchBlocksMetadataRaw_result"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) - } - if p != nil { - if err := p.writeField0(oprot); err != nil { - return err - } - if err := p.writeField1(oprot); err != nil { - return err - } - } - if err := oprot.WriteFieldStop(); err != nil { - return thrift.PrependError("write field stop error: ", err) - } - if err := oprot.WriteStructEnd(); err != nil { - return thrift.PrependError("write struct stop error: ", err) - } - return nil -} - -func (p *NodeFetchBlocksMetadataRawResult) writeField0(oprot thrift.TProtocol) (err error) { - if p.IsSetSuccess() { - if err := oprot.WriteFieldBegin("success", thrift.STRUCT, 0); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 0:success: ", p), err) - } - if err := p.Success.Write(oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.Success), err) - } - if err := oprot.WriteFieldEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 0:success: ", p), err) - } - } - return err -} - -func (p *NodeFetchBlocksMetadataRawResult) writeField1(oprot thrift.TProtocol) (err error) { - if p.IsSetErr() { - if err := oprot.WriteFieldBegin("err", thrift.STRUCT, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:err: ", p), err) - } - if err := p.Err.Write(oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.Err), err) - } - if err := oprot.WriteFieldEnd(); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:err: ", p), err) - } - } - return err -} - -func (p *NodeFetchBlocksMetadataRawResult) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("NodeFetchBlocksMetadataRawResult(%+v)", *p) -} - // Attributes: // - Req type NodeFetchBlocksMetadataRawV2Args struct { @@ -19049,16 +17563,16 @@ func (p *ClusterClient) recvHealth() (value *HealthResult_, err error) { return } if mTypeId == thrift.EXCEPTION { - error167 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error168 error - error168, err = error167.Read(iprot) + error157 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error158 error + error158, err = error157.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error168 + err = error158 return } if mTypeId != thrift.REPLY { @@ -19130,16 +17644,16 @@ func (p *ClusterClient) recvWrite() (err error) { return } if mTypeId == thrift.EXCEPTION { - error169 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error170 error - error170, err = error169.Read(iprot) + error159 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error160 error + error160, err = error159.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error170 + err = error160 return } if mTypeId != thrift.REPLY { @@ -19210,16 +17724,16 @@ func (p *ClusterClient) recvWriteTagged() (err error) { return } if mTypeId == thrift.EXCEPTION { - error171 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error172 error - error172, err = error171.Read(iprot) + error161 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error162 error + error162, err = error161.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error172 + err = error162 return } if mTypeId != thrift.REPLY { @@ -19290,16 +17804,16 @@ func (p *ClusterClient) recvQuery() (value *QueryResult_, err error) { return } if mTypeId == thrift.EXCEPTION { - error173 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error174 error - error174, err = error173.Read(iprot) + error163 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error164 error + error164, err = error163.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error174 + err = error164 return } if mTypeId != thrift.REPLY { @@ -19371,16 +17885,16 @@ func (p *ClusterClient) recvFetch() (value *FetchResult_, err error) { return } if mTypeId == thrift.EXCEPTION { - error175 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error176 error - error176, err = error175.Read(iprot) + error165 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error166 error + error166, err = error165.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error176 + err = error166 return } if mTypeId != thrift.REPLY { @@ -19452,16 +17966,16 @@ func (p *ClusterClient) recvFetchTagged() (value *FetchTaggedResult_, err error) return } if mTypeId == thrift.EXCEPTION { - error177 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error178 error - error178, err = error177.Read(iprot) + error167 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error168 error + error168, err = error167.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error178 + err = error168 return } if mTypeId != thrift.REPLY { @@ -19533,16 +18047,16 @@ func (p *ClusterClient) recvTruncate() (value *TruncateResult_, err error) { return } if mTypeId == thrift.EXCEPTION { - error179 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") - var error180 error - error180, err = error179.Read(iprot) + error169 := thrift.NewTApplicationException(thrift.UNKNOWN_APPLICATION_EXCEPTION, "Unknown Exception") + var error170 error + error170, err = error169.Read(iprot) if err != nil { return } if err = iprot.ReadMessageEnd(); err != nil { return } - err = error180 + err = error170 return } if mTypeId != thrift.REPLY { @@ -19584,15 +18098,15 @@ func (p *ClusterProcessor) ProcessorMap() map[string]thrift.TProcessorFunction { func NewClusterProcessor(handler Cluster) *ClusterProcessor { - self181 := &ClusterProcessor{handler: handler, processorMap: make(map[string]thrift.TProcessorFunction)} - self181.processorMap["health"] = &clusterProcessorHealth{handler: handler} - self181.processorMap["write"] = &clusterProcessorWrite{handler: handler} - self181.processorMap["writeTagged"] = &clusterProcessorWriteTagged{handler: handler} - self181.processorMap["query"] = &clusterProcessorQuery{handler: handler} - self181.processorMap["fetch"] = &clusterProcessorFetch{handler: handler} - self181.processorMap["fetchTagged"] = &clusterProcessorFetchTagged{handler: handler} - self181.processorMap["truncate"] = &clusterProcessorTruncate{handler: handler} - return self181 + self171 := &ClusterProcessor{handler: handler, processorMap: make(map[string]thrift.TProcessorFunction)} + self171.processorMap["health"] = &clusterProcessorHealth{handler: handler} + self171.processorMap["write"] = &clusterProcessorWrite{handler: handler} + self171.processorMap["writeTagged"] = &clusterProcessorWriteTagged{handler: handler} + self171.processorMap["query"] = &clusterProcessorQuery{handler: handler} + self171.processorMap["fetch"] = &clusterProcessorFetch{handler: handler} + self171.processorMap["fetchTagged"] = &clusterProcessorFetchTagged{handler: handler} + self171.processorMap["truncate"] = &clusterProcessorTruncate{handler: handler} + return self171 } func (p *ClusterProcessor) Process(iprot, oprot thrift.TProtocol) (success bool, err thrift.TException) { @@ -19605,12 +18119,12 @@ func (p *ClusterProcessor) Process(iprot, oprot thrift.TProtocol) (success bool, } iprot.Skip(thrift.STRUCT) iprot.ReadMessageEnd() - x182 := thrift.NewTApplicationException(thrift.UNKNOWN_METHOD, "Unknown function "+name) + x172 := thrift.NewTApplicationException(thrift.UNKNOWN_METHOD, "Unknown function "+name) oprot.WriteMessageBegin(name, thrift.EXCEPTION, seqId) - x182.Write(oprot) + x172.Write(oprot) oprot.WriteMessageEnd() oprot.Flush() - return false, x182 + return false, x172 } diff --git a/src/dbnode/generated/thrift/rpc/rpc_mock.go b/src/dbnode/generated/thrift/rpc/rpc_mock.go index 7c72399b89..4a79ba6b57 100644 --- a/src/dbnode/generated/thrift/rpc/rpc_mock.go +++ b/src/dbnode/generated/thrift/rpc/rpc_mock.go @@ -205,19 +205,6 @@ func (mr *MockTChanNodeMockRecorder) FetchBatchRaw(ctx, req interface{}) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchBatchRaw", reflect.TypeOf((*MockTChanNode)(nil).FetchBatchRaw), ctx, req) } -// FetchBlocksMetadataRaw mocks base method -func (m *MockTChanNode) FetchBlocksMetadataRaw(ctx thrift.Context, req *FetchBlocksMetadataRawRequest) (*FetchBlocksMetadataRawResult_, error) { - ret := m.ctrl.Call(m, "FetchBlocksMetadataRaw", ctx, req) - ret0, _ := ret[0].(*FetchBlocksMetadataRawResult_) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// FetchBlocksMetadataRaw indicates an expected call of FetchBlocksMetadataRaw -func (mr *MockTChanNodeMockRecorder) FetchBlocksMetadataRaw(ctx, req interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchBlocksMetadataRaw", reflect.TypeOf((*MockTChanNode)(nil).FetchBlocksMetadataRaw), ctx, req) -} - // FetchBlocksMetadataRawV2 mocks base method func (m *MockTChanNode) FetchBlocksMetadataRawV2(ctx thrift.Context, req *FetchBlocksMetadataRawV2Request) (*FetchBlocksMetadataRawV2Result_, error) { ret := m.ctrl.Call(m, "FetchBlocksMetadataRawV2", ctx, req) diff --git a/src/dbnode/generated/thrift/rpc/tchan-rpc.go b/src/dbnode/generated/thrift/rpc/tchan-rpc.go index 0e40823ea4..97b4a44db2 100644 --- a/src/dbnode/generated/thrift/rpc/tchan-rpc.go +++ b/src/dbnode/generated/thrift/rpc/tchan-rpc.go @@ -48,7 +48,6 @@ type TChanNode interface { Bootstrapped(ctx thrift.Context) (*NodeBootstrappedResult_, error) Fetch(ctx thrift.Context, req *FetchRequest) (*FetchResult_, error) FetchBatchRaw(ctx thrift.Context, req *FetchBatchRawRequest) (*FetchBatchRawResult_, error) - FetchBlocksMetadataRaw(ctx thrift.Context, req *FetchBlocksMetadataRawRequest) (*FetchBlocksMetadataRawResult_, error) FetchBlocksMetadataRawV2(ctx thrift.Context, req *FetchBlocksMetadataRawV2Request) (*FetchBlocksMetadataRawV2Result_, error) FetchBlocksRaw(ctx thrift.Context, req *FetchBlocksRawRequest) (*FetchBlocksRawResult_, error) FetchTagged(ctx thrift.Context, req *FetchTaggedRequest) (*FetchTaggedResult_, error) @@ -526,24 +525,6 @@ func (c *tchanNodeClient) FetchBatchRaw(ctx thrift.Context, req *FetchBatchRawRe return resp.GetSuccess(), err } -func (c *tchanNodeClient) FetchBlocksMetadataRaw(ctx thrift.Context, req *FetchBlocksMetadataRawRequest) (*FetchBlocksMetadataRawResult_, error) { - var resp NodeFetchBlocksMetadataRawResult - args := NodeFetchBlocksMetadataRawArgs{ - Req: req, - } - success, err := c.client.Call(ctx, c.thriftService, "fetchBlocksMetadataRaw", &args, &resp) - if err == nil && !success { - switch { - case resp.Err != nil: - err = resp.Err - default: - err = fmt.Errorf("received no result or unknown exception for fetchBlocksMetadataRaw") - } - } - - return resp.GetSuccess(), err -} - func (c *tchanNodeClient) FetchBlocksMetadataRawV2(ctx thrift.Context, req *FetchBlocksMetadataRawV2Request) (*FetchBlocksMetadataRawV2Result_, error) { var resp NodeFetchBlocksMetadataRawV2Result args := NodeFetchBlocksMetadataRawV2Args{ @@ -895,7 +876,6 @@ func (s *tchanNodeServer) Methods() []string { "bootstrapped", "fetch", "fetchBatchRaw", - "fetchBlocksMetadataRaw", "fetchBlocksMetadataRawV2", "fetchBlocksRaw", "fetchTagged", @@ -926,8 +906,6 @@ func (s *tchanNodeServer) Handle(ctx thrift.Context, methodName string, protocol return s.handleFetch(ctx, protocol) case "fetchBatchRaw": return s.handleFetchBatchRaw(ctx, protocol) - case "fetchBlocksMetadataRaw": - return s.handleFetchBlocksMetadataRaw(ctx, protocol) case "fetchBlocksMetadataRawV2": return s.handleFetchBlocksMetadataRawV2(ctx, protocol) case "fetchBlocksRaw": @@ -1056,34 +1034,6 @@ func (s *tchanNodeServer) handleFetchBatchRaw(ctx thrift.Context, protocol athri return err == nil, &res, nil } -func (s *tchanNodeServer) handleFetchBlocksMetadataRaw(ctx thrift.Context, protocol athrift.TProtocol) (bool, athrift.TStruct, error) { - var req NodeFetchBlocksMetadataRawArgs - var res NodeFetchBlocksMetadataRawResult - - if err := req.Read(protocol); err != nil { - return false, nil, err - } - - r, err := - s.handler.FetchBlocksMetadataRaw(ctx, req.Req) - - if err != nil { - switch v := err.(type) { - case *Error: - if v == nil { - return false, nil, fmt.Errorf("Handler for err returned non-nil error type *Error but nil value") - } - res.Err = v - default: - return false, nil, err - } - } else { - res.Success = r - } - - return err == nil, &res, nil -} - func (s *tchanNodeServer) handleFetchBlocksMetadataRawV2(ctx thrift.Context, protocol athrift.TProtocol) (bool, athrift.TStruct, error) { var req NodeFetchBlocksMetadataRawV2Args var res NodeFetchBlocksMetadataRawV2Result diff --git a/src/dbnode/integration/admin_session_fetch_blocks_test.go b/src/dbnode/integration/admin_session_fetch_blocks_test.go index d51e997625..ce8a98de04 100644 --- a/src/dbnode/integration/admin_session_fetch_blocks_test.go +++ b/src/dbnode/integration/admin_session_fetch_blocks_test.go @@ -26,7 +26,6 @@ import ( "testing" "time" - "github.com/m3db/m3/src/dbnode/client" "github.com/m3db/m3/src/dbnode/integration/generate" "github.com/m3db/m3/src/dbnode/storage/block" "github.com/m3db/m3/src/dbnode/storage/namespace" @@ -105,9 +104,8 @@ func testSetupMetadatas( // FetchMetadataBlocksFromPeers/FetchBlocksFromPeers adminClient := testSetup.m3dbVerificationAdminClient level := topology.ReadConsistencyLevelMajority - version := client.FetchBlocksMetadataEndpointV2 metadatasByShard, err := m3dbClientFetchBlocksMetadata(adminClient, - namespace, testSetup.shardSet.AllIDs(), start, end, level, version) + namespace, testSetup.shardSet.AllIDs(), start, end, level) require.NoError(t, err) return metadatasByShard } diff --git a/src/dbnode/integration/client.go b/src/dbnode/integration/client.go index 3080309212..d238d47ac4 100644 --- a/src/dbnode/integration/client.go +++ b/src/dbnode/integration/client.go @@ -193,7 +193,6 @@ func m3dbClientFetchBlocksMetadata( shards []uint32, start, end time.Time, consistencyLevel topology.ReadConsistencyLevel, - version client.FetchBlocksMetadataEndpointVersion, ) (map[uint32][]block.ReplicaMetadata, error) { session, err := c.DefaultAdminSession() if err != nil { @@ -212,7 +211,7 @@ func m3dbClientFetchBlocksMetadata( var metadatas []block.ReplicaMetadata iter, err := session.FetchBlocksMetadataFromPeers(namespace, - shardID, start, end, consistencyLevel, result.NewOptions(), version) + shardID, start, end, consistencyLevel, result.NewOptions()) if err != nil { return nil, err } diff --git a/src/dbnode/integration/integration.go b/src/dbnode/integration/integration.go index ccfae4a8d7..3b0d913a8e 100644 --- a/src/dbnode/integration/integration.go +++ b/src/dbnode/integration/integration.go @@ -38,7 +38,6 @@ import ( "github.com/m3db/m3/src/dbnode/storage/bootstrap/bootstrapper/uninitialized" "github.com/m3db/m3/src/dbnode/storage/bootstrap/result" "github.com/m3db/m3/src/dbnode/storage/namespace" - "github.com/m3db/m3/src/dbnode/storage/series" "github.com/m3db/m3/src/dbnode/topology" "github.com/m3db/m3/src/dbnode/topology/testutil" xmetrics "github.com/m3db/m3/src/dbnode/x/metrics" @@ -107,15 +106,14 @@ func newMultiAddrAdminClient( } type bootstrappableTestSetupOptions struct { - finalBootstrapper string - fetchBlocksMetadataEndpointVersion client.FetchBlocksMetadataEndpointVersion - bootstrapBlocksBatchSize int - bootstrapBlocksConcurrency int - bootstrapConsistencyLevel topology.ReadConsistencyLevel - topologyInitializer topology.Initializer - testStatsReporter xmetrics.TestStatsReporter - disablePeersBootstrapper bool - useTChannelClientForWriting bool + finalBootstrapper string + bootstrapBlocksBatchSize int + bootstrapBlocksConcurrency int + bootstrapConsistencyLevel topology.ReadConsistencyLevel + topologyInitializer topology.Initializer + testStatsReporter xmetrics.TestStatsReporter + disablePeersBootstrapper bool + useTChannelClientForWriting bool } type closeFn func() @@ -146,17 +144,8 @@ func newDefaultBootstrappableTestSetups( defer cleanupFnsMutex.Unlock() cleanupFns = append(cleanupFns, fn) } - anySetupUsingFetchBlocksMetadataEndpointV1 = false ) - for i := range setupOpts { - v1 := client.FetchBlocksMetadataEndpointV1 - if setupOpts[i].fetchBlocksMetadataEndpointVersion == v1 { - anySetupUsingFetchBlocksMetadataEndpointV1 = true - break - } - } - shardSet, err := newTestShardSet(opts.NumShards()) require.NoError(t, err) for i := 0; i < replicas; i++ { @@ -213,12 +202,6 @@ func newDefaultBootstrappableTestSetups( require.NoError(t, err) topologyInitializer = setup.topoInit - // Force correct series cache policy if using V1 version - // TODO: Remove once v1 endpoint is gone - if anySetupUsingFetchBlocksMetadataEndpointV1 { - setup.storageOpts = setup.storageOpts.SetSeriesCachePolicy(series.CacheAll) - } - instrumentOpts := setup.storageOpts.InstrumentOptions() logger := instrumentOpts.Logger() logger = logger.WithFields(xlog.NewField("instance", instance)) @@ -278,7 +261,6 @@ func newDefaultBootstrappableTestSetups( peersOpts := peers.NewOptions(). SetResultOptions(bsOpts). SetAdminClient(adminClient). - SetFetchBlocksMetadataEndpointVersion(setupOpts[i].fetchBlocksMetadataEndpointVersion). // DatabaseBlockRetrieverManager and PersistManager need to be set or we will never execute // the persist bootstrapping path SetDatabaseBlockRetrieverManager(setup.storageOpts.DatabaseBlockRetrieverManager()). diff --git a/src/dbnode/integration/peers_bootstrap_high_concurrency_test.go b/src/dbnode/integration/peers_bootstrap_high_concurrency_test.go index 0c715bde1e..fccc7c2409 100644 --- a/src/dbnode/integration/peers_bootstrap_high_concurrency_test.go +++ b/src/dbnode/integration/peers_bootstrap_high_concurrency_test.go @@ -27,7 +27,6 @@ import ( "testing" "time" - "github.com/m3db/m3/src/dbnode/client" "github.com/m3db/m3/src/dbnode/integration/generate" "github.com/m3db/m3/src/dbnode/retention" "github.com/m3db/m3/src/dbnode/storage/namespace" @@ -36,17 +35,11 @@ import ( "github.com/stretchr/testify/require" ) -// TODO(rartoul): Delete this once we've tested V2 in prod func TestPeersBootstrapHighConcurrency(t *testing.T) { if testing.Short() { t.SkipNow() } - testPeersBootstrapHighConcurrency(t, client.FetchBlocksMetadataEndpointV1) -} - -func testPeersBootstrapHighConcurrency( - t *testing.T, version client.FetchBlocksMetadataEndpointVersion) { // Test setups log := xlog.SimpleLogger retentionOpts := retention.NewOptions(). @@ -67,10 +60,9 @@ func testPeersBootstrapHighConcurrency( disablePeersBootstrapper: true, }, { - disablePeersBootstrapper: false, - bootstrapBlocksBatchSize: batchSize, - bootstrapBlocksConcurrency: concurrency, - fetchBlocksMetadataEndpointVersion: version, + disablePeersBootstrapper: false, + bootstrapBlocksBatchSize: batchSize, + bootstrapBlocksConcurrency: concurrency, }, } setups, closeFn := newDefaultBootstrappableTestSetups(t, opts, setupOpts) diff --git a/src/dbnode/integration/peers_bootstrap_index_test.go b/src/dbnode/integration/peers_bootstrap_index_test.go index 4ecb202c2d..ae19725e65 100644 --- a/src/dbnode/integration/peers_bootstrap_index_test.go +++ b/src/dbnode/integration/peers_bootstrap_index_test.go @@ -26,7 +26,6 @@ import ( "testing" "time" - "github.com/m3db/m3/src/dbnode/client" "github.com/m3db/m3/src/dbnode/integration/generate" "github.com/m3db/m3/src/dbnode/retention" "github.com/m3db/m3/src/dbnode/storage/index" @@ -45,8 +44,6 @@ func TestPeersBootstrapIndexWithIndexingEnabled(t *testing.T) { log := xlog.SimpleLogger - // Index metadata exchange is only possible with version 2 - version := client.FetchBlocksMetadataEndpointV2 blockSize := 2 * time.Hour rOpts := retention.NewOptions(). @@ -68,7 +65,7 @@ func TestPeersBootstrapIndexWithIndexingEnabled(t *testing.T) { setupOpts := []bootstrappableTestSetupOptions{ {disablePeersBootstrapper: true}, - {disablePeersBootstrapper: false, fetchBlocksMetadataEndpointVersion: version}, + {disablePeersBootstrapper: false}, } setups, closeFn := newDefaultBootstrappableTestSetups(t, opts, setupOpts) defer closeFn() diff --git a/src/dbnode/integration/peers_bootstrap_merge_local_test.go b/src/dbnode/integration/peers_bootstrap_merge_local_test.go index a5004f645a..fc07924de3 100644 --- a/src/dbnode/integration/peers_bootstrap_merge_local_test.go +++ b/src/dbnode/integration/peers_bootstrap_merge_local_test.go @@ -26,7 +26,6 @@ import ( "testing" "time" - "github.com/m3db/m3/src/dbnode/client" "github.com/m3db/m3/src/dbnode/integration/generate" "github.com/m3db/m3/src/dbnode/retention" "github.com/m3db/m3/src/dbnode/storage/namespace" @@ -37,17 +36,11 @@ import ( "github.com/stretchr/testify/require" ) -// TODO(rartoul): Delete this once we've tested V2 in prod func TestPeersBootstrapMergeLocal(t *testing.T) { if testing.Short() { t.SkipNow() } - testPeersBootstrapMergeLocal(t, client.FetchBlocksMetadataEndpointV1) -} - -func testPeersBootstrapMergeLocal( - t *testing.T, version client.FetchBlocksMetadataEndpointVersion) { // Test setups log := xlog.SimpleLogger retentionOpts := retention.NewOptions(). @@ -74,10 +67,9 @@ func testPeersBootstrapMergeLocal( useTChannelClientForWriting: true, }, { - disablePeersBootstrapper: false, - useTChannelClientForWriting: true, - testStatsReporter: reporter, - fetchBlocksMetadataEndpointVersion: version, + disablePeersBootstrapper: false, + useTChannelClientForWriting: true, + testStatsReporter: reporter, }, } ) diff --git a/src/dbnode/integration/peers_bootstrap_merge_peer_blocks_test.go b/src/dbnode/integration/peers_bootstrap_merge_peer_blocks_test.go index d57ab7fcda..68f3db8200 100644 --- a/src/dbnode/integration/peers_bootstrap_merge_peer_blocks_test.go +++ b/src/dbnode/integration/peers_bootstrap_merge_peer_blocks_test.go @@ -26,7 +26,6 @@ import ( "testing" "time" - "github.com/m3db/m3/src/dbnode/client" "github.com/m3db/m3/src/dbnode/integration/generate" "github.com/m3db/m3/src/dbnode/retention" "github.com/m3db/m3/src/dbnode/storage/namespace" @@ -37,17 +36,11 @@ import ( "github.com/stretchr/testify/require" ) -// TODO(rartoul): Delete this once we've tested V2 in prod func TestPeersBootstrapMergePeerBlocks(t *testing.T) { if testing.Short() { t.SkipNow() } - testPeersBootstrapMergePeerBlocks(t, client.FetchBlocksMetadataEndpointV1) -} - -func testPeersBootstrapMergePeerBlocks( - t *testing.T, version client.FetchBlocksMetadataEndpointVersion) { // Test setups log := xlog.SimpleLogger retentionOpts := retention.NewOptions(). @@ -63,7 +56,7 @@ func testPeersBootstrapMergePeerBlocks( setupOpts := []bootstrappableTestSetupOptions{ {disablePeersBootstrapper: true}, {disablePeersBootstrapper: true}, - {disablePeersBootstrapper: false, fetchBlocksMetadataEndpointVersion: version}, + {disablePeersBootstrapper: false}, } setups, closeFn := newDefaultBootstrappableTestSetups(t, opts, setupOpts) defer closeFn() diff --git a/src/dbnode/integration/peers_bootstrap_node_down_test.go b/src/dbnode/integration/peers_bootstrap_node_down_test.go index 3d5164b6a0..09be80649a 100644 --- a/src/dbnode/integration/peers_bootstrap_node_down_test.go +++ b/src/dbnode/integration/peers_bootstrap_node_down_test.go @@ -26,7 +26,6 @@ import ( "testing" "time" - "github.com/m3db/m3/src/dbnode/client" "github.com/m3db/m3/src/dbnode/integration/generate" "github.com/m3db/m3/src/dbnode/retention" "github.com/m3db/m3/src/dbnode/storage/namespace" @@ -35,17 +34,11 @@ import ( "github.com/stretchr/testify/require" ) -// TODO(rartoul): Delete this once we've tested V2 in prod func TestPeersBootstrapNodeDown(t *testing.T) { if testing.Short() { t.SkipNow() } - testPeersBootstrapNodeDown(t, client.FetchBlocksMetadataEndpointV1) -} - -func testPeersBootstrapNodeDown( - t *testing.T, version client.FetchBlocksMetadataEndpointVersion) { // Test setups log := xlog.SimpleLogger retentionOpts := retention.NewOptions(). @@ -63,7 +56,7 @@ func testPeersBootstrapNodeDown( setupOpts := []bootstrappableTestSetupOptions{ {disablePeersBootstrapper: true}, {disablePeersBootstrapper: true}, - {disablePeersBootstrapper: false, fetchBlocksMetadataEndpointVersion: version}, + {disablePeersBootstrapper: false}, } setups, closeFn := newDefaultBootstrappableTestSetups(t, opts, setupOpts) defer closeFn() diff --git a/src/dbnode/integration/peers_bootstrap_none_available_test.go b/src/dbnode/integration/peers_bootstrap_none_available_test.go index 64f5a8d1a6..ec067501ad 100644 --- a/src/dbnode/integration/peers_bootstrap_none_available_test.go +++ b/src/dbnode/integration/peers_bootstrap_none_available_test.go @@ -29,7 +29,6 @@ import ( "github.com/m3db/m3/src/cluster/services" "github.com/m3db/m3/src/cluster/shard" - "github.com/m3db/m3/src/dbnode/client" "github.com/m3db/m3/src/dbnode/retention" "github.com/m3db/m3/src/dbnode/sharding" "github.com/m3db/m3/src/dbnode/storage/namespace" @@ -88,14 +87,12 @@ func TestPeersBootstrapNoneAvailable(t *testing.T) { setupOpts := []bootstrappableTestSetupOptions{ { - disablePeersBootstrapper: false, - fetchBlocksMetadataEndpointVersion: client.FetchBlocksMetadataEndpointV2, - topologyInitializer: topoInit, + disablePeersBootstrapper: false, + topologyInitializer: topoInit, }, { - disablePeersBootstrapper: false, - fetchBlocksMetadataEndpointVersion: client.FetchBlocksMetadataEndpointV2, - topologyInitializer: topoInit, + disablePeersBootstrapper: false, + topologyInitializer: topoInit, }, } setups, closeFn := newDefaultBootstrappableTestSetups(t, opts, setupOpts) diff --git a/src/dbnode/integration/peers_bootstrap_select_best_test.go b/src/dbnode/integration/peers_bootstrap_select_best_test.go index 7adb054d9c..ca2cc70656 100644 --- a/src/dbnode/integration/peers_bootstrap_select_best_test.go +++ b/src/dbnode/integration/peers_bootstrap_select_best_test.go @@ -26,7 +26,6 @@ import ( "testing" "time" - "github.com/m3db/m3/src/dbnode/client" "github.com/m3db/m3/src/dbnode/integration/generate" "github.com/m3db/m3/src/dbnode/retention" "github.com/m3db/m3/src/dbnode/storage/namespace" @@ -37,17 +36,11 @@ import ( "github.com/stretchr/testify/require" ) -// TODO(rartoul): Delete this once we've tested V2 in prod func TestPeersBootstrapSelectBest(t *testing.T) { if testing.Short() { t.SkipNow() } - testPeerBootstrapSelectBest(t, client.FetchBlocksMetadataEndpointV1) -} - -func testPeerBootstrapSelectBest( - t *testing.T, version client.FetchBlocksMetadataEndpointVersion) { // Test setups log := xlog.SimpleLogger retentionOpts := retention.NewOptions(). @@ -63,7 +56,7 @@ func testPeerBootstrapSelectBest( setupOpts := []bootstrappableTestSetupOptions{ {disablePeersBootstrapper: true}, {disablePeersBootstrapper: true}, - {disablePeersBootstrapper: false, fetchBlocksMetadataEndpointVersion: version}, + {disablePeersBootstrapper: false}, } setups, closeFn := newDefaultBootstrappableTestSetups(t, opts, setupOpts) defer closeFn() diff --git a/src/dbnode/integration/peers_bootstrap_simple_test.go b/src/dbnode/integration/peers_bootstrap_simple_test.go index 3184d9d245..2615b3d0b5 100644 --- a/src/dbnode/integration/peers_bootstrap_simple_test.go +++ b/src/dbnode/integration/peers_bootstrap_simple_test.go @@ -26,7 +26,6 @@ import ( "testing" "time" - "github.com/m3db/m3/src/dbnode/client" "github.com/m3db/m3/src/dbnode/integration/generate" "github.com/m3db/m3/src/dbnode/retention" "github.com/m3db/m3/src/dbnode/storage/namespace" @@ -35,17 +34,11 @@ import ( "github.com/stretchr/testify/require" ) -// TODO(rartoul): Delete this once we've tested V2 in prod func TestPeersBootstrapSimple(t *testing.T) { if testing.Short() { t.SkipNow() } - testPeerBootstrapSimple(t, client.FetchBlocksMetadataEndpointV1) -} - -func testPeerBootstrapSimple( - t *testing.T, version client.FetchBlocksMetadataEndpointVersion) { // Test setups log := xlog.SimpleLogger retentionOpts := retention.NewOptions(). @@ -60,7 +53,7 @@ func testPeerBootstrapSimple( setupOpts := []bootstrappableTestSetupOptions{ {disablePeersBootstrapper: true}, - {disablePeersBootstrapper: false, fetchBlocksMetadataEndpointVersion: version}, + {disablePeersBootstrapper: false}, } setups, closeFn := newDefaultBootstrappableTestSetups(t, opts, setupOpts) defer closeFn() diff --git a/src/dbnode/integration/peers_bootstrap_single_node_test.go b/src/dbnode/integration/peers_bootstrap_single_node_test.go index b71d1b9c9d..482eda78d9 100644 --- a/src/dbnode/integration/peers_bootstrap_single_node_test.go +++ b/src/dbnode/integration/peers_bootstrap_single_node_test.go @@ -26,7 +26,6 @@ import ( "testing" "time" - "github.com/m3db/m3/src/dbnode/client" "github.com/m3db/m3/src/dbnode/integration/generate" "github.com/m3db/m3/src/dbnode/retention" "github.com/m3db/m3/src/dbnode/storage/namespace" @@ -55,7 +54,7 @@ func TestPeersBootstrapSingleNode(t *testing.T) { SetNamespaces([]namespace.Metadata{namesp}) setupOpts := []bootstrappableTestSetupOptions{ - {disablePeersBootstrapper: false, fetchBlocksMetadataEndpointVersion: client.FetchBlocksMetadataEndpointV1}, + {disablePeersBootstrapper: false}, } setups, closeFn := newDefaultBootstrappableTestSetups(t, opts, setupOpts) defer closeFn() diff --git a/src/dbnode/integration/peers_bootstrap_v2_high_concurrency_test.go b/src/dbnode/integration/peers_bootstrap_v2_high_concurrency_test.go deleted file mode 100644 index f2ae6a2849..0000000000 --- a/src/dbnode/integration/peers_bootstrap_v2_high_concurrency_test.go +++ /dev/null @@ -1,37 +0,0 @@ -// +build integration - -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package integration - -import ( - "testing" - - "github.com/m3db/m3/src/dbnode/client" -) - -func TestPeersBootstrapHighConcurrencyV2(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - - testPeersBootstrapHighConcurrency(t, client.FetchBlocksMetadataEndpointV2) -} diff --git a/src/dbnode/integration/peers_bootstrap_v2_merge_local_test.go b/src/dbnode/integration/peers_bootstrap_v2_merge_local_test.go deleted file mode 100644 index 821ac8f207..0000000000 --- a/src/dbnode/integration/peers_bootstrap_v2_merge_local_test.go +++ /dev/null @@ -1,37 +0,0 @@ -// +build integration - -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package integration - -import ( - "testing" - - "github.com/m3db/m3/src/dbnode/client" -) - -func TestPeersBootstrapMergeLocalV2(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - - testPeersBootstrapMergeLocal(t, client.FetchBlocksMetadataEndpointV2) -} diff --git a/src/dbnode/integration/peers_bootstrap_v2_merge_peer_blocks_test.go b/src/dbnode/integration/peers_bootstrap_v2_merge_peer_blocks_test.go deleted file mode 100644 index 057fdfb34c..0000000000 --- a/src/dbnode/integration/peers_bootstrap_v2_merge_peer_blocks_test.go +++ /dev/null @@ -1,37 +0,0 @@ -// +build integration - -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package integration - -import ( - "testing" - - "github.com/m3db/m3/src/dbnode/client" -) - -func TestPeersBootstrapMergePeerBlocksV2(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - - testPeersBootstrapMergePeerBlocks(t, client.FetchBlocksMetadataEndpointV2) -} diff --git a/src/dbnode/integration/peers_bootstrap_v2_node_down_test.go b/src/dbnode/integration/peers_bootstrap_v2_node_down_test.go deleted file mode 100644 index 76844bfbe5..0000000000 --- a/src/dbnode/integration/peers_bootstrap_v2_node_down_test.go +++ /dev/null @@ -1,37 +0,0 @@ -// +build integration - -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package integration - -import ( - "testing" - - "github.com/m3db/m3/src/dbnode/client" -) - -func TestPeersBootstrapNodeDownV2(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - - testPeersBootstrapNodeDown(t, client.FetchBlocksMetadataEndpointV2) -} diff --git a/src/dbnode/integration/peers_bootstrap_v2_select_best_test.go b/src/dbnode/integration/peers_bootstrap_v2_select_best_test.go deleted file mode 100644 index 27731375e7..0000000000 --- a/src/dbnode/integration/peers_bootstrap_v2_select_best_test.go +++ /dev/null @@ -1,37 +0,0 @@ -// +build integration - -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package integration - -import ( - "testing" - - "github.com/m3db/m3/src/dbnode/client" -) - -func TestPeersBootstrapSelectBestV2(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - - testPeerBootstrapSelectBest(t, client.FetchBlocksMetadataEndpointV2) -} diff --git a/src/dbnode/integration/peers_bootstrap_v2_simple_test.go b/src/dbnode/integration/peers_bootstrap_v2_simple_test.go deleted file mode 100644 index 8b2e9317dc..0000000000 --- a/src/dbnode/integration/peers_bootstrap_v2_simple_test.go +++ /dev/null @@ -1,37 +0,0 @@ -// +build integration - -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package integration - -import ( - "testing" - - "github.com/m3db/m3/src/dbnode/client" -) - -func TestPeersBootstrapSimpleV2(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - - testPeerBootstrapSimple(t, client.FetchBlocksMetadataEndpointV2) -} diff --git a/src/dbnode/network/server/tchannelthrift/node/service.go b/src/dbnode/network/server/tchannelthrift/node/service.go index d6105df29c..2b3cf26cd5 100644 --- a/src/dbnode/network/server/tchannelthrift/node/service.go +++ b/src/dbnode/network/server/tchannelthrift/node/service.go @@ -131,12 +131,8 @@ type pools struct { checkedBytesWrapper xpool.CheckedBytesWrapperPool segmentsArray segmentsArrayPool writeBatchPooledReqPool *writeBatchPooledReqPool - blockMetadata tchannelthrift.BlockMetadataPool blockMetadataV2 tchannelthrift.BlockMetadataV2Pool - blockMetadataSlice tchannelthrift.BlockMetadataSlicePool blockMetadataV2Slice tchannelthrift.BlockMetadataV2SlicePool - blocksMetadata tchannelthrift.BlocksMetadataPool - blocksMetadataSlice tchannelthrift.BlocksMetadataSlicePool } // ensure `pools` matches a required conversion interface @@ -195,12 +191,8 @@ func NewService(db storage.Database, opts tchannelthrift.Options) rpc.TChanNode id: db.Options().IdentifierPool(), segmentsArray: segmentPool, writeBatchPooledReqPool: writeBatchPooledReqPool, - blockMetadata: opts.BlockMetadataPool(), blockMetadataV2: opts.BlockMetadataV2Pool(), - blockMetadataSlice: opts.BlockMetadataSlicePool(), blockMetadataV2Slice: opts.BlockMetadataV2SlicePool(), - blocksMetadata: opts.BlocksMetadataPool(), - blocksMetadataSlice: opts.BlocksMetadataSlicePool(), }, health: &rpc.NodeHealthResult_{ Ok: true, @@ -599,110 +591,6 @@ func (s *service) FetchBlocksRaw(tctx thrift.Context, req *rpc.FetchBlocksRawReq return res, nil } -func (s *service) FetchBlocksMetadataRaw(tctx thrift.Context, req *rpc.FetchBlocksMetadataRawRequest) (*rpc.FetchBlocksMetadataRawResult_, error) { - if s.db.IsOverloaded() { - s.metrics.overloadRejected.Inc(1) - return nil, tterrors.NewInternalError(errServerIsOverloaded) - } - - callStart := s.nowFn() - ctx := tchannelthrift.Context(tctx) - - start := time.Unix(0, req.RangeStart) - end := time.Unix(0, req.RangeEnd) - - if req.Limit <= 0 { - s.metrics.fetchBlocksMetadata.ReportSuccess(s.nowFn().Sub(callStart)) - return nil, nil - } - - var pageToken int64 - if req.PageToken != nil { - pageToken = *req.PageToken - } - - var opts block.FetchBlocksMetadataOptions - if req.IncludeSizes != nil { - opts.IncludeSizes = *req.IncludeSizes - } - if req.IncludeChecksums != nil { - opts.IncludeChecksums = *req.IncludeChecksums - } - if req.IncludeLastRead != nil { - opts.IncludeLastRead = *req.IncludeLastRead - } - - nsID := s.newID(ctx, req.NameSpace) - - fetched, nextPageToken, err := s.db.FetchBlocksMetadata(ctx, nsID, - uint32(req.Shard), start, end, req.Limit, pageToken, opts) - if err != nil { - s.metrics.fetchBlocksMetadata.ReportError(s.nowFn().Sub(callStart)) - return nil, convert.ToRPCError(err) - } - ctx.RegisterCloser(fetched) - - fetchedResults := fetched.Results() - result := rpc.NewFetchBlocksMetadataRawResult_() - result.NextPageToken = nextPageToken - result.Elements = s.pools.blocksMetadataSlice.Get() - - // NB(xichen): register a closer with context so objects are returned to pool - // when we are done using them - ctx.RegisterFinalizer(s.newCloseableMetadataResult(result)) - - for _, fetchedMetadata := range fetchedResults { - blocksMetadata := s.pools.blocksMetadata.Get() - blocksMetadata.ID = fetchedMetadata.ID.Bytes() - fetchedMetadataBlocks := fetchedMetadata.Blocks.Results() - blocksMetadata.Blocks = s.pools.blockMetadataSlice.Get() - - for _, fetchedMetadataBlock := range fetchedMetadataBlocks { - blockMetadata := s.pools.blockMetadata.Get() - - blockMetadata.Start = fetchedMetadataBlock.Start.UnixNano() - - if opts.IncludeSizes { - size := fetchedMetadataBlock.Size - blockMetadata.Size = &size - } else { - blockMetadata.Size = nil - } - - checksum := fetchedMetadataBlock.Checksum - if opts.IncludeChecksums && checksum != nil { - value := int64(*checksum) - blockMetadata.Checksum = &value - } else { - blockMetadata.Checksum = nil - } - - if opts.IncludeLastRead { - lastRead := fetchedMetadataBlock.LastRead.UnixNano() - blockMetadata.LastRead = &lastRead - blockMetadata.LastReadTimeType = rpc.TimeType_UNIX_NANOSECONDS - } else { - blockMetadata.LastRead = nil - blockMetadata.LastReadTimeType = rpc.TimeType(0) - } - - if err := fetchedMetadataBlock.Err; err != nil { - blockMetadata.Err = convert.ToRPCError(err) - } else { - blockMetadata.Err = nil - } - - blocksMetadata.Blocks = append(blocksMetadata.Blocks, blockMetadata) - } - - result.Elements = append(result.Elements, blocksMetadata) - } - - s.metrics.fetchBlocksMetadata.ReportSuccess(s.nowFn().Sub(callStart)) - - return result, nil -} - func (s *service) FetchBlocksMetadataRawV2(tctx thrift.Context, req *rpc.FetchBlocksMetadataRawV2Request) (*rpc.FetchBlocksMetadataRawV2Result_, error) { if s.db.IsOverloaded() { s.metrics.overloadRejected.Inc(1) @@ -1292,28 +1180,6 @@ func (s *service) newPooledTagsDecoder( return s.newTagsDecoder(ctx, encodedTags) } -func (s *service) newCloseableMetadataResult( - res *rpc.FetchBlocksMetadataRawResult_, -) closeableMetadataResult { - return closeableMetadataResult{s: s, result: res} -} - -type closeableMetadataResult struct { - s *service - result *rpc.FetchBlocksMetadataRawResult_ -} - -func (c closeableMetadataResult) Finalize() { - for _, blocksMetadata := range c.result.Elements { - for _, blockMetadata := range blocksMetadata.Blocks { - c.s.pools.blockMetadata.Put(blockMetadata) - } - c.s.pools.blockMetadataSlice.Put(blocksMetadata.Blocks) - c.s.pools.blocksMetadata.Put(blocksMetadata) - } - c.s.pools.blocksMetadataSlice.Put(c.result.Elements) -} - func (s *service) newCloseableMetadataV2Result( res *rpc.FetchBlocksMetadataRawV2Result_, ) closeableMetadataV2Result { diff --git a/src/dbnode/network/server/tchannelthrift/node/service_test.go b/src/dbnode/network/server/tchannelthrift/node/service_test.go index 6942f381ba..a88f71ac8e 100644 --- a/src/dbnode/network/server/tchannelthrift/node/service_test.go +++ b/src/dbnode/network/server/tchannelthrift/node/service_test.go @@ -680,151 +680,6 @@ func TestServiceFetchBlocksRawIsOverloaded(t *testing.T) { require.Equal(t, tterrors.NewInternalError(errServerIsOverloaded), err) } -func TestServiceFetchBlocksMetadataRaw(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockDB := storage.NewMockDatabase(ctrl) - mockDB.EXPECT().Options().Return(testStorageOpts).AnyTimes() - mockDB.EXPECT().IsOverloaded().Return(false) - - service := NewService(mockDB, nil).(*service) - - tctx, _ := tchannelthrift.NewContext(time.Minute) - ctx := tchannelthrift.Context(tctx) - defer ctx.Close() - - start := time.Now().Add(-4 * time.Hour) - end := start.Add(4 * time.Hour) - - start, end = start.Truncate(time.Hour), end.Truncate(time.Hour) - - limit := int64(2) - pageToken := int64(0) - next := pageToken + limit - nextPageToken := &next - includeSizes := true - includeChecksums := true - includeLastRead := true - - nsID := "metrics" - - series := map[string][]struct { - start time.Time - size int64 - checksum uint32 - lastRead time.Time - }{ - "foo": { - {start.Add(0 * time.Hour), 16, 111, time.Now().Add(-time.Minute)}, - {start.Add(2 * time.Hour), 32, 222, time.Time{}}, - }, - "bar": { - {start.Add(0 * time.Hour), 32, 222, time.Time{}}, - {start.Add(2 * time.Hour), 64, 333, time.Now().Add(-time.Minute)}, - }, - } - ids := make([][]byte, 0, len(series)) - mockResult := block.NewFetchBlocksMetadataResults() - for id, s := range series { - ids = append(ids, []byte(id)) - blocks := block.NewFetchBlockMetadataResults() - metadata := block.FetchBlocksMetadataResult{ - ID: ident.StringID(id), - Blocks: blocks, - } - for _, v := range s { - entry := v - blocks.Add(block.FetchBlockMetadataResult{ - Start: entry.start, - Size: entry.size, - Checksum: &entry.checksum, - LastRead: entry.lastRead, - Err: nil, - }) - } - mockResult.Add(metadata) - } - opts := block.FetchBlocksMetadataOptions{ - IncludeSizes: includeSizes, - IncludeChecksums: includeChecksums, - IncludeLastRead: includeLastRead, - } - mockDB.EXPECT(). - FetchBlocksMetadata(ctx, ident.NewIDMatcher(nsID), uint32(0), start, end, - limit, pageToken, opts). - Return(mockResult, nextPageToken, nil) - - r, err := service.FetchBlocksMetadataRaw(tctx, &rpc.FetchBlocksMetadataRawRequest{ - NameSpace: []byte(nsID), - Shard: 0, - RangeStart: start.UnixNano(), - RangeEnd: end.UnixNano(), - Limit: limit, - PageToken: &pageToken, - IncludeSizes: &includeSizes, - IncludeChecksums: &includeChecksums, - IncludeLastRead: &includeLastRead, - }) - require.NoError(t, err) - - require.Equal(t, len(ids), len(r.Elements)) - - for _, elem := range r.Elements { - require.NotNil(t, elem) - - expectBlocks := series[string(elem.ID)] - require.Equal(t, len(expectBlocks), len(elem.Blocks)) - - for i, expectBlock := range expectBlocks { - block := elem.Blocks[i] - assert.Equal(t, expectBlock.start.UnixNano(), block.Start) - require.NotNil(t, block.Size) - require.NotNil(t, block.Checksum) - require.NotNil(t, block.LastRead) - } - } -} - -func TestServiceFetchBlocksMetadataRawIsOverloaded(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockDB := storage.NewMockDatabase(ctrl) - mockDB.EXPECT().Options().Return(testStorageOpts).AnyTimes() - mockDB.EXPECT().IsOverloaded().Return(true) - - var ( - service = NewService(mockDB, nil).(*service) - tctx, _ = tchannelthrift.NewContext(time.Minute) - ctx = tchannelthrift.Context(tctx) - start = time.Now().Add(-4 * time.Hour) - end = start.Add(4 * time.Hour) - limit = int64(2) - pageToken = int64(0) - includeSizes = true - includeChecksums = true - includeLastRead = true - nsID = "metrics" - ) - - defer ctx.Close() - start, end = start.Truncate(time.Hour), end.Truncate(time.Hour) - - _, err := service.FetchBlocksMetadataRaw(tctx, &rpc.FetchBlocksMetadataRawRequest{ - NameSpace: []byte(nsID), - Shard: 0, - RangeStart: start.UnixNano(), - RangeEnd: end.UnixNano(), - Limit: limit, - PageToken: &pageToken, - IncludeSizes: &includeSizes, - IncludeChecksums: &includeChecksums, - IncludeLastRead: &includeLastRead, - }) - require.Equal(t, tterrors.NewInternalError(errServerIsOverloaded), err) -} - func TestServiceFetchBlocksMetadataEndpointV2Raw(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() diff --git a/src/dbnode/network/server/tchannelthrift/options.go b/src/dbnode/network/server/tchannelthrift/options.go index b086e75258..e9fef98353 100644 --- a/src/dbnode/network/server/tchannelthrift/options.go +++ b/src/dbnode/network/server/tchannelthrift/options.go @@ -28,12 +28,8 @@ import ( type options struct { instrumentOpts instrument.Options - blockMetadataPool BlockMetadataPool blockMetadataV2Pool BlockMetadataV2Pool - blockMetadataSlicePool BlockMetadataSlicePool blockMetadataV2SlicePool BlockMetadataV2SlicePool - blocksMetadataPool BlocksMetadataPool - blocksMetadataSlicePool BlocksMetadataSlicePool tagEncoderPool serialize.TagEncoderPool tagDecoderPool serialize.TagDecoderPool } @@ -52,12 +48,8 @@ func NewOptions() Options { return &options{ instrumentOpts: instrument.NewOptions(), - blockMetadataPool: NewBlockMetadataPool(nil), blockMetadataV2Pool: NewBlockMetadataV2Pool(nil), - blockMetadataSlicePool: NewBlockMetadataSlicePool(nil, 0), blockMetadataV2SlicePool: NewBlockMetadataV2SlicePool(nil, 0), - blocksMetadataPool: NewBlocksMetadataPool(nil), - blocksMetadataSlicePool: NewBlocksMetadataSlicePool(nil, 0), tagEncoderPool: tagEncoderPool, tagDecoderPool: tagDecoderPool, } @@ -73,16 +65,6 @@ func (o *options) InstrumentOptions() instrument.Options { return o.instrumentOpts } -func (o *options) SetBlockMetadataPool(value BlockMetadataPool) Options { - opts := *o - opts.blockMetadataPool = value - return &opts -} - -func (o *options) BlockMetadataPool() BlockMetadataPool { - return o.blockMetadataPool -} - func (o *options) SetBlockMetadataV2Pool(value BlockMetadataV2Pool) Options { opts := *o opts.blockMetadataV2Pool = value @@ -93,16 +75,6 @@ func (o *options) BlockMetadataV2Pool() BlockMetadataV2Pool { return o.blockMetadataV2Pool } -func (o *options) SetBlockMetadataSlicePool(value BlockMetadataSlicePool) Options { - opts := *o - opts.blockMetadataSlicePool = value - return &opts -} - -func (o *options) BlockMetadataSlicePool() BlockMetadataSlicePool { - return o.blockMetadataSlicePool -} - func (o *options) SetBlockMetadataV2SlicePool(value BlockMetadataV2SlicePool) Options { opts := *o opts.blockMetadataV2SlicePool = value @@ -113,26 +85,6 @@ func (o *options) BlockMetadataV2SlicePool() BlockMetadataV2SlicePool { return o.blockMetadataV2SlicePool } -func (o *options) SetBlocksMetadataPool(value BlocksMetadataPool) Options { - opts := *o - opts.blocksMetadataPool = value - return &opts -} - -func (o *options) BlocksMetadataPool() BlocksMetadataPool { - return o.blocksMetadataPool -} - -func (o *options) SetBlocksMetadataSlicePool(value BlocksMetadataSlicePool) Options { - opts := *o - opts.blocksMetadataSlicePool = value - return &opts -} - -func (o *options) BlocksMetadataSlicePool() BlocksMetadataSlicePool { - return o.blocksMetadataSlicePool -} - func (o *options) SetTagEncoderPool(value serialize.TagEncoderPool) Options { opts := *o opts.tagEncoderPool = value diff --git a/src/dbnode/network/server/tchannelthrift/pool.go b/src/dbnode/network/server/tchannelthrift/pool.go index 0a6e72cfb3..f339ccd39a 100644 --- a/src/dbnode/network/server/tchannelthrift/pool.go +++ b/src/dbnode/network/server/tchannelthrift/pool.go @@ -26,20 +26,9 @@ import ( ) var ( - emptyBlockMetadata rpc.BlockMetadata emptyBlockMetadataV2 rpc.BlockMetadataV2 - emptyBlocksMetadata rpc.BlocksMetadata ) -// BlockMetadataPool provides a pool for block metadata -type BlockMetadataPool interface { - // Get returns a block metadata - Get() *rpc.BlockMetadata - - // Put puts a block metadata back to pool - Put(m *rpc.BlockMetadata) -} - // BlockMetadataV2Pool provides a pool for block metadata type BlockMetadataV2Pool interface { // Get returns a block metadata @@ -49,15 +38,6 @@ type BlockMetadataV2Pool interface { Put(m *rpc.BlockMetadataV2) } -// BlockMetadataSlicePool provides a pool for block metadata slices -type BlockMetadataSlicePool interface { - // Get returns a block metadata slice - Get() []*rpc.BlockMetadata - - // Put puts a block metadata slice back to pool - Put(m []*rpc.BlockMetadata) -} - // BlockMetadataV2SlicePool provides a pool for block metadata slices type BlockMetadataV2SlicePool interface { // Get returns a block metadata slice @@ -67,48 +47,6 @@ type BlockMetadataV2SlicePool interface { Put(m []*rpc.BlockMetadataV2) } -// BlocksMetadataPool provides a pool for blocks metadata -type BlocksMetadataPool interface { - // Get returns a blocks metadata - Get() *rpc.BlocksMetadata - - // Put puts a blocks metadata back to pool - Put(m *rpc.BlocksMetadata) -} - -// BlocksMetadataSlicePool provides a pool for blocks metadata slices -type BlocksMetadataSlicePool interface { - // Get returns a blocks metadata slice - Get() []*rpc.BlocksMetadata - - // Put puts a blocks metadata slice back to pool - Put(m []*rpc.BlocksMetadata) -} - -type blockMetadataPool struct { - pool pool.ObjectPool -} - -// NewBlockMetadataPool creates a new block metadata pool -func NewBlockMetadataPool(opts pool.ObjectPoolOptions) BlockMetadataPool { - p := pool.NewObjectPool(opts) - p.Init(func() interface{} { - return rpc.NewBlockMetadata() - }) - return &blockMetadataPool{pool: p} -} - -func (p *blockMetadataPool) Get() *rpc.BlockMetadata { - return p.pool.Get().(*rpc.BlockMetadata) -} - -func (p *blockMetadataPool) Put(metadata *rpc.BlockMetadata) { - if metadata != nil { - *metadata = emptyBlockMetadata - p.pool.Put(metadata) - } -} - type blockMetadataV2Pool struct { pool pool.ObjectPool } @@ -133,36 +71,6 @@ func (p *blockMetadataV2Pool) Put(metadata *rpc.BlockMetadataV2) { } } -type blockMetadataSlicePool struct { - pool pool.ObjectPool - capacity int -} - -// NewBlockMetadataSlicePool creates a new blockMetadataSlice pool -func NewBlockMetadataSlicePool(opts pool.ObjectPoolOptions, capacity int) BlockMetadataSlicePool { - p := pool.NewObjectPool(opts) - p.Init(func() interface{} { - return make([]*rpc.BlockMetadata, 0, capacity) - }) - return &blockMetadataSlicePool{pool: p, capacity: capacity} -} - -func (p *blockMetadataSlicePool) Get() []*rpc.BlockMetadata { - return p.pool.Get().([]*rpc.BlockMetadata) -} - -func (p *blockMetadataSlicePool) Put(res []*rpc.BlockMetadata) { - if res == nil || cap(res) > p.capacity { - // Don't return nil or large slices back to pool - return - } - for i := range res { - res[i] = nil - } - res = res[:0] - p.pool.Put(res) -} - type blockMetadataV2SlicePool struct { pool pool.ObjectPool capacity int @@ -192,57 +100,3 @@ func (p *blockMetadataV2SlicePool) Put(res []*rpc.BlockMetadataV2) { res = res[:0] p.pool.Put(res) } - -type blocksMetadataPool struct { - pool pool.ObjectPool -} - -// NewBlocksMetadataPool creates a new blocks metadata pool -func NewBlocksMetadataPool(opts pool.ObjectPoolOptions) BlocksMetadataPool { - p := pool.NewObjectPool(opts) - p.Init(func() interface{} { - return rpc.NewBlocksMetadata() - }) - return &blocksMetadataPool{pool: p} -} - -func (p *blocksMetadataPool) Get() *rpc.BlocksMetadata { - return p.pool.Get().(*rpc.BlocksMetadata) -} - -func (p *blocksMetadataPool) Put(metadata *rpc.BlocksMetadata) { - if metadata != nil { - *metadata = emptyBlocksMetadata - p.pool.Put(metadata) - } -} - -type blocksMetadataSlicePool struct { - pool pool.ObjectPool - capacity int -} - -// NewBlocksMetadataSlicePool creates a new blocksMetadataSlice pool -func NewBlocksMetadataSlicePool(opts pool.ObjectPoolOptions, capacity int) BlocksMetadataSlicePool { - p := pool.NewObjectPool(opts) - p.Init(func() interface{} { - return make([]*rpc.BlocksMetadata, 0, capacity) - }) - return &blocksMetadataSlicePool{pool: p, capacity: capacity} -} - -func (p *blocksMetadataSlicePool) Get() []*rpc.BlocksMetadata { - return p.pool.Get().([]*rpc.BlocksMetadata) -} - -func (p *blocksMetadataSlicePool) Put(res []*rpc.BlocksMetadata) { - if res == nil || cap(res) > p.capacity { - // Don't return nil or large slices back to pool - return - } - for i := range res { - res[i] = nil - } - res = res[:0] - p.pool.Put(res) -} diff --git a/src/dbnode/network/server/tchannelthrift/types.go b/src/dbnode/network/server/tchannelthrift/types.go index ac3da1f2ec..d1cc569c7d 100644 --- a/src/dbnode/network/server/tchannelthrift/types.go +++ b/src/dbnode/network/server/tchannelthrift/types.go @@ -33,42 +33,18 @@ type Options interface { // InstrumentOptions returns the instrumentation options InstrumentOptions() instrument.Options - // SetBlockMetadataPool sets the block metadata pool - SetBlockMetadataPool(value BlockMetadataPool) Options - - // BlockMetadataPool returns the block metadata pool - BlockMetadataPool() BlockMetadataPool - // SetBlockMetadataV2Pool sets the block metadata pool SetBlockMetadataV2Pool(value BlockMetadataV2Pool) Options // BlockMetadataV2Pool returns the block metadata pool BlockMetadataV2Pool() BlockMetadataV2Pool - // SetBlockMetadataSlicePool sets the block metadata slice pool - SetBlockMetadataSlicePool(value BlockMetadataSlicePool) Options - - // BlockMetadataSlicePool returns the block metadata slice pool - BlockMetadataSlicePool() BlockMetadataSlicePool - // SetBlockMetadataV2SlicePool sets the block metadata slice pool SetBlockMetadataV2SlicePool(value BlockMetadataV2SlicePool) Options // BlockMetadataV2SlicePool returns the block metadata slice pool BlockMetadataV2SlicePool() BlockMetadataV2SlicePool - // SetBlocksMetadataPool sets the blocks metadata pool - SetBlocksMetadataPool(value BlocksMetadataPool) Options - - // BlocksMetadataPool returns the blocks metadata pool - BlocksMetadataPool() BlocksMetadataPool - - // SetBlocksMetadataSlicePool sets the blocks metadata slice pool - SetBlocksMetadataSlicePool(value BlocksMetadataSlicePool) Options - - // BlocksMetadataSlicePool returns the blocks metadata slice pool - BlocksMetadataSlicePool() BlocksMetadataSlicePool - // SetTagEncoderPool sets the tag encoder pool. SetTagEncoderPool(value serialize.TagEncoderPool) Options diff --git a/src/dbnode/server/server.go b/src/dbnode/server/server.go index 153d58409b..530dbaca4b 100644 --- a/src/dbnode/server/server.go +++ b/src/dbnode/server/server.go @@ -454,23 +454,8 @@ func Run(runOpts RunOptions) { SetHostBlockMetadataSlicePool(hostBlockMetadataSlicePool)) // Set tchannelthrift options - blockMetadataPool := tchannelthrift.NewBlockMetadataPool( - poolOptions(policy.BlockMetadataPool, scope.SubScope("block-metadata-pool"))) - blockMetadataSlicePool := tchannelthrift.NewBlockMetadataSlicePool( - capacityPoolOptions(policy.BlockMetadataSlicePool, scope.SubScope("block-metadata-slice-pool")), - policy.BlockMetadataSlicePool.Capacity) - blocksMetadataPool := tchannelthrift.NewBlocksMetadataPool( - poolOptions(policy.BlocksMetadataPool, scope.SubScope("blocks-metadata-pool"))) - blocksMetadataSlicePool := tchannelthrift.NewBlocksMetadataSlicePool( - capacityPoolOptions(policy.BlocksMetadataSlicePool, scope.SubScope("blocks-metadata-slice-pool")), - policy.BlocksMetadataSlicePool.Capacity) - ttopts := tchannelthrift.NewOptions(). SetInstrumentOptions(opts.InstrumentOptions()). - SetBlockMetadataPool(blockMetadataPool). - SetBlockMetadataSlicePool(blockMetadataSlicePool). - SetBlocksMetadataPool(blocksMetadataPool). - SetBlocksMetadataSlicePool(blocksMetadataSlicePool). SetTagEncoderPool(tagEncoderPool). SetTagDecoderPool(tagDecoderPool) diff --git a/src/dbnode/storage/block/block.go b/src/dbnode/storage/block/block.go index 5d77905699..835a5f5db5 100644 --- a/src/dbnode/storage/block/block.go +++ b/src/dbnode/storage/block/block.go @@ -55,8 +55,7 @@ type dbBlock struct { mergeTarget DatabaseBlock - retriever DatabaseShardBlockRetriever - retrieveID ident.ID + seriesID ident.ID onEvicted OnEvictedFromWiredList @@ -97,24 +96,6 @@ func NewDatabaseBlock( return b } -// NewRetrievableDatabaseBlock creates a new retrievable DatabaseBlock instance. -func NewRetrievableDatabaseBlock( - start time.Time, - blockSize time.Duration, - retriever DatabaseShardBlockRetriever, - metadata RetrievableBlockMetadata, - opts Options, -) DatabaseBlock { - b := &dbBlock{ - opts: opts, - startUnixNanos: start.UnixNano(), - blockSize: blockSize, - closed: false, - } - b.resetRetrievableWithLock(retriever, metadata) - return b -} - func (b *dbBlock) StartTime() time.Time { b.RLock() start := b.startWithRLock() @@ -186,26 +167,6 @@ func (b *dbBlock) Checksum() (uint32, error) { return b.checksum, nil } -func (b *dbBlock) OnRetrieveBlock( - id ident.ID, - _ ident.TagIterator, - startTime time.Time, - segment ts.Segment, -) { - b.Lock() - defer b.Unlock() - - if b.closed || - !id.Equal(b.retrieveID) || - !startTime.Equal(b.startWithRLock()) { - return - } - - b.resetSegmentWithLock(segment) - b.retrieveID = id - b.wasRetrievedFromDisk = true -} - func (b *dbBlock) Stream(blocker context.Context) (xio.BlockReader, error) { lockUpgraded := false @@ -263,13 +224,6 @@ func (b *dbBlock) HasMergeTarget() bool { return hasMergeTarget } -func (b *dbBlock) IsRetrieved() bool { - b.RLock() - retrieved := b.retriever == nil - b.RUnlock() - return retrieved -} - func (b *dbBlock) WasRetrievedFromDisk() bool { b.RLock() wasRetrieved := b.wasRetrievedFromDisk @@ -277,14 +231,6 @@ func (b *dbBlock) WasRetrievedFromDisk() bool { return wasRetrieved } -func (b *dbBlock) IsCachedBlock() bool { - b.RLock() - retrieved := b.retriever == nil - wasRetrieved := b.wasRetrievedFromDisk - b.RUnlock() - return !retrieved || wasRetrieved -} - func (b *dbBlock) Merge(other DatabaseBlock) error { b.Lock() if b.wasRetrievedFromDisk || other.WasRetrievedFromDisk() { @@ -312,51 +258,36 @@ func (b *dbBlock) Reset(start time.Time, blockSize time.Duration, segment ts.Seg b.resetSegmentWithLock(segment) } -func (b *dbBlock) ResetRetrievable( - start time.Time, - blockSize time.Duration, - retriever DatabaseShardBlockRetriever, - metadata RetrievableBlockMetadata, -) { +func (b *dbBlock) ResetFromDisk(start time.Time, blockSize time.Duration, segment ts.Segment, id ident.ID) { b.Lock() defer b.Unlock() b.resetNewBlockStartWithLock(start, blockSize) - b.resetRetrievableWithLock(retriever, metadata) + // resetSegmentWithLock sets seriesID to nil + b.resetSegmentWithLock(segment) + b.seriesID = id } func (b *dbBlock) streamWithRLock(ctx context.Context) (xio.BlockReader, error) { start := b.startWithRLock() - // If the block retrieve ID is set then it must be retrieved - var ( - blockReader xio.BlockReader - err error - ) - if b.retriever != nil { - blockReader, err = b.retriever.Stream(ctx, b.retrieveID, start, b) - if err != nil { - return xio.EmptyBlockReader, err - } - } else { - // Take a copy to avoid heavy depends on cycle - segmentReader := b.opts.SegmentReaderPool().Get() - data := b.opts.BytesPool().Get(b.segment.Len()) - data.IncRef() - if b.segment.Head != nil { - data.AppendAll(b.segment.Head.Bytes()) - } - if b.segment.Tail != nil { - data.AppendAll(b.segment.Tail.Bytes()) - } - data.DecRef() - segmentReader.Reset(ts.NewSegment(data, nil, ts.FinalizeHead)) - ctx.RegisterFinalizer(segmentReader) - - blockReader = xio.BlockReader{ - SegmentReader: segmentReader, - Start: start, - BlockSize: b.blockSize, - } + // Take a copy to avoid heavy depends on cycle + segmentReader := b.opts.SegmentReaderPool().Get() + data := b.opts.BytesPool().Get(b.segment.Len()) + data.IncRef() + if b.segment.Head != nil { + data.AppendAll(b.segment.Head.Bytes()) + } + if b.segment.Tail != nil { + data.AppendAll(b.segment.Tail.Bytes()) + } + data.DecRef() + segmentReader.Reset(ts.NewSegment(data, nil, ts.FinalizeHead)) + ctx.RegisterFinalizer(segmentReader) + + blockReader := xio.BlockReader{ + SegmentReader: segmentReader, + Start: start, + BlockSize: b.blockSize, } return blockReader, nil @@ -394,22 +325,7 @@ func (b *dbBlock) resetSegmentWithLock(seg ts.Segment) { b.segment = seg b.length = seg.Len() b.checksum = digest.SegmentChecksum(seg) - - b.retriever = nil - b.retrieveID = nil - b.wasRetrievedFromDisk = false -} - -func (b *dbBlock) resetRetrievableWithLock( - retriever DatabaseShardBlockRetriever, - metadata RetrievableBlockMetadata, -) { - b.segment = ts.Segment{} - b.length = metadata.Length - b.checksum = metadata.Checksum - - b.retriever = retriever - b.retrieveID = metadata.ID + b.seriesID = nil b.wasRetrievedFromDisk = false } @@ -500,7 +416,7 @@ func (b *dbBlock) setEnteredListAtUnixNano(value int64) { // wiredListEntry is a snapshot of a subset of the block's state that the WiredList // uses to determine if a block is eligible for inclusion in the WiredList. type wiredListEntry struct { - retrieveID ident.ID + seriesID ident.ID startTime time.Time closed bool wasRetrievedFromDisk bool @@ -512,7 +428,7 @@ func (b *dbBlock) wiredListEntry() wiredListEntry { b.RLock() result := wiredListEntry{ closed: b.closed, - retrieveID: b.retrieveID, + seriesID: b.seriesID, wasRetrievedFromDisk: b.wasRetrievedFromDisk, startTime: b.startWithRLock(), } diff --git a/src/dbnode/storage/block/block_mock.go b/src/dbnode/storage/block/block_mock.go index e6e757a660..2be890e88e 100644 --- a/src/dbnode/storage/block/block_mock.go +++ b/src/dbnode/storage/block/block_mock.go @@ -265,16 +265,6 @@ func (m *MockDatabaseBlock) EXPECT() *MockDatabaseBlockMockRecorder { return m.recorder } -// OnRetrieveBlock mocks base method -func (m *MockDatabaseBlock) OnRetrieveBlock(id ident.ID, tags ident.TagIterator, startTime time.Time, segment ts.Segment) { - m.ctrl.Call(m, "OnRetrieveBlock", id, tags, startTime, segment) -} - -// OnRetrieveBlock indicates an expected call of OnRetrieveBlock -func (mr *MockDatabaseBlockMockRecorder) OnRetrieveBlock(id, tags, startTime, segment interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OnRetrieveBlock", reflect.TypeOf((*MockDatabaseBlock)(nil).OnRetrieveBlock), id, tags, startTime, segment) -} - // StartTime mocks base method func (m *MockDatabaseBlock) StartTime() time.Time { ret := m.ctrl.Call(m, "StartTime") @@ -383,18 +373,6 @@ func (mr *MockDatabaseBlockMockRecorder) HasMergeTarget() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasMergeTarget", reflect.TypeOf((*MockDatabaseBlock)(nil).HasMergeTarget)) } -// IsRetrieved mocks base method -func (m *MockDatabaseBlock) IsRetrieved() bool { - ret := m.ctrl.Call(m, "IsRetrieved") - ret0, _ := ret[0].(bool) - return ret0 -} - -// IsRetrieved indicates an expected call of IsRetrieved -func (mr *MockDatabaseBlockMockRecorder) IsRetrieved() *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsRetrieved", reflect.TypeOf((*MockDatabaseBlock)(nil).IsRetrieved)) -} - // WasRetrievedFromDisk mocks base method func (m *MockDatabaseBlock) WasRetrievedFromDisk() bool { ret := m.ctrl.Call(m, "WasRetrievedFromDisk") @@ -407,18 +385,6 @@ func (mr *MockDatabaseBlockMockRecorder) WasRetrievedFromDisk() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WasRetrievedFromDisk", reflect.TypeOf((*MockDatabaseBlock)(nil).WasRetrievedFromDisk)) } -// IsCachedBlock mocks base method -func (m *MockDatabaseBlock) IsCachedBlock() bool { - ret := m.ctrl.Call(m, "IsCachedBlock") - ret0, _ := ret[0].(bool) - return ret0 -} - -// IsCachedBlock indicates an expected call of IsCachedBlock -func (mr *MockDatabaseBlockMockRecorder) IsCachedBlock() *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsCachedBlock", reflect.TypeOf((*MockDatabaseBlock)(nil).IsCachedBlock)) -} - // Reset mocks base method func (m *MockDatabaseBlock) Reset(startTime time.Time, blockSize time.Duration, segment ts.Segment) { m.ctrl.Call(m, "Reset", startTime, blockSize, segment) @@ -429,14 +395,14 @@ func (mr *MockDatabaseBlockMockRecorder) Reset(startTime, blockSize, segment int return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reset", reflect.TypeOf((*MockDatabaseBlock)(nil).Reset), startTime, blockSize, segment) } -// ResetRetrievable mocks base method -func (m *MockDatabaseBlock) ResetRetrievable(startTime time.Time, blockSize time.Duration, retriever DatabaseShardBlockRetriever, metadata RetrievableBlockMetadata) { - m.ctrl.Call(m, "ResetRetrievable", startTime, blockSize, retriever, metadata) +// ResetFromDisk mocks base method +func (m *MockDatabaseBlock) ResetFromDisk(startTime time.Time, blockSize time.Duration, segment ts.Segment, id ident.ID) { + m.ctrl.Call(m, "ResetFromDisk", startTime, blockSize, segment, id) } -// ResetRetrievable indicates an expected call of ResetRetrievable -func (mr *MockDatabaseBlockMockRecorder) ResetRetrievable(startTime, blockSize, retriever, metadata interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResetRetrievable", reflect.TypeOf((*MockDatabaseBlock)(nil).ResetRetrievable), startTime, blockSize, retriever, metadata) +// ResetFromDisk indicates an expected call of ResetFromDisk +func (mr *MockDatabaseBlockMockRecorder) ResetFromDisk(startTime, blockSize, segment, id interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResetFromDisk", reflect.TypeOf((*MockDatabaseBlock)(nil).ResetFromDisk), startTime, blockSize, segment, id) } // Discard mocks base method diff --git a/src/dbnode/storage/block/types.go b/src/dbnode/storage/block/types.go index 97fa6eea7d..ea67da6c67 100644 --- a/src/dbnode/storage/block/types.go +++ b/src/dbnode/storage/block/types.go @@ -135,8 +135,6 @@ type NewDatabaseBlockFn func() DatabaseBlock // DatabaseBlock is the interface for a DatabaseBlock type DatabaseBlock interface { - OnRetrieveBlock - // StartTime returns the start time of the block. StartTime() time.Time @@ -168,27 +166,18 @@ type DatabaseBlock interface { // merged during Stream(). HasMergeTarget() bool - // IsRetrieved returns whether the block is already retrieved. Only - // meaningful in the context of the CacheAllMetadata series caching policy. - IsRetrieved() bool - // WasRetrievedFromDisk returns whether the block was retrieved from storage. WasRetrievedFromDisk() bool - // IsCachedBlock returns whether the block is not retrieved, or rather - // only the metadata is currently available, or whether it was retrieved - // from storage to serve as a memory cached block for reads. - IsCachedBlock() bool - // Reset resets the block start time, duration, and the segment. Reset(startTime time.Time, blockSize time.Duration, segment ts.Segment) - // ResetRetrievable resets the block to become retrievable. - ResetRetrievable( + // ResetFromDisk resets the block start time, duration, segment, and id. + ResetFromDisk( startTime time.Time, blockSize time.Duration, - retriever DatabaseShardBlockRetriever, - metadata RetrievableBlockMetadata, + segment ts.Segment, + id ident.ID, ) // Discard closes the block, but returns the (unfinalized) segment. diff --git a/src/dbnode/storage/block/wired_list.go b/src/dbnode/storage/block/wired_list.go index b9c41efa30..7e0f74783c 100644 --- a/src/dbnode/storage/block/wired_list.go +++ b/src/dbnode/storage/block/wired_list.go @@ -283,7 +283,17 @@ func (l *WiredList) insertAfter(v, at DatabaseBlock) { // Evict the block before closing it so that callers of series.ReadEncoded() // don't get errors about trying to read from a closed block. if onEvict := bl.OnEvictedFromWiredList(); onEvict != nil { - onEvict.OnEvictedFromWiredList(entry.retrieveID, entry.startTime) + if entry.seriesID == nil { + // Entry should always have a series ID attached + invariantLogger := instrument.EmitInvariantViolationAndGetLogger(l.iOpts) + invariantLogger.WithFields( + xlog.NewField("blockStart", entry.startTime), + xlog.NewField("closed", entry.closed), + xlog.NewField("wasRetrievedFromDisk", entry.wasRetrievedFromDisk), + ).Errorf("wired list entry does not have seriesID set") + } else { + onEvict.OnEvictedFromWiredList(entry.seriesID, entry.startTime) + } } // bl.CloseIfFromDisk() will return the block to the pool. In order to avoid diff --git a/src/dbnode/storage/block/wired_list_test.go b/src/dbnode/storage/block/wired_list_test.go index baf65c0c30..db1bb42656 100644 --- a/src/dbnode/storage/block/wired_list_test.go +++ b/src/dbnode/storage/block/wired_list_test.go @@ -75,7 +75,7 @@ func newTestUnwireableBlock( bl := NewDatabaseBlock(time.Time{}, 0, segment, opts).(*dbBlock) bl.Lock() - bl.retrieveID = ident.StringID(name) + bl.seriesID = ident.StringID(name) bl.wasRetrievedFromDisk = true bl.Unlock() diff --git a/src/dbnode/storage/bootstrap/bootstrapper/fs/source.go b/src/dbnode/storage/bootstrap/bootstrapper/fs/source.go index fa95913635..f55b0fa95a 100644 --- a/src/dbnode/storage/bootstrap/bootstrapper/fs/source.go +++ b/src/dbnode/storage/bootstrap/bootstrapper/fs/source.go @@ -506,15 +506,6 @@ func (s *fileSystemSource) loadShardReadersDataIntoShardResult( if shardRetrieverMgr != nil { shardRetriever = shardRetrieverMgr.ShardRetriever(shard) } - if seriesCachePolicy == series.CacheAllMetadata && shardRetriever == nil { - s.log.WithFields( - xlog.NewField("has-shard-retriever-mgr", shardRetrieverMgr != nil), - xlog.NewField("has-shard-retriever", shardRetriever != nil), - ).Errorf("shard retriever missing for shard: %d", shard) - s.markRunResultErrorsAndUnfulfilled(runResult, requestedRanges, - remainingRanges, timesWithErrors) - return - } } for _, r := range readers { @@ -558,8 +549,6 @@ func (s *fileSystemSource) loadShardReadersDataIntoShardResult( switch seriesCachePolicy { case series.CacheAll: validateErr = r.Validate() - case series.CacheAllMetadata: - validateErr = r.ValidateMetadata() default: err = fmt.Errorf("invalid series cache policy: %s", seriesCachePolicy.String()) } @@ -639,15 +628,11 @@ func (s *fileSystemSource) readNextEntryAndRecordBlock( id ident.ID tagsIter ident.TagIterator data checked.Bytes - length int - checksum uint32 err error ) switch seriesCachePolicy { case series.CacheAll: - id, tagsIter, data, checksum, err = r.Read() - case series.CacheAllMetadata: - id, tagsIter, length, checksum, err = r.ReadMetadata() + id, tagsIter, data, _, err = r.Read() default: err = fmt.Errorf("invalid series cache policy: %s", seriesCachePolicy.String()) } @@ -683,13 +668,6 @@ func (s *fileSystemSource) readNextEntryAndRecordBlock( case series.CacheAll: seg := ts.NewSegment(data, nil, ts.FinalizeHead) seriesBlock.Reset(blockStart, blockSize, seg) - case series.CacheAllMetadata: - metadata := block.RetrievableBlockMetadata{ - ID: id, - Length: length, - Checksum: checksum, - } - seriesBlock.ResetRetrievable(blockStart, blockSize, shardRetriever, metadata) default: return fmt.Errorf("invalid series cache policy: %s", seriesCachePolicy.String()) } @@ -914,7 +892,6 @@ func (s *fileSystemSource) read( runOpts bootstrap.RunOptions, ) (*runResult, error) { var ( - nsID = md.ID() seriesCachePolicy = s.opts.ResultOptions().SeriesCachePolicy() blockRetriever block.DatabaseBlockRetriever res *runResult @@ -958,13 +935,6 @@ func (s *fileSystemSource) read( switch seriesCachePolicy { case series.CacheAll: // No checks necessary - case series.CacheAllMetadata: - // Need to check block retriever available - if blockRetriever == nil { - return nil, fmt.Errorf( - "missing block retriever when using series cache metadata for namespace: %s", - nsID.String()) - } default: // Unless we're caching all series (or all series metadata) in memory, we // return just the availability of the files we have diff --git a/src/dbnode/storage/bootstrap/bootstrapper/peers/options.go b/src/dbnode/storage/bootstrap/bootstrapper/peers/options.go index aa3935f6d4..ffa08d6422 100644 --- a/src/dbnode/storage/bootstrap/bootstrapper/peers/options.go +++ b/src/dbnode/storage/bootstrap/bootstrapper/peers/options.go @@ -33,39 +33,35 @@ import ( ) var ( - defaultDefaultShardConcurrency = runtime.NumCPU() - defaultShardPersistenceConcurrency = int(math.Max(1, float64(runtime.NumCPU())/2)) - defaultPersistenceMaxQueueSize = 0 - defaultFetchBlocksMetadataEndpointVersion = client.FetchBlocksMetadataEndpointV1 + defaultDefaultShardConcurrency = runtime.NumCPU() + defaultShardPersistenceConcurrency = int(math.Max(1, float64(runtime.NumCPU())/2)) + defaultPersistenceMaxQueueSize = 0 ) var ( - errAdminClientNotSet = errors.New("admin client not set") - errInvalidFetchBlocksMetadataVersion = errors.New("invalid fetch blocks metadata endpoint version") - errPersistManagerNotSet = errors.New("persist manager not set") - errRuntimeOptionsManagerNotSet = errors.New("runtime options manager not set") + errAdminClientNotSet = errors.New("admin client not set") + errPersistManagerNotSet = errors.New("persist manager not set") + errRuntimeOptionsManagerNotSet = errors.New("runtime options manager not set") ) type options struct { - resultOpts result.Options - client client.AdminClient - defaultShardConcurrency int - shardPersistenceConcurrency int - persistenceMaxQueueSize int - persistManager persist.Manager - blockRetrieverManager block.DatabaseBlockRetrieverManager - fetchBlocksMetadataEndpointVersion client.FetchBlocksMetadataEndpointVersion - runtimeOptionsManager m3dbruntime.OptionsManager + resultOpts result.Options + client client.AdminClient + defaultShardConcurrency int + shardPersistenceConcurrency int + persistenceMaxQueueSize int + persistManager persist.Manager + blockRetrieverManager block.DatabaseBlockRetrieverManager + runtimeOptionsManager m3dbruntime.OptionsManager } // NewOptions creates new bootstrap options func NewOptions() Options { return &options{ - resultOpts: result.NewOptions(), - defaultShardConcurrency: defaultDefaultShardConcurrency, - shardPersistenceConcurrency: defaultShardPersistenceConcurrency, - persistenceMaxQueueSize: defaultPersistenceMaxQueueSize, - fetchBlocksMetadataEndpointVersion: defaultFetchBlocksMetadataEndpointVersion, + resultOpts: result.NewOptions(), + defaultShardConcurrency: defaultDefaultShardConcurrency, + shardPersistenceConcurrency: defaultShardPersistenceConcurrency, + persistenceMaxQueueSize: defaultPersistenceMaxQueueSize, } } @@ -73,9 +69,6 @@ func (o *options) Validate() error { if client := o.client; client == nil { return errAdminClientNotSet } - if !client.IsValidFetchBlocksMetadataEndpoint(o.fetchBlocksMetadataEndpointVersion) { - return errInvalidFetchBlocksMetadataVersion - } if o.persistManager == nil { return errPersistManagerNotSet } @@ -157,16 +150,6 @@ func (o *options) DatabaseBlockRetrieverManager() block.DatabaseBlockRetrieverMa return o.blockRetrieverManager } -func (o *options) SetFetchBlocksMetadataEndpointVersion(value client.FetchBlocksMetadataEndpointVersion) Options { - opts := *o - opts.fetchBlocksMetadataEndpointVersion = value - return &opts -} - -func (o *options) FetchBlocksMetadataEndpointVersion() client.FetchBlocksMetadataEndpointVersion { - return o.fetchBlocksMetadataEndpointVersion -} - func (o *options) SetRuntimeOptionsManager(value m3dbruntime.OptionsManager) Options { opts := *o opts.runtimeOptionsManager = value diff --git a/src/dbnode/storage/bootstrap/bootstrapper/peers/source.go b/src/dbnode/storage/bootstrap/bootstrapper/peers/source.go index 83777e1191..70c00df384 100644 --- a/src/dbnode/storage/bootstrap/bootstrapper/peers/source.go +++ b/src/dbnode/storage/bootstrap/bootstrapper/peers/source.go @@ -272,10 +272,9 @@ func (s *peersSource) fetchBootstrapBlocksFromPeers( currRange := it.Value() for blockStart := currRange.Start; blockStart.Before(currRange.End); blockStart = blockStart.Add(blockSize) { - version := s.opts.FetchBlocksMetadataEndpointVersion() blockEnd := blockStart.Add(blockSize) shardResult, err := session.FetchBootstrapBlocksFromPeers( - nsMetadata, shard, blockStart, blockEnd, bopts, version) + nsMetadata, shard, blockStart, blockEnd, bopts) s.logFetchBootstrapBlocksFromPeersOutcome(shard, shardResult, err) @@ -342,12 +341,7 @@ func (s *peersSource) logFetchBootstrapBlocksFromPeersOutcome( // shard/block and flushing it to disk. Depending on the series caching policy, // the series will either be held in memory, or removed from memory once // flushing has completed. -// Once everything has been flushed to disk then depending on the series -// caching policy the function is either done, or in the case of the -// CacheAllMetadata policy we loop through every series and make every block -// retrievable (so that we can retrieve data for the blocks that we're caching -// the metadata for). -// In addition, if the caching policy is not CacheAll or CacheAllMetadata, then +// In addition, if the caching policy is not CacheAll, then // at the end we remove all the series objects from the shard result as well // (since all their corresponding blocks have been removed anyways) to prevent // a huge memory spike caused by adding lots of unused series to the Shard @@ -364,14 +358,10 @@ func (s *peersSource) flush( var ( ropts = nsMetadata.Options().RetentionOptions() blockSize = ropts.BlockSize() - shardRetriever = shardRetrieverMgr.ShardRetriever(shard) tmpCtx = context.NewContext() seriesCachePolicy = s.opts.ResultOptions().SeriesCachePolicy() persistConfig = opts.PersistConfig() ) - if seriesCachePolicy == series.CacheAllMetadata && shardRetriever == nil { - return fmt.Errorf("shard retriever missing for shard: %d", shard) - } for start := tr.Start; start.Before(tr.End); start = start.Add(blockSize) { prepareOpts := persist.DataPrepareOptions{ @@ -440,18 +430,6 @@ func (s *peersSource) flush( case series.CacheAll: // Leave the blocks in the shard result, we need to return all blocks // so we can cache in memory - case series.CacheAllMetadata: - // NB(r): We can now make the flushed blocks retrievable, note that we - // explicitly perform another loop here and lookup the block again - // to avoid a large expensive allocation to hold onto the blocks - // that we just flushed that would have to be pooled. - // We are explicitly trading CPU time here for lower GC pressure. - metadata := block.RetrievableBlockMetadata{ - ID: s.ID, - Length: bl.Len(), - Checksum: checksum, - } - bl.ResetRetrievable(start, blockSize, shardRetriever, metadata) default: // Not caching the series or metadata in memory so finalize the block, // better to do this as we loop through to make blocks return to the @@ -486,15 +464,13 @@ func (s *peersSource) flush( } } - // We only want to retain the series metadata in one of three cases: + // We only want to retain the series metadata in one of two cases: // 1) CacheAll caching policy (because we're expected to cache everything in memory) - // 2) CacheAllMetadata caching policy (because we're expected to cache all metadata in memory) - // 3) PersistConfig.FileSetType is set to FileSetSnapshotType because that means we're bootstrapping + // 2) PersistConfig.FileSetType is set to FileSetSnapshotType because that means we're bootstrapping // an active block that we'll want to perform a flush on later, and we're only flushing here for // the sake of allowing the commit log bootstrapper to be able to recover this data if the node // goes down in-between this bootstrapper completing and the subsequent flush. shouldRetainSeriesMetadata := seriesCachePolicy == series.CacheAll || - seriesCachePolicy == series.CacheAllMetadata || persistConfig.FileSetType == persist.FileSetSnapshotType if !shouldRetainSeriesMetadata { @@ -587,7 +563,6 @@ func (s *peersSource) ReadIndex( dataBlockSize = ns.Options().RetentionOptions().BlockSize() resultOpts = s.opts.ResultOptions() idxOpts = ns.Options().IndexOptions() - version = s.opts.FetchBlocksMetadataEndpointVersion() resultLock = &sync.Mutex{} wg sync.WaitGroup ) @@ -623,7 +598,7 @@ func (s *peersSource) ReadIndex( } metadata, err := session.FetchBootstrapBlocksMetadataFromPeers(ns.ID(), - shard, currRange.Start, currRange.End, resultOpts, version) + shard, currRange.Start, currRange.End, resultOpts) if err != nil { // Make this period unfulfilled markUnfulfilled(err) diff --git a/src/dbnode/storage/bootstrap/bootstrapper/peers/source_data_test.go b/src/dbnode/storage/bootstrap/bootstrapper/peers/source_data_test.go index dc8c1fa499..1b6af55379 100644 --- a/src/dbnode/storage/bootstrap/bootstrapper/peers/source_data_test.go +++ b/src/dbnode/storage/bootstrap/bootstrapper/peers/source_data_test.go @@ -189,11 +189,11 @@ func TestPeersSourceReturnsFulfilledAndUnfulfilled(t *testing.T) { mockAdminSession := client.NewMockAdminSession(ctrl) mockAdminSession.EXPECT(). FetchBootstrapBlocksFromPeers(namespace.NewMetadataMatcher(nsMetadata), - uint32(0), start, end, gomock.Any(), client.FetchBlocksMetadataEndpointV1). + uint32(0), start, end, gomock.Any()). Return(goodResult, nil) mockAdminSession.EXPECT(). FetchBootstrapBlocksFromPeers(namespace.NewMetadataMatcher(nsMetadata), - uint32(1), start, end, gomock.Any(), client.FetchBlocksMetadataEndpointV1). + uint32(1), start, end, gomock.Any()). Return(nil, badErr) mockAdminClient := client.NewMockAdminClient(ctrl) @@ -232,7 +232,6 @@ func TestPeersSourceReturnsFulfilledAndUnfulfilled(t *testing.T) { func TestPeersSourceRunWithPersist(t *testing.T) { for _, cachePolicy := range []series.CachePolicy{ - series.CacheAllMetadata, series.CacheRecentlyRead, } { ctrl := gomock.NewController(t) @@ -268,19 +267,19 @@ func TestPeersSourceRunWithPersist(t *testing.T) { mockAdminSession := client.NewMockAdminSession(ctrl) mockAdminSession.EXPECT(). FetchBootstrapBlocksFromPeers(namespace.NewMetadataMatcher(testNsMd), - uint32(0), start, start.Add(blockSize), gomock.Any(), client.FetchBlocksMetadataEndpointV1). + uint32(0), start, start.Add(blockSize), gomock.Any()). Return(shard0ResultBlock1, nil) mockAdminSession.EXPECT(). FetchBootstrapBlocksFromPeers(namespace.NewMetadataMatcher(testNsMd), - uint32(0), start.Add(blockSize), start.Add(blockSize*2), gomock.Any(), client.FetchBlocksMetadataEndpointV1). + uint32(0), start.Add(blockSize), start.Add(blockSize*2), gomock.Any()). Return(shard0ResultBlock2, nil) mockAdminSession.EXPECT(). FetchBootstrapBlocksFromPeers(namespace.NewMetadataMatcher(testNsMd), - uint32(1), start, start.Add(blockSize), gomock.Any(), client.FetchBlocksMetadataEndpointV1). + uint32(1), start, start.Add(blockSize), gomock.Any()). Return(shard1ResultBlock1, nil) mockAdminSession.EXPECT(). FetchBootstrapBlocksFromPeers(namespace.NewMetadataMatcher(testNsMd), - uint32(1), start.Add(blockSize), start.Add(blockSize*2), gomock.Any(), client.FetchBlocksMetadataEndpointV1). + uint32(1), start.Add(blockSize), start.Add(blockSize*2), gomock.Any()). Return(shard1ResultBlock2, nil) mockAdminClient := client.NewMockAdminClient(ctrl) @@ -406,36 +405,9 @@ func TestPeersSourceRunWithPersist(t *testing.T) { require.True(t, r.Unfulfilled()[0].IsEmpty()) require.True(t, r.Unfulfilled()[1].IsEmpty()) - if cachePolicy == series.CacheAllMetadata { - assert.Equal(t, 2, len(r.ShardResults())) - require.NotNil(t, r.ShardResults()[0]) - require.NotNil(t, r.ShardResults()[1]) - - block, ok := r.ShardResults()[0].BlockAt(ident.StringID("foo"), start) - require.True(t, ok) - fooBlockChecksum, err := fooBlock.Checksum() - require.NoError(t, err) - assertBlockChecksum(t, fooBlockChecksum, block) - assert.False(t, block.IsRetrieved()) - - block, ok = r.ShardResults()[0].BlockAt(ident.StringID("bar"), start.Add(ropts.BlockSize())) - require.True(t, ok) - barBlockChecksum, err := barBlock.Checksum() - require.NoError(t, err) - assertBlockChecksum(t, barBlockChecksum, block) - assert.False(t, block.IsRetrieved()) - - block, ok = r.ShardResults()[1].BlockAt(ident.StringID("baz"), start) - require.True(t, ok) - bazBlockChecksum, err := bazBlock.Checksum() - require.NoError(t, err) - assertBlockChecksum(t, bazBlockChecksum, block) - assert.False(t, block.IsRetrieved()) - } else { - assert.Equal(t, 0, len(r.ShardResults())) - require.Nil(t, r.ShardResults()[0]) - require.Nil(t, r.ShardResults()[1]) - } + assert.Equal(t, 0, len(r.ShardResults())) + require.Nil(t, r.ShardResults()[0]) + require.Nil(t, r.ShardResults()[1]) assert.Equal(t, map[string]int{ "foo": 1, "bar": 1, "baz": 1, @@ -539,7 +511,7 @@ func TestPeersSourceMarksUnfulfilledOnPersistenceErrors(t *testing.T) { mockAdminSession.EXPECT(). FetchBootstrapBlocksFromPeers(namespace.NewMetadataMatcher(testNsMd), key.shard, time.Unix(0, key.start), time.Unix(0, key.end), - gomock.Any(), client.FetchBlocksMetadataEndpointV1). + gomock.Any()). Return(result, nil) } diff --git a/src/dbnode/storage/bootstrap/bootstrapper/peers/source_index_test.go b/src/dbnode/storage/bootstrap/bootstrapper/peers/source_index_test.go index e59750c158..6e77770a37 100644 --- a/src/dbnode/storage/bootstrap/bootstrapper/peers/source_index_test.go +++ b/src/dbnode/storage/bootstrap/bootstrapper/peers/source_index_test.go @@ -71,8 +71,7 @@ func TestBootstrapIndex(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - opts := testDefaultOpts. - SetFetchBlocksMetadataEndpointVersion(client.FetchBlocksMetadataEndpointV2) + opts := testDefaultOpts blockSize := 2 * time.Hour indexBlockSize := 2 * blockSize @@ -180,11 +179,10 @@ func TestBootstrapIndex(t *testing.T) { rangeStart := blockStart rangeEnd := rangeStart.Add(blockSize) - version := opts.FetchBlocksMetadataEndpointVersion() call := mockAdminSession.EXPECT(). FetchBootstrapBlocksMetadataFromPeers(ident.NewIDMatcher(nsID), - uint32(0), rangeStart, rangeEnd, gomock.Any(), version). + uint32(0), rangeStart, rangeEnd, gomock.Any()). Return(mockIter, nil) mockAdminSessionCalls = append(mockAdminSessionCalls, call) break @@ -199,11 +197,10 @@ func TestBootstrapIndex(t *testing.T) { rangeStart := blockStart rangeEnd := rangeStart.Add(blockSize) - version := opts.FetchBlocksMetadataEndpointVersion() call := mockAdminSession.EXPECT(). FetchBootstrapBlocksMetadataFromPeers(ident.NewIDMatcher(nsID), - uint32(0), rangeStart, rangeEnd, gomock.Any(), version). + uint32(0), rangeStart, rangeEnd, gomock.Any()). Return(mockIter, nil) mockAdminSessionCalls = append(mockAdminSessionCalls, call) } @@ -324,8 +321,7 @@ func TestBootstrapIndexErr(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - opts := testDefaultOpts. - SetFetchBlocksMetadataEndpointVersion(client.FetchBlocksMetadataEndpointV2) + opts := testDefaultOpts blockSize := 2 * time.Hour indexBlockSize := 2 * blockSize @@ -420,11 +416,10 @@ func TestBootstrapIndexErr(t *testing.T) { rangeStart := blockStart rangeEnd := rangeStart.Add(blockSize) - version := opts.FetchBlocksMetadataEndpointVersion() call := mockAdminSession.EXPECT(). FetchBootstrapBlocksMetadataFromPeers(ident.NewIDMatcher(nsID), - uint32(0), rangeStart, rangeEnd, gomock.Any(), version). + uint32(0), rangeStart, rangeEnd, gomock.Any()). Return(mockIter, nil) mockAdminSessionCalls = append(mockAdminSessionCalls, call) break @@ -439,11 +434,10 @@ func TestBootstrapIndexErr(t *testing.T) { rangeStart := blockStart rangeEnd := rangeStart.Add(blockSize) - version := opts.FetchBlocksMetadataEndpointVersion() call := mockAdminSession.EXPECT(). FetchBootstrapBlocksMetadataFromPeers(ident.NewIDMatcher(nsID), - uint32(0), rangeStart, rangeEnd, gomock.Any(), version). + uint32(0), rangeStart, rangeEnd, gomock.Any()). Return(mockIter, nil) mockAdminSessionCalls = append(mockAdminSessionCalls, call) } diff --git a/src/dbnode/storage/bootstrap/bootstrapper/peers/types.go b/src/dbnode/storage/bootstrap/bootstrapper/peers/types.go index a01b4453ec..06d6b5a8ee 100644 --- a/src/dbnode/storage/bootstrap/bootstrapper/peers/types.go +++ b/src/dbnode/storage/bootstrap/bootstrapper/peers/types.go @@ -95,14 +95,6 @@ type Options interface { // persistence enabled. DatabaseBlockRetrieverManager() block.DatabaseBlockRetrieverManager - // SetFetchBlocksMetadataEndpointVersion sets the version of the fetch blocks - // metadata endpoint that the peer bootstrapper will use - SetFetchBlocksMetadataEndpointVersion(value client.FetchBlocksMetadataEndpointVersion) Options - - // SetFetchBlocksMetadataEndpointVersion returns the version of the fetch blocks - // metadata endpoint that the peer bootstrapper will use - FetchBlocksMetadataEndpointVersion() client.FetchBlocksMetadataEndpointVersion - // SetRuntimeOptionsManagers sets the RuntimeOptionsManager. SetRuntimeOptionsManager(value m3dbruntime.OptionsManager) Options diff --git a/src/dbnode/storage/database.go b/src/dbnode/storage/database.go index 18f24c3285..0928c45cfa 100644 --- a/src/dbnode/storage/database.go +++ b/src/dbnode/storage/database.go @@ -575,25 +575,6 @@ func (d *db) FetchBlocks( return n.FetchBlocks(ctx, shardID, id, starts) } -func (d *db) FetchBlocksMetadata( - ctx context.Context, - namespace ident.ID, - shardID uint32, - start, end time.Time, - limit int64, - pageToken int64, - opts block.FetchBlocksMetadataOptions, -) (block.FetchBlocksMetadataResults, *int64, error) { - n, err := d.namespaceFor(namespace) - if err != nil { - d.metrics.unknownNamespaceFetchBlocksMetadata.Inc(1) - return nil, nil, xerrors.NewInvalidParamsError(err) - } - - return n.FetchBlocksMetadata(ctx, shardID, start, end, limit, - pageToken, opts) -} - func (d *db) FetchBlocksMetadataV2( ctx context.Context, namespace ident.ID, diff --git a/src/dbnode/storage/database_test.go b/src/dbnode/storage/database_test.go index ccf5879edd..45868f230c 100644 --- a/src/dbnode/storage/database_test.go +++ b/src/dbnode/storage/database_test.go @@ -341,78 +341,6 @@ func TestDatabaseFetchBlocksNamespaceOwned(t *testing.T) { require.NoError(t, err) } -func TestDatabaseFetchBlocksMetadataShardNotOwned(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - ctx := context.NewContext() - defer ctx.Close() - - var ( - ns = ident.StringID("testns1") - shardID = uint32(0) - start = time.Now() - end = start.Add(time.Hour) - limit = int64(100) - pageToken = int64(0) - opts = block.FetchBlocksMetadataOptions{ - IncludeSizes: true, - IncludeChecksums: true, - IncludeLastRead: true, - } - ) - d, mapCh, _ := newTestDatabase(t, ctrl, Bootstrapped) - defer func() { - close(mapCh) - }() - res, nextPageToken, err := d.FetchBlocksMetadata(ctx, ns, shardID, start, end, limit, pageToken, opts) - require.Nil(t, res) - require.Nil(t, nextPageToken) - require.Error(t, err) -} - -func TestDatabaseFetchBlocksMetadataShardOwned(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - ctx := context.NewContext() - defer ctx.Close() - - var ( - ns = ident.StringID("testns1") - shardID = uint32(397) - start = time.Now() - end = start.Add(time.Hour) - limit = int64(100) - pageToken = int64(0) - opts = block.FetchBlocksMetadataOptions{ - IncludeSizes: true, - IncludeChecksums: true, - IncludeLastRead: true, - } - ) - - d, mapCh, _ := newTestDatabase(t, ctrl, Bootstrapped) - defer func() { - close(mapCh) - }() - - expectedBlocks := block.NewFetchBlocksMetadataResults() - expectedBlocks.Add(block.NewFetchBlocksMetadataResult(ident.StringID("bar"), nil, nil)) - expectedToken := new(int64) - mockNamespace := NewMockdatabaseNamespace(ctrl) - mockNamespace. - EXPECT(). - FetchBlocksMetadata(ctx, shardID, start, end, limit, pageToken, opts). - Return(expectedBlocks, expectedToken, nil) - d.namespaces.Set(ns, mockNamespace) - - res, nextToken, err := d.FetchBlocksMetadata(ctx, ns, shardID, start, end, limit, pageToken, opts) - require.Equal(t, expectedBlocks, res) - require.Equal(t, expectedToken, nextToken) - require.Nil(t, err) -} - func TestDatabaseNamespaces(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() diff --git a/src/dbnode/storage/namespace.go b/src/dbnode/storage/namespace.go index 168948f8ee..2e165989dc 100644 --- a/src/dbnode/storage/namespace.go +++ b/src/dbnode/storage/namespace.go @@ -642,27 +642,6 @@ func (n *dbNamespace) FetchBlocks( return res, err } -func (n *dbNamespace) FetchBlocksMetadata( - ctx context.Context, - shardID uint32, - start, end time.Time, - limit int64, - pageToken int64, - opts block.FetchBlocksMetadataOptions, -) (block.FetchBlocksMetadataResults, *int64, error) { - callStart := n.nowFn() - shard, err := n.readableShardAt(shardID) - if err != nil { - n.metrics.fetchBlocksMetadata.ReportError(n.nowFn().Sub(callStart)) - return nil, nil, err - } - - res, nextPageToken, err := shard.FetchBlocksMetadata(ctx, start, end, limit, - pageToken, opts) - n.metrics.fetchBlocksMetadata.ReportSuccessOrError(err, n.nowFn().Sub(callStart)) - return res, nextPageToken, err -} - func (n *dbNamespace) FetchBlocksMetadataV2( ctx context.Context, shardID uint32, diff --git a/src/dbnode/storage/namespace/convert_prop_test.go b/src/dbnode/storage/namespace/convert_prop_test.go index cdb7cc9648..24a05a1778 100644 --- a/src/dbnode/storage/namespace/convert_prop_test.go +++ b/src/dbnode/storage/namespace/convert_prop_test.go @@ -103,7 +103,6 @@ func genMetadata() gopter.Gen { SetBlockSize(retention.BlockSize()))) if err != nil { panic(err.Error()) - return nil } return md }) diff --git a/src/dbnode/storage/namespace_test.go b/src/dbnode/storage/namespace_test.go index 8ac19871b6..4395768797 100644 --- a/src/dbnode/storage/namespace_test.go +++ b/src/dbnode/storage/namespace_test.go @@ -32,7 +32,6 @@ import ( "github.com/m3db/m3/src/dbnode/retention" "github.com/m3db/m3/src/dbnode/runtime" "github.com/m3db/m3/src/dbnode/sharding" - "github.com/m3db/m3/src/dbnode/storage/block" "github.com/m3db/m3/src/dbnode/storage/bootstrap" "github.com/m3db/m3/src/dbnode/storage/bootstrap/result" "github.com/m3db/m3/src/dbnode/storage/index" @@ -252,72 +251,6 @@ func TestNamespaceFetchBlocksShardOwned(t *testing.T) { require.Equal(t, errShardNotBootstrappedToRead, xerrors.GetInnerRetryableError(err)) } -func TestNamespaceFetchBlocksMetadataShardNotOwned(t *testing.T) { - ctx := context.NewContext() - defer ctx.Close() - - ns, closer := newTestNamespace(t) - defer closer() - - for i := range ns.shards { - ns.shards[i] = nil - } - start := time.Now() - end := start.Add(time.Hour) - opts := block.FetchBlocksMetadataOptions{ - IncludeSizes: true, - IncludeChecksums: true, - IncludeLastRead: true, - } - _, _, err := ns.FetchBlocksMetadata(ctx, testShardIDs[0].ID(), start, end, 100, 0, opts) - require.True(t, xerrors.IsRetryableError(err)) - require.Equal(t, "not responsible for shard 0", err.Error()) -} - -func TestNamespaceFetchBlocksMetadataShardOwned(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - ctx := context.NewContext() - defer ctx.Close() - - var ( - start = time.Now() - end = start.Add(time.Hour) - limit = int64(100) - pageToken = int64(0) - nextPageToken = int64(100) - opts = block.FetchBlocksMetadataOptions{ - IncludeSizes: true, - IncludeChecksums: true, - IncludeLastRead: true, - } - ) - - ns, closer := newTestNamespace(t) - defer closer() - - shard := NewMockdatabaseShard(ctrl) - shard.EXPECT(). - FetchBlocksMetadata(ctx, start, end, limit, pageToken, opts). - Return(nil, &nextPageToken, nil) - ns.shards[testShardIDs[0].ID()] = shard - - shard.EXPECT().IsBootstrapped().Return(true) - res, npt, err := ns.FetchBlocksMetadata(ctx, testShardIDs[0].ID(), - start, end, limit, pageToken, opts) - require.Nil(t, res) - require.Equal(t, npt, &nextPageToken) - require.NoError(t, err) - - shard.EXPECT().IsBootstrapped().Return(false) - _, _, err = ns.FetchBlocksMetadata(ctx, testShardIDs[0].ID(), - start, end, limit, pageToken, opts) - require.Error(t, err) - require.True(t, xerrors.IsRetryableError(err)) - require.Equal(t, errShardNotBootstrappedToRead, xerrors.GetInnerRetryableError(err)) -} - func TestNamespaceBootstrapBootstrapping(t *testing.T) { ns, closer := newTestNamespace(t) defer closer() diff --git a/src/dbnode/storage/repair.go b/src/dbnode/storage/repair.go index 26d0517ee9..3d0239b070 100644 --- a/src/dbnode/storage/repair.go +++ b/src/dbnode/storage/repair.go @@ -108,7 +108,7 @@ func (r shardRepairer) Repair( IncludeSizes: true, IncludeChecksums: true, } - localMetadata, _, err := shard.FetchBlocksMetadata(ctx, start, end, math.MaxInt64, 0, opts) + localMetadata, _, err := shard.FetchBlocksMetadataV2(ctx, start, end, math.MaxInt64, PageToken{}, opts) if err != nil { return repair.MetadataComparisonResult{}, err } @@ -123,7 +123,7 @@ func (r shardRepairer) Repair( // Add peer metadata level := r.rpopts.RepairConsistencyLevel() peerIter, err := session.FetchBlocksMetadataFromPeers(namespace, shard.ID(), start, end, - level, result.NewOptions(), client.FetchBlocksMetadataEndpointV2) + level, result.NewOptions()) if err != nil { return repair.MetadataComparisonResult{}, err } diff --git a/src/dbnode/storage/repair_test.go b/src/dbnode/storage/repair_test.go index ebb48cf9fe..dbb65e3304 100644 --- a/src/dbnode/storage/repair_test.go +++ b/src/dbnode/storage/repair_test.go @@ -263,7 +263,7 @@ func TestDatabaseShardRepairerRepair(t *testing.T) { any := gomock.Any() shard.EXPECT(). - FetchBlocksMetadata(any, start, end, any, int64(0), fetchOpts). + FetchBlocksMetadataV2(any, start, end, any, PageToken{}, fetchOpts). Return(expectedResults, nil, nil) shard.EXPECT().ID().Return(shardID).AnyTimes() @@ -292,7 +292,7 @@ func TestDatabaseShardRepairerRepair(t *testing.T) { ) session.EXPECT(). FetchBlocksMetadataFromPeers(namespace, shardID, start, end, - rpOpts.RepairConsistencyLevel(), gomock.Any(), client.FetchBlocksMetadataEndpointV2). + rpOpts.RepairConsistencyLevel(), gomock.Any()). Return(peerIter, nil) var ( diff --git a/src/dbnode/storage/series/buffer.go b/src/dbnode/storage/series/buffer.go index 92600aa954..db404de09a 100644 --- a/src/dbnode/storage/series/buffer.go +++ b/src/dbnode/storage/series/buffer.go @@ -769,21 +769,6 @@ func (b *dbBufferBucket) merge() (mergeResult, error) { encoder := bopts.EncoderPool().Get() encoder.Reset(b.start, bopts.DatabaseBlockAllocSize()) - // If we have to merge bootstrapped from disk during a merge then this - // can make ticking very slow, ensure to notify this bug - if len(b.bootstrapped) > 0 { - unretrieved := 0 - for i := range b.bootstrapped { - if !b.bootstrapped[i].IsRetrieved() { - unretrieved++ - } - } - if unretrieved > 0 { - log := b.opts.InstrumentOptions().Logger() - log.Warnf("buffer merging %d unretrieved blocks", unretrieved) - } - } - var ( start = b.start readers = make([]xio.SegmentReader, 0, len(b.encoders)+len(b.bootstrapped)) diff --git a/src/dbnode/storage/series/policy.go b/src/dbnode/storage/series/policy.go index ccc0bdf974..d506fbddf9 100644 --- a/src/dbnode/storage/series/policy.go +++ b/src/dbnode/storage/series/policy.go @@ -39,11 +39,6 @@ const ( // which requires loading all into cache on bootstrap and never // expiring series from memory until expired from retention. CacheAll - // CacheAllMetadata specifies that all series metadata but not the - // data itself must be cached at all times and the metadata is never - // expired from memory until expired from retention. - // TODO: Remove this once recently read is production grade. - CacheAllMetadata // CacheRecentlyRead specifies that series that are recently read // must be cached, configurable by the namespace block expiry after // not accessed period. @@ -59,7 +54,7 @@ const ( // ValidCachePolicies returns the valid series cache policies. func ValidCachePolicies() []CachePolicy { - return []CachePolicy{CacheNone, CacheAll, CacheAllMetadata, CacheRecentlyRead, CacheLRU} + return []CachePolicy{CacheNone, CacheAll, CacheRecentlyRead, CacheLRU} } func (p CachePolicy) String() string { @@ -68,8 +63,6 @@ func (p CachePolicy) String() string { return "none" case CacheAll: return "all" - case CacheAllMetadata: - return "all_metadata" case CacheRecentlyRead: return "recently_read" case CacheLRU: diff --git a/src/dbnode/storage/series/reader.go b/src/dbnode/storage/series/reader.go index b1cc1f47c2..596ad67d15 100644 --- a/src/dbnode/storage/series/reader.go +++ b/src/dbnode/storage/series/reader.go @@ -140,8 +140,6 @@ func (r Reader) readersWithBlocksMapAndBuffer( switch { case cachePolicy == CacheAll: // No-op, block metadata should have been in-memory - case cachePolicy == CacheAllMetadata: - // No-op, block metadata should have been in-memory case r.retriever != nil: // Try to stream from disk if r.retriever.IsBlockRetrievable(blockAt) { @@ -213,8 +211,6 @@ func (r Reader) fetchBlocksWithBlocksMapAndBuffer( switch { case cachePolicy == CacheAll: // No-op, block metadata should have been in-memory - case cachePolicy == CacheAllMetadata: - // No-op, block metadata should have been in-memory case r.retriever != nil: // Try to stream from disk if r.retriever.IsBlockRetrievable(start) { diff --git a/src/dbnode/storage/series/series.go b/src/dbnode/storage/series/series.go index 275f0bfbde..99d5e9b75a 100644 --- a/src/dbnode/storage/series/series.go +++ b/src/dbnode/storage/series/series.go @@ -202,12 +202,6 @@ func (s *dbSeries) updateBlocksWithLock() (updateBlocksResult, error) { continue } - if cachePolicy == CacheAllMetadata && !currBlock.IsRetrieved() { - // Already unwired - result.UnwiredBlocks++ - continue - } - // Potentially unwire var unwired, shouldUnwire bool // IsBlockRetrievable makes sure that the block has been flushed. This @@ -217,9 +211,6 @@ func (s *dbSeries) updateBlocksWithLock() (updateBlocksResult, error) { switch cachePolicy { case CacheNone: shouldUnwire = true - case CacheAllMetadata: - // Apply RecentlyRead logic (CacheAllMetadata is being removed soon) - fallthrough case CacheRecentlyRead: sinceLastRead := now.Sub(currBlock.LastReadTime()) shouldUnwire = sinceLastRead >= wiredTimeout @@ -235,29 +226,9 @@ func (s *dbSeries) updateBlocksWithLock() (updateBlocksResult, error) { } if shouldUnwire { - switch cachePolicy { - case CacheAllMetadata: - // Keep the metadata but remove contents - - // NB(r): Each block needs shared ref to the series ID - // or else each block needs to have a copy of the ID - id := s.id - checksum, err := currBlock.Checksum() - if err != nil { - return result, err - } - metadata := block.RetrievableBlockMetadata{ - ID: id, - Length: currBlock.Len(), - Checksum: checksum, - } - currBlock.ResetRetrievable(start, currBlock.BlockSize(), retriever, metadata) - default: - // Remove the block and it will be looked up later - s.blocks.RemoveBlockAt(start) - currBlock.Close() - } - + // Remove the block and it will be looked up later + s.blocks.RemoveBlockAt(start) + currBlock.Close() unwired = true result.madeUnwiredBlocks++ } @@ -362,7 +333,7 @@ func (s *dbSeries) FetchBlocksMetadata( if !start.Before(t.Add(blockSize)) || !t.Before(end) { continue } - if !opts.IncludeCachedBlocks && b.IsCachedBlock() { + if !opts.IncludeCachedBlocks && b.WasRetrievedFromDisk() { // Do not include cached blocks if not specified to, this is // to avoid high amounts of duplication if a significant number of // blocks are cached in memory when returning blocks metadata @@ -549,21 +520,8 @@ func (s *dbSeries) OnRetrieveBlock( } b = s.opts.DatabaseBlockOptions().DatabaseBlockPool().Get() - metadata := block.RetrievableBlockMetadata{ - ID: s.id, - Length: segment.Len(), - Checksum: digest.SegmentChecksum(segment), - } blockSize := s.opts.RetentionOptions().BlockSize() - b.ResetRetrievable(startTime, blockSize, s.blockRetriever, metadata) - // Use s.id instead of id here, because id is finalized by the context whereas - // we rely on the G.C to reclaim s.id. This is important because the block will - // hold onto the id ref, and (if the LRU caching policy is enabled) the shard - // will need it later when the WiredList calls its OnEvictedFromWiredList method. - // Also note that ResetRetrievable will mark the block as not retrieved from disk, - // but OnRetrieveBlock will then properly mark it as retrieved from disk so subsequent - // calls to WasRetrievedFromDisk will return true. - b.OnRetrieveBlock(s.id, tags, startTime, segment) + b.ResetFromDisk(startTime, blockSize, segment, s.id) // NB(r): Blocks retrieved have been triggered by a read, so set the last // read time as now so caching policies are followed. diff --git a/src/dbnode/storage/series/series_test.go b/src/dbnode/storage/series/series_test.go index 07fdd3e47b..e82caba6f0 100644 --- a/src/dbnode/storage/series/series_test.go +++ b/src/dbnode/storage/series/series_test.go @@ -279,12 +279,10 @@ func TestSeriesTickNeedsBlockExpiry(t *testing.T) { blockStart := curr.Add(-ropts.RetentionPeriod()).Add(-ropts.BlockSize()) b := block.NewMockDatabaseBlock(ctrl) b.EXPECT().StartTime().Return(blockStart) - b.EXPECT().IsRetrieved().Return(true).AnyTimes() b.EXPECT().Close() series.blocks.AddBlock(b) b = block.NewMockDatabaseBlock(ctrl) b.EXPECT().StartTime().Return(curr) - b.EXPECT().IsRetrieved().Return(true).AnyTimes() series.blocks.AddBlock(b) require.Equal(t, blockStart, series.blocks.MinTime()) require.Equal(t, 2, series.blocks.Len()) @@ -304,34 +302,6 @@ func TestSeriesTickNeedsBlockExpiry(t *testing.T) { require.True(t, exists) } -func TestSeriesTickNotRetrieved(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - opts := newSeriesTestOptions(). - SetCachePolicy(CacheAllMetadata) - ropts := opts.RetentionOptions() - curr := time.Now().Truncate(ropts.BlockSize()) - opts = opts.SetClockOptions(opts.ClockOptions().SetNowFn(func() time.Time { - return curr - })) - series := NewDatabaseSeries(ident.StringID("foo"), ident.Tags{}, opts).(*dbSeries) - blockRetriever := NewMockQueryableBlockRetriever(ctrl) - series.blockRetriever = blockRetriever - _, err := series.Bootstrap(nil) - assert.NoError(t, err) - - b := block.NewMockDatabaseBlock(ctrl) - b.EXPECT().StartTime().Return(curr) - b.EXPECT().IsRetrieved().Return(false).AnyTimes() - - series.blocks.AddBlock(b) - - tickResult, err := series.Tick() - require.NoError(t, err) - require.Equal(t, 1, tickResult.UnwiredBlocks) -} - func TestSeriesTickRecentlyRead(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -354,7 +324,6 @@ func TestSeriesTickRecentlyRead(t *testing.T) { // Test case where block has been read within expiry period - won't be removed b := block.NewMockDatabaseBlock(ctrl) b.EXPECT().StartTime().Return(curr) - b.EXPECT().IsRetrieved().Return(true).AnyTimes() b.EXPECT().LastReadTime().Return( curr.Add(-opts.RetentionOptions().BlockDataExpiryAfterNotAccessedPeriod() / 2)) b.EXPECT().HasMergeTarget().Return(true) @@ -370,7 +339,6 @@ func TestSeriesTickRecentlyRead(t *testing.T) { // Test case where block has not been read within expiry period - will be removed b = block.NewMockDatabaseBlock(ctrl) b.EXPECT().StartTime().Return(curr) - b.EXPECT().IsRetrieved().Return(true).AnyTimes() b.EXPECT().LastReadTime().Return( curr.Add(-opts.RetentionOptions().BlockDataExpiryAfterNotAccessedPeriod() * 2)) b.EXPECT().Close().Return() @@ -387,7 +355,6 @@ func TestSeriesTickRecentlyRead(t *testing.T) { b = block.NewMockDatabaseBlock(ctrl) b.EXPECT().StartTime().Return(curr) b.EXPECT().HasMergeTarget().Return(true) - b.EXPECT().IsRetrieved().Return(true).AnyTimes() series.blocks.AddBlock(b) blockRetriever.EXPECT().IsBlockRetrievable(curr).Return(false) @@ -420,7 +387,6 @@ func TestSeriesTickCacheLRU(t *testing.T) { // Test case where block was not retrieved from disk - Will be removed b := block.NewMockDatabaseBlock(ctrl) b.EXPECT().StartTime().Return(curr) - b.EXPECT().IsRetrieved().Return(true).AnyTimes() b.EXPECT().WasRetrievedFromDisk().Return(false) b.EXPECT().Close().Return() series.blocks.AddBlock(b) @@ -435,7 +401,6 @@ func TestSeriesTickCacheLRU(t *testing.T) { // Test case where block was retrieved from disk - Will not be removed b = block.NewMockDatabaseBlock(ctrl) b.EXPECT().StartTime().Return(curr) - b.EXPECT().IsRetrieved().Return(true).AnyTimes() b.EXPECT().HasMergeTarget().Return(true) b.EXPECT().WasRetrievedFromDisk().Return(true) series.blocks.AddBlock(b) @@ -450,7 +415,6 @@ func TestSeriesTickCacheLRU(t *testing.T) { // Test case where block is not flushed yet (not retrievable) - Will not be removed b = block.NewMockDatabaseBlock(ctrl) b.EXPECT().StartTime().Return(curr) - b.EXPECT().IsRetrieved().Return(true).AnyTimes() b.EXPECT().HasMergeTarget().Return(true) series.blocks.AddBlock(b) blockRetriever.EXPECT().IsBlockRetrievable(curr).Return(false) @@ -458,7 +422,6 @@ func TestSeriesTickCacheLRU(t *testing.T) { // Test case where block was retrieved from disk and is out of retention. Will be removed, but not closed. b = block.NewMockDatabaseBlock(ctrl) b.EXPECT().StartTime().Return(curr.Add(-2 * retentionPeriod)) - b.EXPECT().IsRetrieved().Return(true).AnyTimes() b.EXPECT().WasRetrievedFromDisk().Return(true) series.blocks.AddBlock(b) _, expiredBlockExists := series.blocks.BlockAt(curr.Add(-2 * retentionPeriod)) @@ -472,76 +435,6 @@ func TestSeriesTickCacheLRU(t *testing.T) { require.Equal(t, false, expiredBlockExists) } -func TestSeriesTickCacheAllMetadata(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - opts := newSeriesTestOptions() - opts = opts. - SetCachePolicy(CacheAllMetadata). - SetRetentionOptions(opts.RetentionOptions().SetBlockDataExpiryAfterNotAccessedPeriod(10 * time.Minute)) - ropts := opts.RetentionOptions() - curr := time.Now().Truncate(ropts.BlockSize()) - opts = opts.SetClockOptions(opts.ClockOptions().SetNowFn(func() time.Time { - return curr - })) - series := NewDatabaseSeries(ident.StringID("foo"), ident.Tags{}, opts).(*dbSeries) - blockRetriever := NewMockQueryableBlockRetriever(ctrl) - series.blockRetriever = blockRetriever - _, err := series.Bootstrap(nil) - assert.NoError(t, err) - - blockSize := time.Second * 1337 - - // Test case where block has been read within expiry period - won't be reset to only have metadata - b := block.NewMockDatabaseBlock(ctrl) - b.EXPECT().StartTime().Return(curr) - b.EXPECT().IsRetrieved().Return(true).AnyTimes() - b.EXPECT().HasMergeTarget().Return(true) - b.EXPECT().LastReadTime().Return( - curr.Add(-opts.RetentionOptions().BlockDataExpiryAfterNotAccessedPeriod() / 2)) - series.blocks.AddBlock(b) - - blockRetriever.EXPECT().IsBlockRetrievable(curr).Return(true) - - tickResult, err := series.Tick() - require.NoError(t, err) - require.Equal(t, 0, tickResult.UnwiredBlocks) - require.Equal(t, 1, tickResult.PendingMergeBlocks) - - // Test case where block has not been read within expiry period - will be reset to only have metadata - b = block.NewMockDatabaseBlock(ctrl) - b.EXPECT().BlockSize().Return(blockSize) - b.EXPECT().StartTime().Return(curr) - b.EXPECT().IsRetrieved().Return(true).AnyTimes() - b.EXPECT().LastReadTime().Return( - curr.Add(-opts.RetentionOptions().BlockDataExpiryAfterNotAccessedPeriod() * 2)) - b.EXPECT().Len().Return(1) - b.EXPECT().Checksum().Return(uint32(0), nil) - b.EXPECT().ResetRetrievable(curr, blockSize, blockRetriever, gomock.Any()).Return() - series.blocks.AddBlock(b) - - blockRetriever.EXPECT().IsBlockRetrievable(curr).Return(true) - - tickResult, err = series.Tick() - require.NoError(t, err) - require.Equal(t, 1, tickResult.UnwiredBlocks) - require.Equal(t, 0, tickResult.PendingMergeBlocks) - - // Test case where block is not flushed yet (not retrievable) - won't be reset to only have metadata - b = block.NewMockDatabaseBlock(ctrl) - b.EXPECT().StartTime().Return(curr) - b.EXPECT().HasMergeTarget().Return(true) - b.EXPECT().IsRetrieved().Return(true).AnyTimes() - series.blocks.AddBlock(b) - blockRetriever.EXPECT().IsBlockRetrievable(curr).Return(false) - - tickResult, err = series.Tick() - require.NoError(t, err) - require.Equal(t, 0, tickResult.UnwiredBlocks) - require.Equal(t, 1, tickResult.PendingMergeBlocks) -} - func TestSeriesTickCacheNone(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -710,7 +603,7 @@ func TestSeriesFetchBlocksMetadata(t *testing.T) { b.EXPECT().Checksum().Return(expectedChecksum, nil) expectedLastRead := time.Now() b.EXPECT().LastReadTime().Return(expectedLastRead) - b.EXPECT().IsCachedBlock().Return(false) + b.EXPECT().WasRetrievedFromDisk().Return(false) blocks[xtime.ToUnixNano(starts[0])] = b blocks[xtime.ToUnixNano(starts[3])] = nil diff --git a/src/dbnode/storage/series_wired_list_interaction_test.go b/src/dbnode/storage/series_wired_list_interaction_test.go index ad333991ab..e9ae475b96 100644 --- a/src/dbnode/storage/series_wired_list_interaction_test.go +++ b/src/dbnode/storage/series_wired_list_interaction_test.go @@ -108,7 +108,6 @@ func TestSeriesWiredListConcurrentInteractions(t *testing.T) { return default: bl := blPool.Get() - bl.ResetRetrievable(time.Time{}, blockSize, nil, block.RetrievableBlockMetadata{}) bl.Close() } } diff --git a/src/dbnode/storage/shard.go b/src/dbnode/storage/shard.go index afa4249ede..67a1a942b1 100644 --- a/src/dbnode/storage/shard.go +++ b/src/dbnode/storage/shard.go @@ -911,9 +911,6 @@ func (s *dbShard) ReadEncoded( case series.CacheAll: // No-op, would be in memory if cached return nil, nil - case series.CacheAllMetadata: - // No-op, would be in memory if metadata cached - return nil, nil } } else if err != nil { return nil, err @@ -1367,9 +1364,6 @@ func (s *dbShard) FetchBlocks( case series.CacheAll: // No-op, would be in memory if cached return nil, nil - case series.CacheAllMetadata: - // No-op, would be in memory if metadata cached - return nil, nil } } else if err != nil { return nil, err @@ -1442,35 +1436,6 @@ func (s *dbShard) fetchActiveBlocksMetadata( return res, nextIndexCursor, loopErr } -func (s *dbShard) FetchBlocksMetadata( - ctx context.Context, - start, end time.Time, - limit int64, - pageToken int64, - opts block.FetchBlocksMetadataOptions, -) (block.FetchBlocksMetadataResults, *int64, error) { - switch s.opts.SeriesCachePolicy() { - case series.CacheAll: - case series.CacheAllMetadata: - default: - // If not using CacheAll or CacheAllMetadata then calling the v1 - // API will only return active block metadata (mutable and cached) - // hence this call is invalid - return nil, nil, fmt.Errorf( - "fetch blocks metadata v1 endpoint invalid with cache policy: %s", - s.opts.SeriesCachePolicy().String()) - } - - // For v1 endpoint we always include cached blocks because when using - // CacheAllMetadata the blocks will appear cached - seriesFetchBlocksMetadataOpts := series.FetchBlocksMetadataOptions{ - FetchBlocksMetadataOptions: opts, - IncludeCachedBlocks: true, - } - return s.fetchActiveBlocksMetadata(ctx, start, end, - limit, pageToken, seriesFetchBlocksMetadataOpts) -} - func (s *dbShard) FetchBlocksMetadataV2( ctx context.Context, start, end time.Time, @@ -1489,7 +1454,7 @@ func (s *dbShard) FetchBlocksMetadataV2( flushedPhase := token.FlushedSeriesPhase cachePolicy := s.opts.SeriesCachePolicy() - if cachePolicy == series.CacheAll || cachePolicy == series.CacheAllMetadata { + if cachePolicy == series.CacheAll { // If we are using a series cache policy that caches all block metadata // in memory then we only ever perform the active phase as all metadata // is actively held in memory @@ -1497,8 +1462,7 @@ func (s *dbShard) FetchBlocksMetadataV2( if activePhase != nil { indexCursor = activePhase.IndexCursor } - // We always include cached blocks because when using - // CacheAllMetadata the blocks will appear cached + // We always include cached blocks seriesFetchBlocksMetadataOpts := series.FetchBlocksMetadataOptions{ FetchBlocksMetadataOptions: opts, IncludeCachedBlocks: true, @@ -1746,9 +1710,9 @@ func (s *dbShard) Bootstrap( // them for insertion. // FOLLOWUP(r): Audit places that keep refs to the ID from a // bootstrap result, newShardEntry copies it but some of the - // bootstrapped blocks when using all_metadata and perhaps - // another series cache policy keeps refs to the ID with - // retrieveID, so for now these IDs will be garbage collected) + // bootstrapped blocks when using certain series cache policies + // keeps refs to the ID with seriesID, so for now these IDs will + // be garbage collected) dbBlocks.Tags.Finalize() } diff --git a/src/dbnode/storage/shard_fetch_blocks_metadata_test.go b/src/dbnode/storage/shard_fetch_blocks_metadata_test.go index de05e49dc8..36e9fd034d 100644 --- a/src/dbnode/storage/shard_fetch_blocks_metadata_test.go +++ b/src/dbnode/storage/shard_fetch_blocks_metadata_test.go @@ -42,64 +42,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestShardFetchBlocksMetadata(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - opts := testDatabaseOptions() - ctx := opts.ContextPool().Get() - defer ctx.Close() - - shard := testDatabaseShard(t, opts) - defer shard.Close() - start := time.Now() - end := start.Add(defaultTestRetentionOpts.BlockSize()) - - var ids []ident.ID - fetchOpts := block.FetchBlocksMetadataOptions{ - IncludeSizes: true, - IncludeChecksums: true, - IncludeLastRead: true, - } - seriesFetchOpts := series.FetchBlocksMetadataOptions{ - FetchBlocksMetadataOptions: fetchOpts, - IncludeCachedBlocks: true, - } - lastRead := time.Now().Add(-time.Minute) - for i := 0; i < 10; i++ { - id := ident.StringID(fmt.Sprintf("foo.%d", i)) - tags := ident.NewTags( - ident.StringTag("aaa", "bbb"), - ident.StringTag("ccc", "ddd"), - ) - tagsIter := ident.NewTagsIterator(tags) - series := addMockSeries(ctrl, shard, id, tags, uint64(i)) - if i == 2 { - series.EXPECT(). - FetchBlocksMetadata(gomock.Not(nil), start, end, seriesFetchOpts). - Return(block.NewFetchBlocksMetadataResult(id, tagsIter, - block.NewFetchBlockMetadataResults()), nil) - } else if i > 2 && i <= 7 { - ids = append(ids, id) - blocks := block.NewFetchBlockMetadataResults() - at := start.Add(time.Duration(i)) - blocks.Add(block.NewFetchBlockMetadataResult(at, 0, nil, lastRead, nil)) - series.EXPECT(). - FetchBlocksMetadata(gomock.Not(nil), start, end, seriesFetchOpts). - Return(block.NewFetchBlocksMetadataResult(id, tagsIter, - blocks), nil) - } - } - - res, nextPageToken, err := shard.FetchBlocksMetadata(ctx, start, end, 5, 2, fetchOpts) - require.NoError(t, err) - require.Equal(t, len(ids), len(res.Results())) - require.Equal(t, int64(8), *nextPageToken) - for i := 0; i < len(res.Results()); i++ { - require.Equal(t, ids[i], res.Results()[i].ID) - } -} - func TestShardFetchBlocksMetadataV2WithSeriesCachePolicyCacheAll(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() diff --git a/src/dbnode/storage/shard_race_prop_test.go b/src/dbnode/storage/shard_race_prop_test.go index 6f01003384..a3fc2a386f 100644 --- a/src/dbnode/storage/shard_race_prop_test.go +++ b/src/dbnode/storage/shard_race_prop_test.go @@ -55,7 +55,7 @@ func TestShardTickReadFnRace(t *testing.T) { }, anyIDs().WithLabel("ids"), gen.UInt8().WithLabel("tickBatchSize").SuchThat(func(x uint8) bool { return x > 0 }), - gen.OneConstOf(fetchBlocksMetadataShardFn, fetchBlocksMetadataV2ShardFn), + gen.OneConstOf(fetchBlocksMetadataV2ShardFn), )) reporter := gopter.NewFormatedReporter(true, 160, os.Stdout) @@ -92,18 +92,6 @@ func testShardTickReadFnRace(t *testing.T, ids []ident.ID, tickBatchSize int, fn type testShardReadFn func(shard *dbShard) -var fetchBlocksMetadataShardFn testShardReadFn = func(shard *dbShard) { - ctx := context.NewContext() - start := time.Time{} - end := time.Now() - shard.FetchBlocksMetadata(ctx, start, end, 100, 0, block.FetchBlocksMetadataOptions{ - IncludeChecksums: true, - IncludeLastRead: true, - IncludeSizes: true, - }) - ctx.BlockingClose() -} - var fetchBlocksMetadataV2ShardFn testShardReadFn = func(shard *dbShard) { ctx := context.NewContext() start := time.Time{} diff --git a/src/dbnode/storage/storage_mock.go b/src/dbnode/storage/storage_mock.go index 0bae3e50a3..a70330f9f1 100644 --- a/src/dbnode/storage/storage_mock.go +++ b/src/dbnode/storage/storage_mock.go @@ -235,20 +235,6 @@ func (mr *MockDatabaseMockRecorder) FetchBlocks(ctx, namespace, shard, id, start return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchBlocks", reflect.TypeOf((*MockDatabase)(nil).FetchBlocks), ctx, namespace, shard, id, starts) } -// FetchBlocksMetadata mocks base method -func (m *MockDatabase) FetchBlocksMetadata(ctx context.Context, namespace ident.ID, shard uint32, start, end time.Time, limit, pageToken int64, opts block.FetchBlocksMetadataOptions) (block.FetchBlocksMetadataResults, *int64, error) { - ret := m.ctrl.Call(m, "FetchBlocksMetadata", ctx, namespace, shard, start, end, limit, pageToken, opts) - ret0, _ := ret[0].(block.FetchBlocksMetadataResults) - ret1, _ := ret[1].(*int64) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// FetchBlocksMetadata indicates an expected call of FetchBlocksMetadata -func (mr *MockDatabaseMockRecorder) FetchBlocksMetadata(ctx, namespace, shard, start, end, limit, pageToken, opts interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchBlocksMetadata", reflect.TypeOf((*MockDatabase)(nil).FetchBlocksMetadata), ctx, namespace, shard, start, end, limit, pageToken, opts) -} - // FetchBlocksMetadataV2 mocks base method func (m *MockDatabase) FetchBlocksMetadataV2(ctx context.Context, namespace ident.ID, shard uint32, start, end time.Time, limit int64, pageToken PageToken, opts block.FetchBlocksMetadataOptions) (block.FetchBlocksMetadataResults, PageToken, error) { ret := m.ctrl.Call(m, "FetchBlocksMetadataV2", ctx, namespace, shard, start, end, limit, pageToken, opts) @@ -517,20 +503,6 @@ func (mr *MockdatabaseMockRecorder) FetchBlocks(ctx, namespace, shard, id, start return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchBlocks", reflect.TypeOf((*Mockdatabase)(nil).FetchBlocks), ctx, namespace, shard, id, starts) } -// FetchBlocksMetadata mocks base method -func (m *Mockdatabase) FetchBlocksMetadata(ctx context.Context, namespace ident.ID, shard uint32, start, end time.Time, limit, pageToken int64, opts block.FetchBlocksMetadataOptions) (block.FetchBlocksMetadataResults, *int64, error) { - ret := m.ctrl.Call(m, "FetchBlocksMetadata", ctx, namespace, shard, start, end, limit, pageToken, opts) - ret0, _ := ret[0].(block.FetchBlocksMetadataResults) - ret1, _ := ret[1].(*int64) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// FetchBlocksMetadata indicates an expected call of FetchBlocksMetadata -func (mr *MockdatabaseMockRecorder) FetchBlocksMetadata(ctx, namespace, shard, start, end, limit, pageToken, opts interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchBlocksMetadata", reflect.TypeOf((*Mockdatabase)(nil).FetchBlocksMetadata), ctx, namespace, shard, start, end, limit, pageToken, opts) -} - // FetchBlocksMetadataV2 mocks base method func (m *Mockdatabase) FetchBlocksMetadataV2(ctx context.Context, namespace ident.ID, shard uint32, start, end time.Time, limit int64, pageToken PageToken, opts block.FetchBlocksMetadataOptions) (block.FetchBlocksMetadataResults, PageToken, error) { ret := m.ctrl.Call(m, "FetchBlocksMetadataV2", ctx, namespace, shard, start, end, limit, pageToken, opts) @@ -907,20 +879,6 @@ func (mr *MockdatabaseNamespaceMockRecorder) FetchBlocks(ctx, shardID, id, start return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchBlocks", reflect.TypeOf((*MockdatabaseNamespace)(nil).FetchBlocks), ctx, shardID, id, starts) } -// FetchBlocksMetadata mocks base method -func (m *MockdatabaseNamespace) FetchBlocksMetadata(ctx context.Context, shardID uint32, start, end time.Time, limit, pageToken int64, opts block.FetchBlocksMetadataOptions) (block.FetchBlocksMetadataResults, *int64, error) { - ret := m.ctrl.Call(m, "FetchBlocksMetadata", ctx, shardID, start, end, limit, pageToken, opts) - ret0, _ := ret[0].(block.FetchBlocksMetadataResults) - ret1, _ := ret[1].(*int64) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// FetchBlocksMetadata indicates an expected call of FetchBlocksMetadata -func (mr *MockdatabaseNamespaceMockRecorder) FetchBlocksMetadata(ctx, shardID, start, end, limit, pageToken, opts interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchBlocksMetadata", reflect.TypeOf((*MockdatabaseNamespace)(nil).FetchBlocksMetadata), ctx, shardID, start, end, limit, pageToken, opts) -} - // FetchBlocksMetadataV2 mocks base method func (m *MockdatabaseNamespace) FetchBlocksMetadataV2(ctx context.Context, shardID uint32, start, end time.Time, limit int64, pageToken PageToken, opts block.FetchBlocksMetadataOptions) (block.FetchBlocksMetadataResults, PageToken, error) { ret := m.ctrl.Call(m, "FetchBlocksMetadataV2", ctx, shardID, start, end, limit, pageToken, opts) @@ -1272,20 +1230,6 @@ func (mr *MockdatabaseShardMockRecorder) FetchBlocks(ctx, id, starts interface{} return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchBlocks", reflect.TypeOf((*MockdatabaseShard)(nil).FetchBlocks), ctx, id, starts) } -// FetchBlocksMetadata mocks base method -func (m *MockdatabaseShard) FetchBlocksMetadata(ctx context.Context, start, end time.Time, limit, pageToken int64, opts block.FetchBlocksMetadataOptions) (block.FetchBlocksMetadataResults, *int64, error) { - ret := m.ctrl.Call(m, "FetchBlocksMetadata", ctx, start, end, limit, pageToken, opts) - ret0, _ := ret[0].(block.FetchBlocksMetadataResults) - ret1, _ := ret[1].(*int64) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// FetchBlocksMetadata indicates an expected call of FetchBlocksMetadata -func (mr *MockdatabaseShardMockRecorder) FetchBlocksMetadata(ctx, start, end, limit, pageToken, opts interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchBlocksMetadata", reflect.TypeOf((*MockdatabaseShard)(nil).FetchBlocksMetadata), ctx, start, end, limit, pageToken, opts) -} - // FetchBlocksMetadataV2 mocks base method func (m *MockdatabaseShard) FetchBlocksMetadataV2(ctx context.Context, start, end time.Time, limit int64, pageToken PageToken, opts block.FetchBlocksMetadataOptions) (block.FetchBlocksMetadataResults, PageToken, error) { ret := m.ctrl.Call(m, "FetchBlocksMetadataV2", ctx, start, end, limit, pageToken, opts) diff --git a/src/dbnode/storage/types.go b/src/dbnode/storage/types.go index 0e97a40fd3..df41cefcf0 100644 --- a/src/dbnode/storage/types.go +++ b/src/dbnode/storage/types.go @@ -128,19 +128,6 @@ type Database interface { starts []time.Time, ) ([]block.FetchBlockResult, error) - // FetchBlocksMetadata retrieves blocks metadata for a given shard, returns the - // fetched block metadata results, the next page token, and any error encountered. - // If we have fetched all the block metadata, we return nil as the next page token. - FetchBlocksMetadata( - ctx context.Context, - namespace ident.ID, - shard uint32, - start, end time.Time, - limit int64, - pageToken int64, - opts block.FetchBlocksMetadataOptions, - ) (block.FetchBlocksMetadataResults, *int64, error) - // FetchBlocksMetadata retrieves blocks metadata for a given shard, returns the // fetched block metadata results, the next page token, and any error encountered. // If we have fetched all the block metadata, we return nil as the next page token. @@ -269,16 +256,6 @@ type databaseNamespace interface { starts []time.Time, ) ([]block.FetchBlockResult, error) - // FetchBlocksMetadata retrieves the blocks metadata. - FetchBlocksMetadata( - ctx context.Context, - shardID uint32, - start, end time.Time, - limit int64, - pageToken int64, - opts block.FetchBlocksMetadataOptions, - ) (block.FetchBlocksMetadataResults, *int64, error) - // FetchBlocksMetadata retrieves blocks metadata. FetchBlocksMetadataV2( ctx context.Context, @@ -394,15 +371,6 @@ type databaseShard interface { starts []time.Time, ) ([]block.FetchBlockResult, error) - // FetchBlocksMetadata retrieves the blocks metadata. - FetchBlocksMetadata( - ctx context.Context, - start, end time.Time, - limit int64, - pageToken int64, - opts block.FetchBlocksMetadataOptions, - ) (block.FetchBlocksMetadataResults, *int64, error) - // FetchBlocksMetadataV2 retrieves blocks metadata. FetchBlocksMetadataV2( ctx context.Context,